X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/c110a9aeed738f1b2dd456950a8ee3018cd97fed..2d2897ef969c28bc7b8ae420d1d9ff7aaf5130e1:/sdk/cwl/tests/test_container.py diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py index bfffe6eacb..975fcdf8a3 100644 --- a/sdk/cwl/tests/test_container.py +++ b/sdk/cwl/tests/test_container.py @@ -16,6 +16,7 @@ import mock import unittest import os import functools +import threading import cwltool.process import cwltool.secrets import cwltool.load_tool @@ -62,6 +63,13 @@ class TestContainer(unittest.TestCase): cwltool.process._names = set() arv_docker_clear_cache() + def tearDown(self): + root_logger = logging.getLogger('') + + # Remove existing RuntimeStatusLoggingHandlers if they exist + handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)] + root_logger.handlers = handlers + def helper(self, runner, enable_reuse=True): document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION) @@ -75,7 +83,9 @@ class TestContainer(unittest.TestCase): "basedir": "", "make_fs_access": make_fs_access, "construct_tool_object": runner.arv_make_tool, - "fetcher_constructor": functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=fs_access) + "fetcher_constructor": functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=fs_access), + "loader": Loader({}), + "metadata": cmap({"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}) }) runtimeContext = arvados_cwl.context.ArvRuntimeContext( {"work_api": "containers", @@ -83,9 +93,11 @@ class TestContainer(unittest.TestCase): "name": "test_run_"+str(enable_reuse), "make_fs_access": make_fs_access, "tmpdir": "/tmp", + "outdir": "/tmp", "enable_reuse": enable_reuse, "priority": 500, - "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz" + "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz", + "workflow_eval_lock": threading.Condition(threading.RLock()) }) if isinstance(runner, mock.MagicMock): @@ -1170,13 +1182,13 @@ class TestContainer(unittest.TestCase): "baseCommand": "echo", "arguments": [], "id": "", - "cwlVersion": "v1.2", - "class": "CommandLineTool" + "cwlVersion": "v1.0", + "class": "org.w3id.cwl.cwl.CommandLineTool" }) loadingContext, runtimeContext = self.helper(runner, True) - arvtool = cwltool.load_tool.load_tool(tool, loadingContext) + arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext) arvtool.formatgraph = None container_request = { @@ -1187,7 +1199,7 @@ class TestContainer(unittest.TestCase): 'name': 'test_run_True', 'runtime_constraints': { 'vcpus': 1, - 'ram': 268435456 + 'ram': 1073741824, }, 'use_existing': True, 'priority': 500, @@ -1229,6 +1241,103 @@ class TestContainer(unittest.TestCase): body=JsonDiffMatcher(container_request)) + # The test passes no builder.resources + # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024} + @mock.patch("arvados.commands.keepdocker.list_images_in_arv") + def test_run_preemptible_hint(self, keepdocker): + arvados_cwl.add_arv_hints() + for enable_preemptible in (None, True, False): + for preemptible_hint in (None, True, False): + arv_docker_clear_cache() + + runner = mock.MagicMock() + runner.ignore_docker_for_reuse = False + runner.intermediate_output_ttl = 0 + runner.secret_store = cwltool.secrets.SecretStore() + runner.api._rootDesc = {"revision": "20210628"} + + keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] + runner.api.collections().get().execute.return_value = { + "portable_data_hash": "99999999999999999999999999999993+99"} + + if preemptible_hint is not None: + hints = [{ + "class": "http://arvados.org/cwl#UsePreemptible", + "usePreemptible": preemptible_hint + }] + else: + hints = [] + + tool = cmap({ + "inputs": [], + "outputs": [], + "baseCommand": "ls", + "arguments": [{"valueFrom": "$(runtime.outdir)"}], + "id": "", + "class": "CommandLineTool", + "cwlVersion": "v1.2", + "hints": hints + }) + + loadingContext, runtimeContext = self.helper(runner) + + runtimeContext.name = 'test_run_enable_preemptible_'+str(enable_preemptible)+str(preemptible_hint) + runtimeContext.enable_preemptible = enable_preemptible + + arvtool = cwltool.load_tool.load_tool(tool, loadingContext) + arvtool.formatgraph = None + + # Test the interactions between --enable/disable-preemptible + # and UsePreemptible hint + + if enable_preemptible is None: + if preemptible_hint is None: + sched = {} + else: + sched = {'preemptible': preemptible_hint} + else: + if preemptible_hint is None: + sched = {'preemptible': enable_preemptible} + else: + sched = {'preemptible': enable_preemptible and preemptible_hint} + + for j in arvtool.job({}, mock.MagicMock(), runtimeContext): + j.run(runtimeContext) + runner.api.container_requests().create.assert_called_with( + body=JsonDiffMatcher({ + 'environment': { + 'HOME': '/var/spool/cwl', + 'TMPDIR': '/tmp' + }, + 'name': runtimeContext.name, + 'runtime_constraints': { + 'vcpus': 1, + 'ram': 268435456 + }, + 'use_existing': True, + 'priority': 500, + 'mounts': { + '/tmp': {'kind': 'tmp', + "capacity": 1073741824 + }, + '/var/spool/cwl': {'kind': 'tmp', + "capacity": 1073741824 } + }, + 'state': 'Committed', + 'output_name': 'Output for step '+runtimeContext.name, + 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', + 'output_path': '/var/spool/cwl', + 'output_ttl': 0, + 'container_image': '99999999999999999999999999999993+99', + 'command': ['ls', '/var/spool/cwl'], + 'cwd': '/var/spool/cwl', + 'scheduling_parameters': sched, + 'properties': {}, + 'secret_mounts': {}, + 'output_storage_classes': ["default"] + })) + + class TestWorkflow(unittest.TestCase): def setUp(self): @@ -1283,6 +1392,8 @@ class TestWorkflow(unittest.TestCase): runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999993+99"}]} + runner.api.containers().current().execute.return_value = {} + runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz" runner.ignore_docker_for_reuse = False runner.num_retries = 0