X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/cef860fc944f66b4b52aa2ea6d66561944a7c065..ba34a22d9918ae97306472c04701e69090821c82:/sdk/cwl/tests/test_container.py diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py index 09983f87a2..75371e2b78 100644 --- a/sdk/cwl/tests/test_container.py +++ b/sdk/cwl/tests/test_container.py @@ -16,8 +16,10 @@ import mock import unittest import os import functools +import threading import cwltool.process import cwltool.secrets +import cwltool.load_tool from cwltool.update import INTERNAL_VERSION from schema_salad.ref_resolver import Loader from schema_salad.sourceline import cmap @@ -61,28 +63,48 @@ class TestContainer(unittest.TestCase): cwltool.process._names = set() arv_docker_clear_cache() + def tearDown(self): + root_logger = logging.getLogger('') + + # Remove existing RuntimeStatusLoggingHandlers if they exist + handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)] + root_logger.handlers = handlers + def helper(self, runner, enable_reuse=True): document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION) make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0)) + fs_access = mock.MagicMock() + fs_access.exists.return_value = True + loadingContext = arvados_cwl.context.ArvLoadingContext( {"avsc_names": avsc_names, "basedir": "", "make_fs_access": make_fs_access, + "construct_tool_object": runner.arv_make_tool, + "fetcher_constructor": functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=fs_access), "loader": Loader({}), - "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}}) + "metadata": cmap({"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}) + }) runtimeContext = arvados_cwl.context.ArvRuntimeContext( {"work_api": "containers", "basedir": "", "name": "test_run_"+str(enable_reuse), "make_fs_access": make_fs_access, "tmpdir": "/tmp", + "outdir": "/tmp", "enable_reuse": enable_reuse, "priority": 500, - "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz" + "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz", + "workflow_eval_lock": threading.Condition(threading.RLock()) }) + if isinstance(runner, mock.MagicMock): + def make_tool(toolpath_object, loadingContext): + return arvados_cwl.ArvadosCommandTool(runner, toolpath_object, loadingContext) + runner.arv_make_tool.side_effect = make_tool + return loadingContext, runtimeContext # Helper function to set up the ArvCwlExecutor to use the containers api @@ -112,6 +134,7 @@ class TestContainer(unittest.TestCase): runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() + runner.api._rootDesc = {"revision": "20210628"} keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { @@ -122,13 +145,14 @@ class TestContainer(unittest.TestCase): "outputs": [], "baseCommand": "ls", "arguments": [{"valueFrom": "$(runtime.outdir)"}], - "id": "#", - "class": "CommandLineTool" + "id": "", + "class": "CommandLineTool", + "cwlVersion": "v1.2" }) loadingContext, runtimeContext = self.helper(runner, enable_reuse) - arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext) + arvtool = cwltool.load_tool.load_tool(tool, loadingContext) arvtool.formatgraph = None for j in arvtool.job({}, mock.MagicMock(), runtimeContext): @@ -142,7 +166,7 @@ class TestContainer(unittest.TestCase): 'name': 'test_run_'+str(enable_reuse), 'runtime_constraints': { 'vcpus': 1, - 'ram': 1073741824 + 'ram': 268435456 }, 'use_existing': enable_reuse, 'priority': 500, @@ -154,7 +178,7 @@ class TestContainer(unittest.TestCase): "capacity": 1073741824 } }, 'state': 'Committed', - 'output_name': 'Output for step test_run_'+str(enable_reuse), + 'output_name': 'Output from step test_run_'+str(enable_reuse), 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'output_path': '/var/spool/cwl', 'output_ttl': 0, @@ -162,18 +186,22 @@ class TestContainer(unittest.TestCase): 'command': ['ls', '/var/spool/cwl'], 'cwd': '/var/spool/cwl', 'scheduling_parameters': {}, - 'properties': {}, - 'secret_mounts': {} + 'properties': {'cwl_input': {}}, + 'secret_mounts': {}, + 'output_storage_classes': ["default"] })) # The test passes some fields in builder.resources # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024} @mock.patch("arvados.commands.keepdocker.list_images_in_arv") def test_resource_requirements(self, keepdocker): + arvados_cwl.add_arv_hints() runner = mock.MagicMock() runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 3600 runner.secret_store = cwltool.secrets.SecretStore() + runner.api._rootDesc = {"revision": "20210628"} + runner.api.config.return_value = {"Containers": {"DefaultKeepCacheDisk": 0}} keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { @@ -200,18 +228,19 @@ class TestContainer(unittest.TestCase): "class": "http://arvados.org/cwl#IntermediateOutput", "outputTTL": 7200 }, { - "class": "http://arvados.org/cwl#ReuseRequirement", + "class": "WorkReuse", "enableReuse": False }], "baseCommand": "ls", - "id": "#", - "class": "CommandLineTool" + "id": "", + "class": "CommandLineTool", + "cwlVersion": "v1.2" }) loadingContext, runtimeContext = self.helper(runner) runtimeContext.name = "test_resource_requirements" - arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext) + arvtool = cwltool.load_tool.load_tool(tool, loadingContext) arvtool.formatgraph = None for j in arvtool.job({}, mock.MagicMock(), runtimeContext): j.run(runtimeContext) @@ -239,7 +268,7 @@ class TestContainer(unittest.TestCase): "capacity": 5242880000 } }, 'state': 'Committed', - 'output_name': 'Output for step test_resource_requirements', + 'output_name': 'Output from step test_resource_requirements', 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'output_path': '/var/spool/cwl', 'output_ttl': 7200, @@ -249,8 +278,9 @@ class TestContainer(unittest.TestCase): 'scheduling_parameters': { 'partitions': ['blurb'] }, - 'properties': {}, - 'secret_mounts': {} + 'properties': {'cwl_input': {}}, + 'secret_mounts': {}, + 'output_storage_classes': ["default"] } call_body = call_kwargs.get('body', None) @@ -268,6 +298,7 @@ class TestContainer(unittest.TestCase): runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() + runner.api._rootDesc = {"revision": "20210628"} keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { @@ -311,14 +342,16 @@ class TestContainer(unittest.TestCase): } ] }], "baseCommand": "ls", - "id": "#", - "class": "CommandLineTool" + "class": "CommandLineTool", + "cwlVersion": "v1.2", + "id": "" }) loadingContext, runtimeContext = self.helper(runner) runtimeContext.name = "test_initial_work_dir" - arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext) + arvtool = cwltool.load_tool.load_tool(tool, loadingContext) + arvtool.formatgraph = None for j in arvtool.job({}, mock.MagicMock(), runtimeContext): j.run(runtimeContext) @@ -338,7 +371,7 @@ class TestContainer(unittest.TestCase): 'name': 'test_initial_work_dir', 'runtime_constraints': { 'vcpus': 1, - 'ram': 1073741824 + 'ram': 268435456 }, 'use_existing': True, 'priority': 500, @@ -369,7 +402,7 @@ class TestContainer(unittest.TestCase): } }, 'state': 'Committed', - 'output_name': 'Output for step test_initial_work_dir', + 'output_name': 'Output from step test_initial_work_dir', 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'output_path': '/var/spool/cwl', 'output_ttl': 0, @@ -378,8 +411,9 @@ class TestContainer(unittest.TestCase): 'cwd': '/var/spool/cwl', 'scheduling_parameters': { }, - 'properties': {}, - 'secret_mounts': {} + 'properties': {'cwl_input': {}}, + 'secret_mounts': {}, + 'output_storage_classes': ["default"] } call_body = call_kwargs.get('body', None) @@ -395,6 +429,7 @@ class TestContainer(unittest.TestCase): runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() + runner.api._rootDesc = {"revision": "20210628"} keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { @@ -410,14 +445,15 @@ class TestContainer(unittest.TestCase): "stderr": "stderr.txt", "stdin": "/keep/99999999999999999999999999999996+99/file.txt", "arguments": [{"valueFrom": "$(runtime.outdir)"}], - "id": "#", - "class": "CommandLineTool" + "id": "", + "class": "CommandLineTool", + "cwlVersion": "v1.2" }) loadingContext, runtimeContext = self.helper(runner) runtimeContext.name = "test_run_redirect" - arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext) + arvtool = cwltool.load_tool.load_tool(tool, loadingContext) arvtool.formatgraph = None for j in arvtool.job({}, mock.MagicMock(), runtimeContext): j.run(runtimeContext) @@ -430,7 +466,7 @@ class TestContainer(unittest.TestCase): 'name': 'test_run_redirect', 'runtime_constraints': { 'vcpus': 1, - 'ram': 1073741824 + 'ram': 268435456 }, 'use_existing': True, 'priority': 500, @@ -454,7 +490,7 @@ class TestContainer(unittest.TestCase): }, }, 'state': 'Committed', - "output_name": "Output for step test_run_redirect", + "output_name": "Output from step test_run_redirect", 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'output_path': '/var/spool/cwl', 'output_ttl': 0, @@ -462,8 +498,9 @@ class TestContainer(unittest.TestCase): 'command': ['ls', '/var/spool/cwl'], 'cwd': '/var/spool/cwl', 'scheduling_parameters': {}, - 'properties': {}, - 'secret_mounts': {} + 'properties': {'cwl_input': {}}, + 'secret_mounts': {}, + 'output_storage_classes': ["default"] })) @mock.patch("arvados.collection.Collection") @@ -498,6 +535,7 @@ class TestContainer(unittest.TestCase): arvjob.successCodes = [0] arvjob.outdir = "/var/spool/cwl" arvjob.output_ttl = 3600 + arvjob.uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzz1" arvjob.collect_outputs.return_value = {"out": "stuff"} @@ -507,7 +545,8 @@ class TestContainer(unittest.TestCase): "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2", "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz", "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz", - "modified_at": "2017-05-26T12:01:22Z" + "modified_at": "2017-05-26T12:01:22Z", + "properties": {} }) self.assertFalse(api.collections().create.called) @@ -517,6 +556,10 @@ class TestContainer(unittest.TestCase): arvjob.output_callback.assert_called_with({"out": "stuff"}, "success") runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2") + runner.api.container_requests().update.assert_called_with(uuid="zzzzz-xvhdp-zzzzzzzzzzzzzz1", + body={'container_request': {'properties': {'cwl_output': {'out': 'stuff'}}}}) + + # Test to make sure we dont call runtime_status_update if we already did # some where higher up in the call stack @mock.patch("arvados_cwl.util.get_current_container") @@ -600,7 +643,8 @@ class TestContainer(unittest.TestCase): "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2", "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz", "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz", - "modified_at": "2017-05-26T12:01:22Z" + "modified_at": "2017-05-26T12:01:22Z", + "properties": {} }) rts_mock.assert_called_with( @@ -618,6 +662,7 @@ class TestContainer(unittest.TestCase): runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() + runner.api._rootDesc = {"revision": "20210628"} keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { @@ -634,14 +679,15 @@ class TestContainer(unittest.TestCase): "outputs": [], "baseCommand": "ls", "arguments": [{"valueFrom": "$(runtime.outdir)"}], - "id": "#", - "class": "CommandLineTool" + "id": "", + "class": "CommandLineTool", + "cwlVersion": "v1.2" }) loadingContext, runtimeContext = self.helper(runner) runtimeContext.name = "test_run_mounts" - arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext) + arvtool = cwltool.load_tool.load_tool(tool, loadingContext) arvtool.formatgraph = None job_order = { "p1": { @@ -671,7 +717,7 @@ class TestContainer(unittest.TestCase): 'name': 'test_run_mounts', 'runtime_constraints': { 'vcpus': 1, - 'ram': 1073741824 + 'ram': 268435456 }, 'use_existing': True, 'priority': 500, @@ -687,7 +733,7 @@ class TestContainer(unittest.TestCase): "capacity": 1073741824 } }, 'state': 'Committed', - 'output_name': 'Output for step test_run_mounts', + 'output_name': 'Output from step test_run_mounts', 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'output_path': '/var/spool/cwl', 'output_ttl': 0, @@ -695,18 +741,52 @@ class TestContainer(unittest.TestCase): 'command': ['ls', '/var/spool/cwl'], 'cwd': '/var/spool/cwl', 'scheduling_parameters': {}, - 'properties': {}, - 'secret_mounts': {} + 'properties': {'cwl_input': { + "p1": { + "basename": "99999999999999999999999999999994+44", + "class": "Directory", + "dirname": "/keep", + "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz", + "listing": [ + { + "basename": "file1", + "class": "File", + "dirname": "/keep/99999999999999999999999999999994+44", + "location": "keep:99999999999999999999999999999994+44/file1", + "nameext": "", + "nameroot": "file1", + "path": "/keep/99999999999999999999999999999994+44/file1", + "size": 0 + }, + { + "basename": "file2", + "class": "File", + "dirname": "/keep/99999999999999999999999999999994+44", + "location": "keep:99999999999999999999999999999994+44/file2", + "nameext": "", + "nameroot": "file2", + "path": "/keep/99999999999999999999999999999994+44/file2", + "size": 0 + } + ], + "location": "keep:99999999999999999999999999999994+44", + "path": "/keep/99999999999999999999999999999994+44" + } + }}, + 'secret_mounts': {}, + 'output_storage_classes': ["default"] })) # The test passes no builder.resources # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024} @mock.patch("arvados.commands.keepdocker.list_images_in_arv") def test_secrets(self, keepdocker): + arvados_cwl.add_arv_hints() runner = mock.MagicMock() runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() + runner.api._rootDesc = {"revision": "20210628"} keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { @@ -716,6 +796,7 @@ class TestContainer(unittest.TestCase): tool = cmap({"arguments": ["md5sum", "example.conf"], "class": "CommandLineTool", + "cwlVersion": "v1.2", "hints": [ { "class": "http://commonwl.org/cwltool#Secrets", @@ -724,7 +805,7 @@ class TestContainer(unittest.TestCase): ] } ], - "id": "#secret_job.cwl", + "id": "", "inputs": [ { "id": "#secret_job.cwl/pw", @@ -748,7 +829,7 @@ class TestContainer(unittest.TestCase): loadingContext, runtimeContext = self.helper(runner) runtimeContext.name = "test_secrets" - arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext) + arvtool = cwltool.load_tool.load_tool(tool, loadingContext) arvtool.formatgraph = None job_order = {"pw": "blorp"} @@ -765,7 +846,7 @@ class TestContainer(unittest.TestCase): 'name': 'test_secrets', 'runtime_constraints': { 'vcpus': 1, - 'ram': 1073741824 + 'ram': 268435456 }, 'use_existing': True, 'priority': 500, @@ -777,7 +858,7 @@ class TestContainer(unittest.TestCase): "capacity": 1073741824 } }, 'state': 'Committed', - 'output_name': 'Output for step test_secrets', + 'output_name': 'Output from step test_secrets', 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'output_path': '/var/spool/cwl', 'output_ttl': 0, @@ -785,13 +866,14 @@ class TestContainer(unittest.TestCase): 'command': ['md5sum', 'example.conf'], 'cwd': '/var/spool/cwl', 'scheduling_parameters': {}, - 'properties': {}, + 'properties': {'cwl_input': job_order}, "secret_mounts": { "/var/spool/cwl/example.conf": { "content": "username: user\npassword: blorp\n", "kind": "text" } - } + }, + 'output_storage_classes': ["default"] })) # The test passes no builder.resources @@ -802,6 +884,7 @@ class TestContainer(unittest.TestCase): runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() + runner.api._rootDesc = {"revision": "20210628"} keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { @@ -812,7 +895,8 @@ class TestContainer(unittest.TestCase): "outputs": [], "baseCommand": "ls", "arguments": [{"valueFrom": "$(runtime.outdir)"}], - "id": "#", + "id": "", + "cwlVersion": "v1.2", "class": "CommandLineTool", "hints": [ { @@ -825,7 +909,7 @@ class TestContainer(unittest.TestCase): loadingContext, runtimeContext = self.helper(runner) runtimeContext.name = "test_timelimit" - arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext) + arvtool = cwltool.load_tool.load_tool(tool, loadingContext) arvtool.formatgraph = None for j in arvtool.job({}, mock.MagicMock(), runtimeContext): @@ -835,6 +919,517 @@ class TestContainer(unittest.TestCase): self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time')) + # The test passes no builder.resources + # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024} + @mock.patch("arvados.commands.keepdocker.list_images_in_arv") + def test_setting_storage_class(self, keepdocker): + arv_docker_clear_cache() + + runner = mock.MagicMock() + runner.ignore_docker_for_reuse = False + runner.intermediate_output_ttl = 0 + runner.secret_store = cwltool.secrets.SecretStore() + runner.api._rootDesc = {"revision": "20210628"} + + keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] + runner.api.collections().get().execute.return_value = { + "portable_data_hash": "99999999999999999999999999999993+99"} + + tool = cmap({ + "inputs": [], + "outputs": [], + "baseCommand": "ls", + "arguments": [{"valueFrom": "$(runtime.outdir)"}], + "id": "", + "cwlVersion": "v1.2", + "class": "CommandLineTool", + "hints": [ + { + "class": "http://arvados.org/cwl#OutputStorageClass", + "finalStorageClass": ["baz_sc", "qux_sc"], + "intermediateStorageClass": ["foo_sc", "bar_sc"] + } + ] + }) + + loadingContext, runtimeContext = self.helper(runner, True) + + arvtool = cwltool.load_tool.load_tool(tool, loadingContext) + arvtool.formatgraph = None + + for j in arvtool.job({}, mock.MagicMock(), runtimeContext): + j.run(runtimeContext) + runner.api.container_requests().create.assert_called_with( + body=JsonDiffMatcher({ + 'environment': { + 'HOME': '/var/spool/cwl', + 'TMPDIR': '/tmp' + }, + 'name': 'test_run_True', + 'runtime_constraints': { + 'vcpus': 1, + 'ram': 268435456 + }, + 'use_existing': True, + 'priority': 500, + 'mounts': { + '/tmp': {'kind': 'tmp', + "capacity": 1073741824 + }, + '/var/spool/cwl': {'kind': 'tmp', + "capacity": 1073741824 } + }, + 'state': 'Committed', + 'output_name': 'Output from step test_run_True', + 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', + 'output_path': '/var/spool/cwl', + 'output_ttl': 0, + 'container_image': '99999999999999999999999999999993+99', + 'command': ['ls', '/var/spool/cwl'], + 'cwd': '/var/spool/cwl', + 'scheduling_parameters': {}, + 'properties': {'cwl_input': {}}, + 'secret_mounts': {}, + 'output_storage_classes': ["foo_sc", "bar_sc"] + })) + + + # The test passes no builder.resources + # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024} + @mock.patch("arvados.commands.keepdocker.list_images_in_arv") + def test_setting_process_properties(self, keepdocker): + arv_docker_clear_cache() + + runner = mock.MagicMock() + runner.ignore_docker_for_reuse = False + runner.intermediate_output_ttl = 0 + runner.secret_store = cwltool.secrets.SecretStore() + runner.api._rootDesc = {"revision": "20210628"} + + keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] + runner.api.collections().get().execute.return_value = { + "portable_data_hash": "99999999999999999999999999999993+99"} + + tool = cmap({ + "inputs": [ + {"id": "x", "type": "string"}], + "outputs": [], + "baseCommand": "ls", + "arguments": [{"valueFrom": "$(runtime.outdir)"}], + "id": "", + "class": "CommandLineTool", + "cwlVersion": "v1.2", + "hints": [ + { + "class": "http://arvados.org/cwl#ProcessProperties", + "processProperties": [ + {"propertyName": "foo", + "propertyValue": "bar"}, + {"propertyName": "baz", + "propertyValue": "$(inputs.x)"}, + {"propertyName": "quux", + "propertyValue": { + "q1": 1, + "q2": 2 + } + } + ], + } + ] + }) + + loadingContext, runtimeContext = self.helper(runner, True) + + arvtool = cwltool.load_tool.load_tool(tool, loadingContext) + arvtool.formatgraph = None + + for j in arvtool.job({"x": "blorp"}, mock.MagicMock(), runtimeContext): + j.run(runtimeContext) + runner.api.container_requests().create.assert_called_with( + body=JsonDiffMatcher({ + 'environment': { + 'HOME': '/var/spool/cwl', + 'TMPDIR': '/tmp' + }, + 'name': 'test_run_True', + 'runtime_constraints': { + 'vcpus': 1, + 'ram': 268435456 + }, + 'use_existing': True, + 'priority': 500, + 'mounts': { + '/tmp': {'kind': 'tmp', + "capacity": 1073741824 + }, + '/var/spool/cwl': {'kind': 'tmp', + "capacity": 1073741824 } + }, + 'state': 'Committed', + 'output_name': 'Output from step test_run_True', + 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', + 'output_path': '/var/spool/cwl', + 'output_ttl': 0, + 'container_image': '99999999999999999999999999999993+99', + 'command': ['ls', '/var/spool/cwl'], + 'cwd': '/var/spool/cwl', + 'scheduling_parameters': {}, + 'properties': { + "baz": "blorp", + "cwl_input": {"x": "blorp"}, + "foo": "bar", + "quux": { + "q1": 1, + "q2": 2 + } + }, + 'secret_mounts': {}, + 'output_storage_classes': ["default"] + })) + + + # The test passes no builder.resources + # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024} + @mock.patch("arvados.commands.keepdocker.list_images_in_arv") + def test_cuda_requirement(self, keepdocker): + arvados_cwl.add_arv_hints() + arv_docker_clear_cache() + + runner = mock.MagicMock() + runner.ignore_docker_for_reuse = False + runner.intermediate_output_ttl = 0 + runner.secret_store = cwltool.secrets.SecretStore() + runner.api._rootDesc = {"revision": "20210628"} + + keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] + runner.api.collections().get().execute.return_value = { + "portable_data_hash": "99999999999999999999999999999993+99"} + + test_cwl_req = [{ + "class": "http://commonwl.org/cwltool#CUDARequirement", + "cudaVersionMin": "11.0", + "cudaComputeCapability": "9.0", + }, { + "class": "http://commonwl.org/cwltool#CUDARequirement", + "cudaVersionMin": "11.0", + "cudaComputeCapability": "9.0", + "cudaDeviceCountMin": 2 + }, { + "class": "http://commonwl.org/cwltool#CUDARequirement", + "cudaVersionMin": "11.0", + "cudaComputeCapability": ["4.0", "5.0"], + "cudaDeviceCountMin": 2 + }] + + test_arv_req = [{ + 'device_count': 1, + 'driver_version': "11.0", + 'hardware_capability': "9.0" + }, { + 'device_count': 2, + 'driver_version': "11.0", + 'hardware_capability': "9.0" + }, { + 'device_count': 2, + 'driver_version': "11.0", + 'hardware_capability': "4.0" + }] + + for test_case in range(0, len(test_cwl_req)): + + tool = cmap({ + "inputs": [], + "outputs": [], + "baseCommand": "nvidia-smi", + "arguments": [], + "id": "", + "cwlVersion": "v1.2", + "class": "CommandLineTool", + "requirements": [test_cwl_req[test_case]] + }) + + loadingContext, runtimeContext = self.helper(runner, True) + + arvtool = cwltool.load_tool.load_tool(tool, loadingContext) + arvtool.formatgraph = None + + for j in arvtool.job({}, mock.MagicMock(), runtimeContext): + j.run(runtimeContext) + runner.api.container_requests().create.assert_called_with( + body=JsonDiffMatcher({ + 'environment': { + 'HOME': '/var/spool/cwl', + 'TMPDIR': '/tmp' + }, + 'name': 'test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)), + 'runtime_constraints': { + 'vcpus': 1, + 'ram': 268435456, + 'cuda': test_arv_req[test_case] + }, + 'use_existing': True, + 'priority': 500, + 'mounts': { + '/tmp': {'kind': 'tmp', + "capacity": 1073741824 + }, + '/var/spool/cwl': {'kind': 'tmp', + "capacity": 1073741824 } + }, + 'state': 'Committed', + 'output_name': 'Output from step test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)), + 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', + 'output_path': '/var/spool/cwl', + 'output_ttl': 0, + 'container_image': '99999999999999999999999999999993+99', + 'command': ['nvidia-smi'], + 'cwd': '/var/spool/cwl', + 'scheduling_parameters': {}, + 'properties': {'cwl_input': {}}, + 'secret_mounts': {}, + 'output_storage_classes': ["default"] + })) + + + # The test passes no builder.resources + # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024} + @mock.patch("arvados_cwl.arvdocker.determine_image_id") + @mock.patch("arvados.commands.keepdocker.list_images_in_arv") + def test_match_local_docker(self, keepdocker, determine_image_id): + arvados_cwl.add_arv_hints() + arv_docker_clear_cache() + + runner = mock.MagicMock() + runner.ignore_docker_for_reuse = False + runner.intermediate_output_ttl = 0 + runner.secret_store = cwltool.secrets.SecretStore() + runner.api._rootDesc = {"revision": "20210628"} + + keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz4", {"dockerhash": "456"}), + ("zzzzz-4zz18-zzzzzzzzzzzzzz3", {"dockerhash": "123"})] + determine_image_id.side_effect = lambda x: "123" + def execute(uuid): + ex = mock.MagicMock() + lookup = {"zzzzz-4zz18-zzzzzzzzzzzzzz4": {"portable_data_hash": "99999999999999999999999999999994+99"}, + "zzzzz-4zz18-zzzzzzzzzzzzzz3": {"portable_data_hash": "99999999999999999999999999999993+99"}} + ex.execute.return_value = lookup[uuid] + return ex + runner.api.collections().get.side_effect = execute + + tool = cmap({ + "inputs": [], + "outputs": [], + "baseCommand": "echo", + "arguments": [], + "id": "", + "cwlVersion": "v1.0", + "class": "org.w3id.cwl.cwl.CommandLineTool" + }) + + loadingContext, runtimeContext = self.helper(runner, True) + + arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext) + arvtool.formatgraph = None + + container_request = { + 'environment': { + 'HOME': '/var/spool/cwl', + 'TMPDIR': '/tmp' + }, + 'name': 'test_run_True', + 'runtime_constraints': { + 'vcpus': 1, + 'ram': 1073741824, + }, + 'use_existing': True, + 'priority': 500, + 'mounts': { + '/tmp': {'kind': 'tmp', + "capacity": 1073741824 + }, + '/var/spool/cwl': {'kind': 'tmp', + "capacity": 1073741824 } + }, + 'state': 'Committed', + 'output_name': 'Output from step test_run_True', + 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', + 'output_path': '/var/spool/cwl', + 'output_ttl': 0, + 'container_image': '99999999999999999999999999999994+99', + 'command': ['echo'], + 'cwd': '/var/spool/cwl', + 'scheduling_parameters': {}, + 'properties': {'cwl_input': {}}, + 'secret_mounts': {}, + 'output_storage_classes': ["default"] + } + + runtimeContext.match_local_docker = False + for j in arvtool.job({}, mock.MagicMock(), runtimeContext): + j.run(runtimeContext) + runner.api.container_requests().create.assert_called_with( + body=JsonDiffMatcher(container_request)) + + arv_docker_clear_cache() + runtimeContext.match_local_docker = True + container_request['container_image'] = '99999999999999999999999999999993+99' + container_request['name'] = 'test_run_True_2' + container_request['output_name'] = 'Output from step test_run_True_2' + for j in arvtool.job({}, mock.MagicMock(), runtimeContext): + j.run(runtimeContext) + runner.api.container_requests().create.assert_called_with( + body=JsonDiffMatcher(container_request)) + + + # The test passes no builder.resources + # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024} + @mock.patch("arvados.commands.keepdocker.list_images_in_arv") + def test_run_preemptible_hint(self, keepdocker): + arvados_cwl.add_arv_hints() + for enable_preemptible in (None, True, False): + for preemptible_hint in (None, True, False): + arv_docker_clear_cache() + + runner = mock.MagicMock() + runner.ignore_docker_for_reuse = False + runner.intermediate_output_ttl = 0 + runner.secret_store = cwltool.secrets.SecretStore() + runner.api._rootDesc = {"revision": "20210628"} + + keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] + runner.api.collections().get().execute.return_value = { + "portable_data_hash": "99999999999999999999999999999993+99"} + + if preemptible_hint is not None: + hints = [{ + "class": "http://arvados.org/cwl#UsePreemptible", + "usePreemptible": preemptible_hint + }] + else: + hints = [] + + tool = cmap({ + "inputs": [], + "outputs": [], + "baseCommand": "ls", + "arguments": [{"valueFrom": "$(runtime.outdir)"}], + "id": "", + "class": "CommandLineTool", + "cwlVersion": "v1.2", + "hints": hints + }) + + loadingContext, runtimeContext = self.helper(runner) + + runtimeContext.name = 'test_run_enable_preemptible_'+str(enable_preemptible)+str(preemptible_hint) + runtimeContext.enable_preemptible = enable_preemptible + + arvtool = cwltool.load_tool.load_tool(tool, loadingContext) + arvtool.formatgraph = None + + # Test the interactions between --enable/disable-preemptible + # and UsePreemptible hint + + if enable_preemptible is None: + if preemptible_hint is None: + sched = {} + else: + sched = {'preemptible': preemptible_hint} + else: + if preemptible_hint is None: + sched = {'preemptible': enable_preemptible} + else: + sched = {'preemptible': enable_preemptible and preemptible_hint} + + for j in arvtool.job({}, mock.MagicMock(), runtimeContext): + j.run(runtimeContext) + runner.api.container_requests().create.assert_called_with( + body=JsonDiffMatcher({ + 'environment': { + 'HOME': '/var/spool/cwl', + 'TMPDIR': '/tmp' + }, + 'name': runtimeContext.name, + 'runtime_constraints': { + 'vcpus': 1, + 'ram': 268435456 + }, + 'use_existing': True, + 'priority': 500, + 'mounts': { + '/tmp': {'kind': 'tmp', + "capacity": 1073741824 + }, + '/var/spool/cwl': {'kind': 'tmp', + "capacity": 1073741824 } + }, + 'state': 'Committed', + 'output_name': 'Output from step '+runtimeContext.name, + 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', + 'output_path': '/var/spool/cwl', + 'output_ttl': 0, + 'container_image': '99999999999999999999999999999993+99', + 'command': ['ls', '/var/spool/cwl'], + 'cwd': '/var/spool/cwl', + 'scheduling_parameters': sched, + 'properties': {'cwl_input': {}}, + 'secret_mounts': {}, + 'output_storage_classes': ["default"] + })) + + + @mock.patch("arvados.commands.keepdocker.list_images_in_arv") + def test_output_properties(self, keepdocker): + arvados_cwl.add_arv_hints() + for rev in ["20210628", "20220510"]: + runner = mock.MagicMock() + runner.ignore_docker_for_reuse = False + runner.intermediate_output_ttl = 0 + runner.secret_store = cwltool.secrets.SecretStore() + runner.api._rootDesc = {"revision": rev} + + keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] + runner.api.collections().get().execute.return_value = { + "portable_data_hash": "99999999999999999999999999999993+99"} + + tool = cmap({ + "inputs": [{ + "id": "inp", + "type": "string" + }], + "outputs": [], + "baseCommand": "ls", + "arguments": [{"valueFrom": "$(runtime.outdir)"}], + "id": "", + "cwlVersion": "v1.2", + "class": "CommandLineTool", + "hints": [ + { + "class": "http://arvados.org/cwl#OutputCollectionProperties", + "outputProperties": { + "foo": "bar", + "baz": "$(inputs.inp)" + } + } + ] + }) + + loadingContext, runtimeContext = self.helper(runner) + runtimeContext.name = "test_timelimit" + + arvtool = cwltool.load_tool.load_tool(tool, loadingContext) + arvtool.formatgraph = None + + for j in arvtool.job({"inp": "quux"}, mock.MagicMock(), runtimeContext): + j.run(runtimeContext) + + _, kwargs = runner.api.container_requests().create.call_args + if rev == "20220510": + self.assertEqual({"foo": "bar", "baz": "quux"}, kwargs['body'].get('output_properties')) + else: + self.assertEqual(None, kwargs['body'].get('output_properties')) + + class TestWorkflow(unittest.TestCase): def setUp(self): cwltool.process._names = set() @@ -888,6 +1483,8 @@ class TestWorkflow(unittest.TestCase): runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999993+99"}]} + runner.api.containers().current().execute.return_value = {} + runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz" runner.ignore_docker_for_reuse = False runner.num_retries = 0 @@ -960,11 +1557,23 @@ class TestWorkflow(unittest.TestCase): } }, "name": "scatterstep", - "output_name": "Output for step scatterstep", + "output_name": "Output from step scatterstep", "output_path": "/var/spool/cwl", "output_ttl": 0, "priority": 500, - "properties": {}, + "properties": {'cwl_input': { + "fileblub": { + "basename": "token.txt", + "class": "File", + "dirname": "/keep/99999999999999999999999999999999+118", + "location": "keep:99999999999999999999999999999999+118/token.txt", + "nameext": ".txt", + "nameroot": "token", + "path": "/keep/99999999999999999999999999999999+118/token.txt", + "size": 0 + }, + "sleeptime": 5 + }}, "runtime_constraints": { "ram": 1073741824, "vcpus": 1 @@ -972,7 +1581,8 @@ class TestWorkflow(unittest.TestCase): "scheduling_parameters": {}, "secret_mounts": {}, "state": "Committed", - "use_existing": True + "use_existing": True, + 'output_storage_classes': ["default"] })) mockc.open().__enter__().write.assert_has_calls([mock.call(subwf)]) mockc.open().__enter__().write.assert_has_calls([mock.call( @@ -1036,7 +1646,7 @@ class TestWorkflow(unittest.TestCase): 'name': u'echo-subwf', 'secret_mounts': {}, 'runtime_constraints': {'API': True, 'vcpus': 3, 'ram': 1073741824}, - 'properties': {}, + 'properties': {'cwl_input': {}}, 'priority': 500, 'mounts': { '/var/spool/cwl/cwl.input.yml': { @@ -1073,8 +1683,9 @@ class TestWorkflow(unittest.TestCase): u'cwl.input.yml' ], 'use_existing': True, - 'output_name': u'Output for step echo-subwf', - 'cwd': '/var/spool/cwl' + 'output_name': u'Output from step echo-subwf', + 'cwd': '/var/spool/cwl', + 'output_storage_classes': ["default"] })) def test_default_work_api(self):