X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/b9262c0e8c04f6b3d68f1ca301cc2ec56c2ef164..HEAD:/sdk/cwl/tests/test_container.py diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py index 798c5af289..6161670839 100644 --- a/sdk/cwl/tests/test_container.py +++ b/sdk/cwl/tests/test_container.py @@ -2,17 +2,13 @@ # # SPDX-License-Identifier: Apache-2.0 -from builtins import str -from builtins import object - import arvados_cwl import arvados_cwl.context import arvados_cwl.util -from arvados_cwl.arvdocker import arv_docker_clear_cache +#from arvados_cwl.arvdocker import arv_docker_clear_cache import copy import arvados.config import logging -import mock import unittest import os import functools @@ -23,6 +19,9 @@ import cwltool.load_tool from cwltool.update import INTERNAL_VERSION from schema_salad.ref_resolver import Loader from schema_salad.sourceline import cmap +import io + +from unittest import mock from .matcher import JsonDiffMatcher, StripYAMLComments from .mock_discovery import get_rootDesc @@ -61,7 +60,14 @@ class TestContainer(unittest.TestCase): def setUp(self): cwltool.process._names = set() - arv_docker_clear_cache() + #arv_docker_clear_cache() + + def tearDown(self): + root_logger = logging.getLogger('') + + # Remove existing RuntimeStatusLoggingHandlers if they exist + handlers = [h for h in root_logger.handlers if not isinstance(h, arvados_cwl.executor.RuntimeStatusLoggingHandler)] + root_logger.handlers = handlers def helper(self, runner, enable_reuse=True): document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION) @@ -78,7 +84,8 @@ class TestContainer(unittest.TestCase): "construct_tool_object": runner.arv_make_tool, "fetcher_constructor": functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=fs_access), "loader": Loader({}), - "metadata": cmap({"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}) + "metadata": cmap({"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}), + "default_docker_image": "arvados/jobs:"+arvados_cwl.__version__ }) runtimeContext = arvados_cwl.context.ArvRuntimeContext( {"work_api": "containers", @@ -121,13 +128,14 @@ class TestContainer(unittest.TestCase): @mock.patch("arvados.commands.keepdocker.list_images_in_arv") def test_run(self, keepdocker): for enable_reuse in (True, False): - arv_docker_clear_cache() + #arv_docker_clear_cache() runner = mock.MagicMock() runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() runner.api._rootDesc = {"revision": "20210628"} + runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}} keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { @@ -171,7 +179,7 @@ class TestContainer(unittest.TestCase): "capacity": 1073741824 } }, 'state': 'Committed', - 'output_name': 'Output for step test_run_'+str(enable_reuse), + 'output_name': 'Output from step test_run_'+str(enable_reuse), 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'output_path': '/var/spool/cwl', 'output_ttl': 0, @@ -179,7 +187,7 @@ class TestContainer(unittest.TestCase): 'command': ['ls', '/var/spool/cwl'], 'cwd': '/var/spool/cwl', 'scheduling_parameters': {}, - 'properties': {}, + 'properties': {'cwl_input': {}}, 'secret_mounts': {}, 'output_storage_classes': ["default"] })) @@ -194,6 +202,7 @@ class TestContainer(unittest.TestCase): runner.intermediate_output_ttl = 3600 runner.secret_store = cwltool.secrets.SecretStore() runner.api._rootDesc = {"revision": "20210628"} + runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}} keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { @@ -260,7 +269,7 @@ class TestContainer(unittest.TestCase): "capacity": 5242880000 } }, 'state': 'Committed', - 'output_name': 'Output for step test_resource_requirements', + 'output_name': 'Output from step test_resource_requirements', 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'output_path': '/var/spool/cwl', 'output_ttl': 7200, @@ -270,7 +279,7 @@ class TestContainer(unittest.TestCase): 'scheduling_parameters': { 'partitions': ['blurb'] }, - 'properties': {}, + 'properties': {'cwl_input': {}}, 'secret_mounts': {}, 'output_storage_classes': ["default"] } @@ -291,6 +300,7 @@ class TestContainer(unittest.TestCase): runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() runner.api._rootDesc = {"revision": "20210628"} + runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}} keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { @@ -394,7 +404,7 @@ class TestContainer(unittest.TestCase): } }, 'state': 'Committed', - 'output_name': 'Output for step test_initial_work_dir', + 'output_name': 'Output from step test_initial_work_dir', 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'output_path': '/var/spool/cwl', 'output_ttl': 0, @@ -403,7 +413,7 @@ class TestContainer(unittest.TestCase): 'cwd': '/var/spool/cwl', 'scheduling_parameters': { }, - 'properties': {}, + 'properties': {'cwl_input': {}}, 'secret_mounts': {}, 'output_storage_classes': ["default"] } @@ -422,6 +432,7 @@ class TestContainer(unittest.TestCase): runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() runner.api._rootDesc = {"revision": "20210628"} + runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}} keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { @@ -482,7 +493,7 @@ class TestContainer(unittest.TestCase): }, }, 'state': 'Committed', - "output_name": "Output for step test_run_redirect", + "output_name": "Output from step test_run_redirect", 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'output_path': '/var/spool/cwl', 'output_ttl': 0, @@ -490,7 +501,7 @@ class TestContainer(unittest.TestCase): 'command': ['ls', '/var/spool/cwl'], 'cwd': '/var/spool/cwl', 'scheduling_parameters': {}, - 'properties': {}, + 'properties': {'cwl_input': {}}, 'secret_mounts': {}, 'output_storage_classes': ["default"] })) @@ -506,16 +517,53 @@ class TestContainer(unittest.TestCase): runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() + runner.api.container_requests().get().execute.return_value = {"container_uuid":"zzzzz-xvhdp-zzzzzzzzzzzzzzz"} + runner.api.containers().get().execute.return_value = {"state":"Complete", "output": "abc+123", "exit_code": 0} - col().open.return_value = [] + # Need to noop-out the close method otherwise it gets + # discarded when closed and we can't call getvalue() to check + # it. + class NoopCloseStringIO(io.StringIO): + def close(self): + pass + + usage_report = NoopCloseStringIO() + def colreader_action(name, mode): + nonlocal usage_report + if name == "node.json": + return io.StringIO("""{ + "ProviderType": "c5.large", + "VCPUs": 2, + "RAM": 4294967296, + "IncludedScratch": 8000000000000, + "AddedScratch": 0, + "Price": 0.085, + "Preemptible": false, + "CUDA": { + "DriverVersion": "", + "HardwareCapability": "", + "DeviceCount": 0 + } +}""") + if name == 'crunchstat.txt': + return open("tests/container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y-arv-mount.txt", "rt") + if name == 'arv-mount.txt': + return open("tests/container_request_9tee4-xvhdp-kk0ja1cl8b2kr1y-crunchstat.txt", "rt") + if name == 'usage_report.html': + return usage_report + return None + + col().open.side_effect = colreader_action + col().__iter__.return_value = ['node.json', 'crunchstat.txt', 'arv-mount.txt'] loadingContext, runtimeContext = self.helper(runner) arvjob = arvados_cwl.ArvadosContainer(runner, runtimeContext, + [], mock.MagicMock(), {}, None, @@ -527,6 +575,7 @@ class TestContainer(unittest.TestCase): arvjob.successCodes = [0] arvjob.outdir = "/var/spool/cwl" arvjob.output_ttl = 3600 + arvjob.uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzz1" arvjob.collect_outputs.return_value = {"out": "stuff"} @@ -536,16 +585,25 @@ class TestContainer(unittest.TestCase): "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2", "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz", "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz", - "modified_at": "2017-05-26T12:01:22Z" + "modified_at": "2017-05-26T12:01:22Z", + "properties": {}, + "name": "testjob" }) self.assertFalse(api.collections().create.called) self.assertFalse(runner.runtime_status_error.called) + # Assert that something was written to the usage report + self.assertTrue(len(usage_report.getvalue()) > 0) + arvjob.collect_outputs.assert_called_with("keep:abc+123", 0) arvjob.output_callback.assert_called_with({"out": "stuff"}, "success") runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2") + runner.api.container_requests().update.assert_called_with(uuid="zzzzz-xvhdp-zzzzzzzzzzzzzz1", + body={'container_request': {'properties': {'cwl_output': {'out': 'stuff'}}}}) + + # Test to make sure we dont call runtime_status_update if we already did # some where higher up in the call stack @mock.patch("arvados_cwl.util.get_current_container") @@ -610,6 +668,7 @@ class TestContainer(unittest.TestCase): arvjob = arvados_cwl.ArvadosContainer(runner, runtimeContext, + [], mock.MagicMock(), {}, None, @@ -629,14 +688,18 @@ class TestContainer(unittest.TestCase): "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2", "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz", "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz", - "modified_at": "2017-05-26T12:01:22Z" + "modified_at": "2017-05-26T12:01:22Z", + "properties": {} }) - rts_mock.assert_called_with( - 'error', - 'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:', - ' ** log is empty **' - ) + rts_mock.assert_has_calls([ + mock.call('error', + 'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:', + ' ** log is empty **' + ), + mock.call('warning', + 'arvados.cwl-runner: [container testjob] unable to generate resource usage report' + )]) arvjob.output_callback.assert_called_with({"out": "stuff"}, "permanentFail") # The test passes no builder.resources @@ -648,6 +711,7 @@ class TestContainer(unittest.TestCase): runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() runner.api._rootDesc = {"revision": "20210628"} + runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}} keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { @@ -718,7 +782,7 @@ class TestContainer(unittest.TestCase): "capacity": 1073741824 } }, 'state': 'Committed', - 'output_name': 'Output for step test_run_mounts', + 'output_name': 'Output from step test_run_mounts', 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'output_path': '/var/spool/cwl', 'output_ttl': 0, @@ -726,7 +790,38 @@ class TestContainer(unittest.TestCase): 'command': ['ls', '/var/spool/cwl'], 'cwd': '/var/spool/cwl', 'scheduling_parameters': {}, - 'properties': {}, + 'properties': {'cwl_input': { + "p1": { + "basename": "99999999999999999999999999999994+44", + "class": "Directory", + "dirname": "/keep", + "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz", + "listing": [ + { + "basename": "file1", + "class": "File", + "dirname": "/keep/99999999999999999999999999999994+44", + "location": "keep:99999999999999999999999999999994+44/file1", + "nameext": "", + "nameroot": "file1", + "path": "/keep/99999999999999999999999999999994+44/file1", + "size": 0 + }, + { + "basename": "file2", + "class": "File", + "dirname": "/keep/99999999999999999999999999999994+44", + "location": "keep:99999999999999999999999999999994+44/file2", + "nameext": "", + "nameroot": "file2", + "path": "/keep/99999999999999999999999999999994+44/file2", + "size": 0 + } + ], + "location": "keep:99999999999999999999999999999994+44", + "path": "/keep/99999999999999999999999999999994+44" + } + }}, 'secret_mounts': {}, 'output_storage_classes': ["default"] })) @@ -741,6 +836,7 @@ class TestContainer(unittest.TestCase): runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() runner.api._rootDesc = {"revision": "20210628"} + runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}} keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { @@ -812,7 +908,7 @@ class TestContainer(unittest.TestCase): "capacity": 1073741824 } }, 'state': 'Committed', - 'output_name': 'Output for step test_secrets', + 'output_name': 'Output from step test_secrets', 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'output_path': '/var/spool/cwl', 'output_ttl': 0, @@ -820,7 +916,7 @@ class TestContainer(unittest.TestCase): 'command': ['md5sum', 'example.conf'], 'cwd': '/var/spool/cwl', 'scheduling_parameters': {}, - 'properties': {}, + 'properties': {'cwl_input': job_order}, "secret_mounts": { "/var/spool/cwl/example.conf": { "content": "username: user\npassword: blorp\n", @@ -839,6 +935,7 @@ class TestContainer(unittest.TestCase): runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() runner.api._rootDesc = {"revision": "20210628"} + runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}} keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { @@ -877,13 +974,14 @@ class TestContainer(unittest.TestCase): # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024} @mock.patch("arvados.commands.keepdocker.list_images_in_arv") def test_setting_storage_class(self, keepdocker): - arv_docker_clear_cache() + #arv_docker_clear_cache() runner = mock.MagicMock() runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() runner.api._rootDesc = {"revision": "20210628"} + runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}} keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { @@ -934,7 +1032,7 @@ class TestContainer(unittest.TestCase): "capacity": 1073741824 } }, 'state': 'Committed', - 'output_name': 'Output for step test_run_True', + 'output_name': 'Output from step test_run_True', 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'output_path': '/var/spool/cwl', 'output_ttl': 0, @@ -942,7 +1040,7 @@ class TestContainer(unittest.TestCase): 'command': ['ls', '/var/spool/cwl'], 'cwd': '/var/spool/cwl', 'scheduling_parameters': {}, - 'properties': {}, + 'properties': {'cwl_input': {}}, 'secret_mounts': {}, 'output_storage_classes': ["foo_sc", "bar_sc"] })) @@ -952,13 +1050,14 @@ class TestContainer(unittest.TestCase): # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024} @mock.patch("arvados.commands.keepdocker.list_images_in_arv") def test_setting_process_properties(self, keepdocker): - arv_docker_clear_cache() + #arv_docker_clear_cache() runner = mock.MagicMock() runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() runner.api._rootDesc = {"revision": "20210628"} + runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}} keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { @@ -1020,7 +1119,7 @@ class TestContainer(unittest.TestCase): "capacity": 1073741824 } }, 'state': 'Committed', - 'output_name': 'Output for step test_run_True', + 'output_name': 'Output from step test_run_True', 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'output_path': '/var/spool/cwl', 'output_ttl': 0, @@ -1030,6 +1129,7 @@ class TestContainer(unittest.TestCase): 'scheduling_parameters': {}, 'properties': { "baz": "blorp", + "cwl_input": {"x": "blorp"}, "foo": "bar", "quux": { "q1": 1, @@ -1046,13 +1146,14 @@ class TestContainer(unittest.TestCase): @mock.patch("arvados.commands.keepdocker.list_images_in_arv") def test_cuda_requirement(self, keepdocker): arvados_cwl.add_arv_hints() - arv_docker_clear_cache() + #arv_docker_clear_cache() runner = mock.MagicMock() runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() runner.api._rootDesc = {"revision": "20210628"} + runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}} keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { @@ -1130,7 +1231,7 @@ class TestContainer(unittest.TestCase): "capacity": 1073741824 } }, 'state': 'Committed', - 'output_name': 'Output for step test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)), + 'output_name': 'Output from step test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)), 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'output_path': '/var/spool/cwl', 'output_ttl': 0, @@ -1138,7 +1239,7 @@ class TestContainer(unittest.TestCase): 'command': ['nvidia-smi'], 'cwd': '/var/spool/cwl', 'scheduling_parameters': {}, - 'properties': {}, + 'properties': {'cwl_input': {}}, 'secret_mounts': {}, 'output_storage_classes': ["default"] })) @@ -1150,13 +1251,13 @@ class TestContainer(unittest.TestCase): @mock.patch("arvados.commands.keepdocker.list_images_in_arv") def test_match_local_docker(self, keepdocker, determine_image_id): arvados_cwl.add_arv_hints() - arv_docker_clear_cache() runner = mock.MagicMock() runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() runner.api._rootDesc = {"revision": "20210628"} + runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}} keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz4", {"dockerhash": "456"}), ("zzzzz-4zz18-zzzzzzzzzzzzzz3", {"dockerhash": "123"})] @@ -1204,7 +1305,7 @@ class TestContainer(unittest.TestCase): "capacity": 1073741824 } }, 'state': 'Committed', - 'output_name': 'Output for step test_run_True', + 'output_name': 'Output from step test_run_True', 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'output_path': '/var/spool/cwl', 'output_ttl': 0, @@ -1212,7 +1313,7 @@ class TestContainer(unittest.TestCase): 'command': ['echo'], 'cwd': '/var/spool/cwl', 'scheduling_parameters': {}, - 'properties': {}, + 'properties': {'cwl_input': {}}, 'secret_mounts': {}, 'output_storage_classes': ["default"] } @@ -1223,11 +1324,11 @@ class TestContainer(unittest.TestCase): runner.api.container_requests().create.assert_called_with( body=JsonDiffMatcher(container_request)) - arv_docker_clear_cache() + runtimeContext.cached_docker_lookups.clear() runtimeContext.match_local_docker = True container_request['container_image'] = '99999999999999999999999999999993+99' container_request['name'] = 'test_run_True_2' - container_request['output_name'] = 'Output for step test_run_True_2' + container_request['output_name'] = 'Output from step test_run_True_2' for j in arvtool.job({}, mock.MagicMock(), runtimeContext): j.run(runtimeContext) runner.api.container_requests().create.assert_called_with( @@ -1241,13 +1342,14 @@ class TestContainer(unittest.TestCase): arvados_cwl.add_arv_hints() for enable_preemptible in (None, True, False): for preemptible_hint in (None, True, False): - arv_docker_clear_cache() + #arv_docker_clear_cache() runner = mock.MagicMock() runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() runner.api._rootDesc = {"revision": "20210628"} + runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}} keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { @@ -1317,7 +1419,7 @@ class TestContainer(unittest.TestCase): "capacity": 1073741824 } }, 'state': 'Committed', - 'output_name': 'Output for step '+runtimeContext.name, + 'output_name': 'Output from step '+runtimeContext.name, 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'output_path': '/var/spool/cwl', 'output_ttl': 0, @@ -1325,17 +1427,168 @@ class TestContainer(unittest.TestCase): 'command': ['ls', '/var/spool/cwl'], 'cwd': '/var/spool/cwl', 'scheduling_parameters': sched, - 'properties': {}, + 'properties': {'cwl_input': {}}, 'secret_mounts': {}, 'output_storage_classes': ["default"] })) + @mock.patch("arvados.commands.keepdocker.list_images_in_arv") + def test_output_properties(self, keepdocker): + arvados_cwl.add_arv_hints() + for rev in ["20210628", "20220510"]: + runner = mock.MagicMock() + runner.ignore_docker_for_reuse = False + runner.intermediate_output_ttl = 0 + runner.secret_store = cwltool.secrets.SecretStore() + runner.api._rootDesc = {"revision": rev} + runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}} + + keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] + runner.api.collections().get().execute.return_value = { + "portable_data_hash": "99999999999999999999999999999993+99"} + + tool = cmap({ + "inputs": [{ + "id": "inp", + "type": "string" + }], + "outputs": [], + "baseCommand": "ls", + "arguments": [{"valueFrom": "$(runtime.outdir)"}], + "id": "", + "cwlVersion": "v1.2", + "class": "CommandLineTool", + "hints": [ + { + "class": "http://arvados.org/cwl#OutputCollectionProperties", + "outputProperties": { + "foo": "bar", + "baz": "$(inputs.inp)" + } + } + ] + }) + + loadingContext, runtimeContext = self.helper(runner) + runtimeContext.name = "test_timelimit" + + arvtool = cwltool.load_tool.load_tool(tool, loadingContext) + arvtool.formatgraph = None + + for j in arvtool.job({"inp": "quux"}, mock.MagicMock(), runtimeContext): + j.run(runtimeContext) + + _, kwargs = runner.api.container_requests().create.call_args + if rev == "20220510": + self.assertEqual({"foo": "bar", "baz": "quux"}, kwargs['body'].get('output_properties')) + else: + self.assertEqual(None, kwargs['body'].get('output_properties')) + + @mock.patch("arvados.commands.keepdocker.list_images_in_arv") + def test_output_glob(self, keepdocker): + arvados_cwl.add_arv_hints() + for rev in ["20231117", "20240502"]: + runner = mock.MagicMock() + runner.ignore_docker_for_reuse = False + runner.intermediate_output_ttl = 0 + runner.secret_store = cwltool.secrets.SecretStore() + runner.api._rootDesc = {"revision": rev} + runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}} + + keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] + runner.api.collections().get().execute.return_value = { + "portable_data_hash": "99999999999999999999999999999993+99"} + + tool = cmap({ + "inputs": [{ + "id": "inp", + "type": "string" + }], + "outputs": [ + { + "id": "o1", + "type": "File", + "outputBinding": { + "glob": "*.txt" + } + }, + { + "id": "o2", + "type": "File", + "outputBinding": { + "glob": ["*.dat", "*.bat"] + } + }, + { + "id": "o3", + "type": { + "type": "record", + "fields": [ + { + "name": "f1", + "type": "File", + "outputBinding": { + "glob": ["*.cat"] + } + } + ] + } + }, + { + "id": "o4", + "type": "File", + "outputBinding": { + "glob": "$(inputs.inp)" + } + }, + { + "id": "o5", + "type": "File", + "outputBinding": { + "glob": "*.foo" + }, + "secondaryFiles": [".goo", "^.hoo"] + }, + + ], + "baseCommand": "ls", + "arguments": [{"valueFrom": "$(runtime.outdir)"}], + "id": "", + "cwlVersion": "v1.2", + "class": "CommandLineTool", + "hints": [ ] + }) + + loadingContext, runtimeContext = self.helper(runner) + runtimeContext.name = "test_timelimit" + + arvtool = cwltool.load_tool.load_tool(tool, loadingContext) + arvtool.formatgraph = None + + for j in arvtool.job({"inp": "quux"}, mock.MagicMock(), runtimeContext): + j.run(runtimeContext) + + _, kwargs = runner.api.container_requests().create.call_args + if rev == "20240502": + self.assertEqual(['*.txt', '*.txt/**', + '*.dat', '*.dat/**', + '*.bat', '*.bat/**', + '*.cat', '*.cat/**', + 'quux', 'quux/**', + '*.foo', '*.foo/**', + '*.foo.goo', '*.foo.goo/**', + '*.hoo', '*.hoo/**', + 'cwl.output.json', + ], kwargs['body'].get('output_glob')) + else: + self.assertEqual(None, kwargs['body'].get('output_glob')) + class TestWorkflow(unittest.TestCase): def setUp(self): cwltool.process._names = set() - arv_docker_clear_cache() + #arv_docker_clear_cache() def helper(self, runner, enable_reuse=True): document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0") @@ -1354,7 +1607,8 @@ class TestWorkflow(unittest.TestCase): "make_fs_access": make_fs_access, "loader": document_loader, "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}, - "construct_tool_object": runner.arv_make_tool}) + "construct_tool_object": runner.arv_make_tool, + "default_docker_image": "arvados/jobs:"+arvados_cwl.__version__}) runtimeContext = arvados_cwl.context.ArvRuntimeContext( {"work_api": "containers", "basedir": "", @@ -1376,6 +1630,7 @@ class TestWorkflow(unittest.TestCase): api = mock.MagicMock() api._rootDesc = get_rootDesc() + api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}} runner = arvados_cwl.executor.ArvCwlExecutor(api) self.assertEqual(runner.work_api, 'containers') @@ -1385,6 +1640,8 @@ class TestWorkflow(unittest.TestCase): runner.api.collections().list().execute.return_value = {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz", "portable_data_hash": "99999999999999999999999999999993+99"}]} + runner.api.containers().current().execute.return_value = {} + runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz" runner.ignore_docker_for_reuse = False runner.num_retries = 0 @@ -1457,11 +1714,23 @@ class TestWorkflow(unittest.TestCase): } }, "name": "scatterstep", - "output_name": "Output for step scatterstep", + "output_name": "Output from step scatterstep", "output_path": "/var/spool/cwl", "output_ttl": 0, "priority": 500, - "properties": {}, + "properties": {'cwl_input': { + "fileblub": { + "basename": "token.txt", + "class": "File", + "dirname": "/keep/99999999999999999999999999999999+118", + "location": "keep:99999999999999999999999999999999+118/token.txt", + "nameext": ".txt", + "nameroot": "token", + "path": "/keep/99999999999999999999999999999999+118/token.txt", + "size": 0 + }, + "sleeptime": 5 + }}, "runtime_constraints": { "ram": 1073741824, "vcpus": 1 @@ -1494,6 +1763,7 @@ class TestWorkflow(unittest.TestCase): api = mock.MagicMock() api._rootDesc = get_rootDesc() + api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}} runner = arvados_cwl.executor.ArvCwlExecutor(api) self.assertEqual(runner.work_api, 'containers') @@ -1534,7 +1804,7 @@ class TestWorkflow(unittest.TestCase): 'name': u'echo-subwf', 'secret_mounts': {}, 'runtime_constraints': {'API': True, 'vcpus': 3, 'ram': 1073741824}, - 'properties': {}, + 'properties': {'cwl_input': {}}, 'priority': 500, 'mounts': { '/var/spool/cwl/cwl.input.yml': { @@ -1571,7 +1841,7 @@ class TestWorkflow(unittest.TestCase): u'cwl.input.yml' ], 'use_existing': True, - 'output_name': u'Output for step echo-subwf', + 'output_name': u'Output from step echo-subwf', 'cwd': '/var/spool/cwl', 'output_storage_classes': ["default"] }))