X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/cf9afd31cf9bc682ea3f8c0d59012fa7080b0e9b..aece35d20c892aa7b3df29b15ac0a8a167b0b3ab:/sdk/cwl/tests/test_container.py diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py index cb57b446da..a2f404d7eb 100644 --- a/sdk/cwl/tests/test_container.py +++ b/sdk/cwl/tests/test_container.py @@ -8,7 +8,7 @@ from builtins import object import arvados_cwl import arvados_cwl.context import arvados_cwl.util -from arvados_cwl.arvdocker import arv_docker_clear_cache +#from arvados_cwl.arvdocker import arv_docker_clear_cache import copy import arvados.config import logging @@ -61,7 +61,7 @@ class TestContainer(unittest.TestCase): def setUp(self): cwltool.process._names = set() - arv_docker_clear_cache() + #arv_docker_clear_cache() def tearDown(self): root_logger = logging.getLogger('') @@ -128,13 +128,14 @@ class TestContainer(unittest.TestCase): @mock.patch("arvados.commands.keepdocker.list_images_in_arv") def test_run(self, keepdocker): for enable_reuse in (True, False): - arv_docker_clear_cache() + #arv_docker_clear_cache() runner = mock.MagicMock() runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() runner.api._rootDesc = {"revision": "20210628"} + runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}} keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { @@ -186,7 +187,7 @@ class TestContainer(unittest.TestCase): 'command': ['ls', '/var/spool/cwl'], 'cwd': '/var/spool/cwl', 'scheduling_parameters': {}, - 'properties': {}, + 'properties': {'cwl_input': {}}, 'secret_mounts': {}, 'output_storage_classes': ["default"] })) @@ -201,6 +202,7 @@ class TestContainer(unittest.TestCase): runner.intermediate_output_ttl = 3600 runner.secret_store = cwltool.secrets.SecretStore() runner.api._rootDesc = {"revision": "20210628"} + runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}} keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { @@ -277,7 +279,7 @@ class TestContainer(unittest.TestCase): 'scheduling_parameters': { 'partitions': ['blurb'] }, - 'properties': {}, + 'properties': {'cwl_input': {}}, 'secret_mounts': {}, 'output_storage_classes': ["default"] } @@ -298,6 +300,7 @@ class TestContainer(unittest.TestCase): runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() runner.api._rootDesc = {"revision": "20210628"} + runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}} keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { @@ -410,7 +413,7 @@ class TestContainer(unittest.TestCase): 'cwd': '/var/spool/cwl', 'scheduling_parameters': { }, - 'properties': {}, + 'properties': {'cwl_input': {}}, 'secret_mounts': {}, 'output_storage_classes': ["default"] } @@ -429,6 +432,7 @@ class TestContainer(unittest.TestCase): runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() runner.api._rootDesc = {"revision": "20210628"} + runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}} keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { @@ -497,7 +501,7 @@ class TestContainer(unittest.TestCase): 'command': ['ls', '/var/spool/cwl'], 'cwd': '/var/spool/cwl', 'scheduling_parameters': {}, - 'properties': {}, + 'properties': {'cwl_input': {}}, 'secret_mounts': {}, 'output_storage_classes': ["default"] })) @@ -534,6 +538,7 @@ class TestContainer(unittest.TestCase): arvjob.successCodes = [0] arvjob.outdir = "/var/spool/cwl" arvjob.output_ttl = 3600 + arvjob.uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzz1" arvjob.collect_outputs.return_value = {"out": "stuff"} @@ -543,7 +548,8 @@ class TestContainer(unittest.TestCase): "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2", "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz", "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz", - "modified_at": "2017-05-26T12:01:22Z" + "modified_at": "2017-05-26T12:01:22Z", + "properties": {} }) self.assertFalse(api.collections().create.called) @@ -553,6 +559,10 @@ class TestContainer(unittest.TestCase): arvjob.output_callback.assert_called_with({"out": "stuff"}, "success") runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2") + runner.api.container_requests().update.assert_called_with(uuid="zzzzz-xvhdp-zzzzzzzzzzzzzz1", + body={'container_request': {'properties': {'cwl_output': {'out': 'stuff'}}}}) + + # Test to make sure we dont call runtime_status_update if we already did # some where higher up in the call stack @mock.patch("arvados_cwl.util.get_current_container") @@ -636,7 +646,8 @@ class TestContainer(unittest.TestCase): "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2", "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz", "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz", - "modified_at": "2017-05-26T12:01:22Z" + "modified_at": "2017-05-26T12:01:22Z", + "properties": {} }) rts_mock.assert_called_with( @@ -655,6 +666,7 @@ class TestContainer(unittest.TestCase): runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() runner.api._rootDesc = {"revision": "20210628"} + runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}} keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { @@ -733,7 +745,38 @@ class TestContainer(unittest.TestCase): 'command': ['ls', '/var/spool/cwl'], 'cwd': '/var/spool/cwl', 'scheduling_parameters': {}, - 'properties': {}, + 'properties': {'cwl_input': { + "p1": { + "basename": "99999999999999999999999999999994+44", + "class": "Directory", + "dirname": "/keep", + "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz", + "listing": [ + { + "basename": "file1", + "class": "File", + "dirname": "/keep/99999999999999999999999999999994+44", + "location": "keep:99999999999999999999999999999994+44/file1", + "nameext": "", + "nameroot": "file1", + "path": "/keep/99999999999999999999999999999994+44/file1", + "size": 0 + }, + { + "basename": "file2", + "class": "File", + "dirname": "/keep/99999999999999999999999999999994+44", + "location": "keep:99999999999999999999999999999994+44/file2", + "nameext": "", + "nameroot": "file2", + "path": "/keep/99999999999999999999999999999994+44/file2", + "size": 0 + } + ], + "location": "keep:99999999999999999999999999999994+44", + "path": "/keep/99999999999999999999999999999994+44" + } + }}, 'secret_mounts': {}, 'output_storage_classes': ["default"] })) @@ -748,6 +791,7 @@ class TestContainer(unittest.TestCase): runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() runner.api._rootDesc = {"revision": "20210628"} + runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}} keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { @@ -827,7 +871,7 @@ class TestContainer(unittest.TestCase): 'command': ['md5sum', 'example.conf'], 'cwd': '/var/spool/cwl', 'scheduling_parameters': {}, - 'properties': {}, + 'properties': {'cwl_input': job_order}, "secret_mounts": { "/var/spool/cwl/example.conf": { "content": "username: user\npassword: blorp\n", @@ -846,6 +890,7 @@ class TestContainer(unittest.TestCase): runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() runner.api._rootDesc = {"revision": "20210628"} + runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}} keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { @@ -884,13 +929,14 @@ class TestContainer(unittest.TestCase): # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024} @mock.patch("arvados.commands.keepdocker.list_images_in_arv") def test_setting_storage_class(self, keepdocker): - arv_docker_clear_cache() + #arv_docker_clear_cache() runner = mock.MagicMock() runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() runner.api._rootDesc = {"revision": "20210628"} + runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}} keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { @@ -949,7 +995,7 @@ class TestContainer(unittest.TestCase): 'command': ['ls', '/var/spool/cwl'], 'cwd': '/var/spool/cwl', 'scheduling_parameters': {}, - 'properties': {}, + 'properties': {'cwl_input': {}}, 'secret_mounts': {}, 'output_storage_classes': ["foo_sc", "bar_sc"] })) @@ -959,13 +1005,14 @@ class TestContainer(unittest.TestCase): # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024} @mock.patch("arvados.commands.keepdocker.list_images_in_arv") def test_setting_process_properties(self, keepdocker): - arv_docker_clear_cache() + #arv_docker_clear_cache() runner = mock.MagicMock() runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() runner.api._rootDesc = {"revision": "20210628"} + runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}} keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { @@ -1037,6 +1084,7 @@ class TestContainer(unittest.TestCase): 'scheduling_parameters': {}, 'properties': { "baz": "blorp", + "cwl_input": {"x": "blorp"}, "foo": "bar", "quux": { "q1": 1, @@ -1053,13 +1101,14 @@ class TestContainer(unittest.TestCase): @mock.patch("arvados.commands.keepdocker.list_images_in_arv") def test_cuda_requirement(self, keepdocker): arvados_cwl.add_arv_hints() - arv_docker_clear_cache() + #arv_docker_clear_cache() runner = mock.MagicMock() runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() runner.api._rootDesc = {"revision": "20210628"} + runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}} keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { @@ -1145,7 +1194,7 @@ class TestContainer(unittest.TestCase): 'command': ['nvidia-smi'], 'cwd': '/var/spool/cwl', 'scheduling_parameters': {}, - 'properties': {}, + 'properties': {'cwl_input': {}}, 'secret_mounts': {}, 'output_storage_classes': ["default"] })) @@ -1157,13 +1206,13 @@ class TestContainer(unittest.TestCase): @mock.patch("arvados.commands.keepdocker.list_images_in_arv") def test_match_local_docker(self, keepdocker, determine_image_id): arvados_cwl.add_arv_hints() - arv_docker_clear_cache() runner = mock.MagicMock() runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() runner.api._rootDesc = {"revision": "20210628"} + runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}} keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz4", {"dockerhash": "456"}), ("zzzzz-4zz18-zzzzzzzzzzzzzz3", {"dockerhash": "123"})] @@ -1219,7 +1268,7 @@ class TestContainer(unittest.TestCase): 'command': ['echo'], 'cwd': '/var/spool/cwl', 'scheduling_parameters': {}, - 'properties': {}, + 'properties': {'cwl_input': {}}, 'secret_mounts': {}, 'output_storage_classes': ["default"] } @@ -1230,7 +1279,7 @@ class TestContainer(unittest.TestCase): runner.api.container_requests().create.assert_called_with( body=JsonDiffMatcher(container_request)) - arv_docker_clear_cache() + runtimeContext.cached_docker_lookups.clear() runtimeContext.match_local_docker = True container_request['container_image'] = '99999999999999999999999999999993+99' container_request['name'] = 'test_run_True_2' @@ -1248,13 +1297,14 @@ class TestContainer(unittest.TestCase): arvados_cwl.add_arv_hints() for enable_preemptible in (None, True, False): for preemptible_hint in (None, True, False): - arv_docker_clear_cache() + #arv_docker_clear_cache() runner = mock.MagicMock() runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() runner.api._rootDesc = {"revision": "20210628"} + runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}} keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { @@ -1332,7 +1382,7 @@ class TestContainer(unittest.TestCase): 'command': ['ls', '/var/spool/cwl'], 'cwd': '/var/spool/cwl', 'scheduling_parameters': sched, - 'properties': {}, + 'properties': {'cwl_input': {}}, 'secret_mounts': {}, 'output_storage_classes': ["default"] })) @@ -1347,6 +1397,7 @@ class TestContainer(unittest.TestCase): runner.intermediate_output_ttl = 0 runner.secret_store = cwltool.secrets.SecretStore() runner.api._rootDesc = {"revision": rev} + runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}} keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { @@ -1393,7 +1444,7 @@ class TestContainer(unittest.TestCase): class TestWorkflow(unittest.TestCase): def setUp(self): cwltool.process._names = set() - arv_docker_clear_cache() + #arv_docker_clear_cache() def helper(self, runner, enable_reuse=True): document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0") @@ -1434,6 +1485,7 @@ class TestWorkflow(unittest.TestCase): api = mock.MagicMock() api._rootDesc = get_rootDesc() + api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}} runner = arvados_cwl.executor.ArvCwlExecutor(api) self.assertEqual(runner.work_api, 'containers') @@ -1521,7 +1573,19 @@ class TestWorkflow(unittest.TestCase): "output_path": "/var/spool/cwl", "output_ttl": 0, "priority": 500, - "properties": {}, + "properties": {'cwl_input': { + "fileblub": { + "basename": "token.txt", + "class": "File", + "dirname": "/keep/99999999999999999999999999999999+118", + "location": "keep:99999999999999999999999999999999+118/token.txt", + "nameext": ".txt", + "nameroot": "token", + "path": "/keep/99999999999999999999999999999999+118/token.txt", + "size": 0 + }, + "sleeptime": 5 + }}, "runtime_constraints": { "ram": 1073741824, "vcpus": 1 @@ -1554,6 +1618,7 @@ class TestWorkflow(unittest.TestCase): api = mock.MagicMock() api._rootDesc = get_rootDesc() + api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}} runner = arvados_cwl.executor.ArvCwlExecutor(api) self.assertEqual(runner.work_api, 'containers') @@ -1594,7 +1659,7 @@ class TestWorkflow(unittest.TestCase): 'name': u'echo-subwf', 'secret_mounts': {}, 'runtime_constraints': {'API': True, 'vcpus': 3, 'ram': 1073741824}, - 'properties': {}, + 'properties': {'cwl_input': {}}, 'priority': 500, 'mounts': { '/var/spool/cwl/cwl.input.yml': {