X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/74ec5b86db46257f75ea1eb94c136ce18e65c906..8668135855b400e7f6047ecacd9cfa27fed723f7:/sdk/cwl/tests/test_container.py diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py index 455aa286e8..b13bcdecb3 100644 --- a/sdk/cwl/tests/test_container.py +++ b/sdk/cwl/tests/test_container.py @@ -1,4 +1,5 @@ import arvados_cwl +from arvados_cwl.arvdocker import arv_docker_clear_cache import logging import mock import unittest @@ -20,55 +21,62 @@ class TestContainer(unittest.TestCase): # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024} @mock.patch("arvados.commands.keepdocker.list_images_in_arv") def test_run(self, keepdocker): - runner = mock.MagicMock() - runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz" - runner.ignore_docker_for_reuse = False - - keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] - runner.api.collections().get().execute.return_value = { - "portable_data_hash": "99999999999999999999999999999993+99"} - - document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0") - - tool = { - "inputs": [], - "outputs": [], - "baseCommand": "ls", - "arguments": [{"valueFrom": "$(runtime.outdir)"}] - } - make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api) - arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names, - basedir="", make_fs_access=make_fs_access, loader=Loader({})) - arvtool.formatgraph = None - for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run", - make_fs_access=make_fs_access, tmpdir="/tmp"): - j.run() - runner.api.container_requests().create.assert_called_with( - body={ - 'environment': { - 'HOME': '/var/spool/cwl', - 'TMPDIR': '/tmp' - }, - 'name': 'test_run', - 'runtime_constraints': { - 'vcpus': 1, - 'ram': 1073741824 - }, 'priority': 1, - 'mounts': { - '/var/spool/cwl': {'kind': 'tmp'} - }, - 'state': 'Committed', - 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', - 'output_path': '/var/spool/cwl', - 'container_image': '99999999999999999999999999999993+99', - 'command': ['ls', '/var/spool/cwl'], - 'cwd': '/var/spool/cwl' - }) + for enable_reuse in (True, False): + arv_docker_clear_cache() + + runner = mock.MagicMock() + runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz" + runner.ignore_docker_for_reuse = False + + keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] + runner.api.collections().get().execute.return_value = { + "portable_data_hash": "99999999999999999999999999999993+99"} + + document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0") + + tool = { + "inputs": [], + "outputs": [], + "baseCommand": "ls", + "arguments": [{"valueFrom": "$(runtime.outdir)"}] + } + make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api) + arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names, + basedir="", make_fs_access=make_fs_access, loader=Loader({})) + arvtool.formatgraph = None + for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run_"+str(enable_reuse), + make_fs_access=make_fs_access, tmpdir="/tmp"): + j.run(enable_reuse=enable_reuse) + runner.api.container_requests().create.assert_called_with( + body={ + 'environment': { + 'HOME': '/var/spool/cwl', + 'TMPDIR': '/tmp' + }, + 'name': 'test_run_'+str(enable_reuse), + 'runtime_constraints': { + 'vcpus': 1, + 'ram': 1073741824 + }, + 'use_existing': enable_reuse, + 'priority': 1, + 'mounts': { + '/var/spool/cwl': {'kind': 'tmp'} + }, + 'state': 'Committed', + 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', + 'output_path': '/var/spool/cwl', + 'container_image': '99999999999999999999999999999993+99', + 'command': ['ls', '/var/spool/cwl'], + 'cwd': '/var/spool/cwl', + 'scheduling_parameters': {} + }) # The test passes some fields in builder.resources # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024} @mock.patch("arvados.commands.keepdocker.list_images_in_arv") def test_resource_requirements(self, keepdocker): + arv_docker_clear_cache() runner = mock.MagicMock() runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz" runner.ignore_docker_for_reuse = False @@ -91,6 +99,9 @@ class TestContainer(unittest.TestCase): "keep_cache": 512 }, { "class": "http://arvados.org/cwl#APIRequirement", + }, { + "class": "http://arvados.org/cwl#PartitionRequirement", + "partition": "blurb" }], "baseCommand": "ls" } @@ -103,8 +114,9 @@ class TestContainer(unittest.TestCase): make_fs_access=make_fs_access, tmpdir="/tmp"): j.run() - runner.api.container_requests().create.assert_called_with( - body={ + call_args, call_kwargs = runner.api.container_requests().create.call_args + + call_body_expected = { 'environment': { 'HOME': '/var/spool/cwl', 'TMPDIR': '/tmp' @@ -113,8 +125,11 @@ class TestContainer(unittest.TestCase): 'runtime_constraints': { 'vcpus': 3, 'ram': 3145728000, + 'keep_cache_ram': 512, 'API': True - }, 'priority': 1, + }, + 'use_existing': True, + 'priority': 1, 'mounts': { '/var/spool/cwl': {'kind': 'tmp'} }, @@ -123,8 +138,16 @@ class TestContainer(unittest.TestCase): 'output_path': '/var/spool/cwl', 'container_image': '99999999999999999999999999999993+99', 'command': ['ls'], - 'cwd': '/var/spool/cwl' - }) + 'cwd': '/var/spool/cwl', + 'scheduling_parameters': { + 'partitions': ['blurb'] + } + } + + call_body = call_kwargs.get('body', None) + self.assertNotEqual(None, call_body) + for key in call_body: + self.assertEqual(call_body_expected.get(key), call_body.get(key)) @mock.patch("arvados.collection.Collection") def test_done(self, col): @@ -149,29 +172,14 @@ class TestContainer(unittest.TestCase): arvjob.outdir = "/var/spool/cwl" arvjob.done({ - "state": "Complete", - "output": "99999999999999999999999999999993+99", - "log": "99999999999999999999999999999994+99", - "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz", - "exit_code": 0 + "state": "Final", + "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1", + "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2", + "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz", + "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz" }) - api.collections().list.assert_has_calls([ - mock.call(), - mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'], - ['portable_data_hash', '=', '99999999999999999999999999999993+99'], - ['name', '=', 'Output 9999999 of testjob']]), - mock.call().execute(num_retries=0), - mock.call(limit=1, filters=[['portable_data_hash', '=', '99999999999999999999999999999993+99']], - select=['manifest_text']), - mock.call().execute(num_retries=0)]) - - api.collections().create.assert_called_with( - ensure_unique_name=True, - body={'portable_data_hash': '99999999999999999999999999999993+99', - 'manifest_text': 'XYZ', - 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', - 'name': 'Output 9999999 of testjob'}) + self.assertFalse(api.collections().create.called) @mock.patch("arvados.collection.Collection") def test_done_use_existing_collection(self, col): @@ -194,18 +202,12 @@ class TestContainer(unittest.TestCase): arvjob.outdir = "/var/spool/cwl" arvjob.done({ - "state": "Complete", - "output": "99999999999999999999999999999993+99", - "log": "99999999999999999999999999999994+99", - "uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz", - "exit_code": 0 + "state": "Final", + "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1", + "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2", + "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2", + "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz", + "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz" }) - api.collections().list.assert_has_calls([ - mock.call(), - mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'], - ['portable_data_hash', '=', '99999999999999999999999999999993+99'], - ['name', '=', 'Output 9999999 of testjob']]), - mock.call().execute(num_retries=0)]) - self.assertFalse(api.collections().create.called)