X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/31d76600cdb691251d0823cc6be601d958b4e1a4..607fe087f6167061714a524dd53cbbc21b974973:/sdk/cwl/tests/test_container.py diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py index 68cb3273fd..b06eae8105 100644 --- a/sdk/cwl/tests/test_container.py +++ b/sdk/cwl/tests/test_container.py @@ -41,7 +41,8 @@ class TestContainer(unittest.TestCase): "baseCommand": "ls", "arguments": [{"valueFrom": "$(runtime.outdir)"}] }) - make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api) + make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, + collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0)) arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names, basedir="", make_fs_access=make_fs_access, loader=Loader({})) arvtool.formatgraph = None @@ -62,12 +63,13 @@ class TestContainer(unittest.TestCase): 'use_existing': enable_reuse, 'priority': 1, 'mounts': { + '/tmp': {'kind': 'tmp'}, '/var/spool/cwl': {'kind': 'tmp'} }, 'state': 'Committed', 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'output_path': '/var/spool/cwl', - 'container_image': '99999999999999999999999999999993+99', + 'container_image': 'arvados/jobs', 'command': ['ls', '/var/spool/cwl'], 'cwd': '/var/spool/cwl', 'scheduling_parameters': {}, @@ -107,7 +109,8 @@ class TestContainer(unittest.TestCase): }], "baseCommand": "ls" }) - make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api) + make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, + collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0)) arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names, make_fs_access=make_fs_access, loader=Loader({})) @@ -133,12 +136,13 @@ class TestContainer(unittest.TestCase): 'use_existing': True, 'priority': 1, 'mounts': { + '/tmp': {'kind': 'tmp'}, '/var/spool/cwl': {'kind': 'tmp'} }, 'state': 'Committed', 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'output_path': '/var/spool/cwl', - 'container_image': '99999999999999999999999999999993+99', + 'container_image': 'arvados/jobs', 'command': ['ls'], 'cwd': '/var/spool/cwl', 'scheduling_parameters': { @@ -208,7 +212,8 @@ class TestContainer(unittest.TestCase): }], "baseCommand": "ls" }) - make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api) + make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, + collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0)) arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names, make_fs_access=make_fs_access, loader=Loader({})) @@ -237,6 +242,7 @@ class TestContainer(unittest.TestCase): 'use_existing': True, 'priority': 1, 'mounts': { + '/tmp': {'kind': 'tmp'}, '/var/spool/cwl': {'kind': 'tmp'}, '/var/spool/cwl/foo': { 'kind': 'collection', @@ -262,7 +268,7 @@ class TestContainer(unittest.TestCase): 'state': 'Committed', 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', 'output_path': '/var/spool/cwl', - 'container_image': '99999999999999999999999999999993+99', + 'container_image': 'arvados/jobs', 'command': ['ls'], 'cwd': '/var/spool/cwl', 'scheduling_parameters': { @@ -275,6 +281,79 @@ class TestContainer(unittest.TestCase): for key in call_body: self.assertEqual(call_body_expected.get(key), call_body.get(key)) + + # Test redirecting stdin/stdout/stderr + @mock.patch("arvados.commands.keepdocker.list_images_in_arv") + def test_redirects(self, keepdocker): + arv_docker_clear_cache() + + runner = mock.MagicMock() + runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz" + runner.ignore_docker_for_reuse = False + + keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] + runner.api.collections().get().execute.return_value = { + "portable_data_hash": "99999999999999999999999999999993+99"} + + document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0") + + tool = cmap({ + "inputs": [], + "outputs": [], + "baseCommand": "ls", + "stdout": "stdout.txt", + "stderr": "stderr.txt", + "stdin": "/keep/99999999999999999999999999999996+99/file.txt", + "arguments": [{"valueFrom": "$(runtime.outdir)"}] + }) + make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, + collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0)) + arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names, + basedir="", make_fs_access=make_fs_access, loader=Loader({})) + arvtool.formatgraph = None + for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run_redirect", + make_fs_access=make_fs_access, tmpdir="/tmp"): + j.run() + runner.api.container_requests().create.assert_called_with( + body=JsonDiffMatcher({ + 'environment': { + 'HOME': '/var/spool/cwl', + 'TMPDIR': '/tmp' + }, + 'name': 'test_run_redirect', + 'runtime_constraints': { + 'vcpus': 1, + 'ram': 1073741824 + }, + 'use_existing': True, + 'priority': 1, + 'mounts': { + '/tmp': {'kind': 'tmp'}, + '/var/spool/cwl': {'kind': 'tmp'}, + "stderr": { + "kind": "file", + "path": "/var/spool/cwl/stderr.txt" + }, + "stdin": { + "kind": "collection", + "path": "file.txt", + "portable_data_hash": "99999999999999999999999999999996+99" + }, + "stdout": { + "kind": "file", + "path": "/var/spool/cwl/stdout.txt" + }, + }, + 'state': 'Committed', + 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', + 'output_path': '/var/spool/cwl', + 'container_image': 'arvados/jobs', + 'command': ['ls', '/var/spool/cwl'], + 'cwd': '/var/spool/cwl', + 'scheduling_parameters': {}, + 'properties': {}, + })) + @mock.patch("arvados.collection.Collection") def test_done(self, col): api = mock.MagicMock() @@ -313,3 +392,83 @@ class TestContainer(unittest.TestCase): arvjob.collect_outputs.assert_called_with("keep:abc+123") arvjob.output_callback.assert_called_with({"out": "stuff"}, "success") + + # The test passes no builder.resources + # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024} + @mock.patch("arvados.commands.keepdocker.list_images_in_arv") + def test_mounts(self, keepdocker): + arv_docker_clear_cache() + + runner = mock.MagicMock() + runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz" + runner.ignore_docker_for_reuse = False + + keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] + runner.api.collections().get().execute.return_value = { + "portable_data_hash": "99999999999999999999999999999993+99"} + + document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0") + + tool = cmap({ + "inputs": [ + {"id": "p1", + "type": "Directory"} + ], + "outputs": [], + "baseCommand": "ls", + "arguments": [{"valueFrom": "$(runtime.outdir)"}] + }) + make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, + collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0)) + arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names, + basedir="", make_fs_access=make_fs_access, loader=Loader({})) + arvtool.formatgraph = None + job_order = { + "p1": { + "class": "Directory", + "location": "keep:99999999999999999999999999999994+44", + "listing": [ + { + "class": "File", + "location": "keep:99999999999999999999999999999994+44/file1", + }, + { + "class": "File", + "location": "keep:99999999999999999999999999999994+44/file2", + } + ] + } + } + for j in arvtool.job(job_order, mock.MagicMock(), basedir="", name="test_run_mounts", + make_fs_access=make_fs_access, tmpdir="/tmp"): + j.run() + runner.api.container_requests().create.assert_called_with( + body=JsonDiffMatcher({ + 'environment': { + 'HOME': '/var/spool/cwl', + 'TMPDIR': '/tmp' + }, + 'name': 'test_run_mounts', + 'runtime_constraints': { + 'vcpus': 1, + 'ram': 1073741824 + }, + 'use_existing': True, + 'priority': 1, + 'mounts': { + "/keep/99999999999999999999999999999994+44": { + "kind": "collection", + "portable_data_hash": "99999999999999999999999999999994+44" + }, + '/tmp': {'kind': 'tmp'}, + '/var/spool/cwl': {'kind': 'tmp'} + }, + 'state': 'Committed', + 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', + 'output_path': '/var/spool/cwl', + 'container_image': 'arvados/jobs', + 'command': ['ls', '/var/spool/cwl'], + 'cwd': '/var/spool/cwl', + 'scheduling_parameters': {}, + 'properties': {}, + }))