projects / arvados.git / blobdiff
summary | shortlog | log | commit | commitdiff | tree
raw | inline | side by side
11308: Merge branch 'master' into 11308-python3
[arvados.git] / sdk / cwl / tests / test_container.py
diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index 7f67b7a0b3710ba7d37d48265dc75de6ff55b3a3..b06eae8105aad06d35ece8611f3b7a5103ab838c 100644 (file)
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -41,7 +41,8 @@ class TestContainer(unittest.TestCase):
                 "baseCommand": "ls",
                 "arguments": [{"valueFrom": "$(runtime.outdir)"}]
             })
-            make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
+            make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
+                                         collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
             arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
                                                      basedir="", make_fs_access=make_fs_access, loader=Loader({}))
             arvtool.formatgraph = None
@@ -62,12 +63,13 @@ class TestContainer(unittest.TestCase):
                         'use_existing': enable_reuse,
                         'priority': 1,
                         'mounts': {
+                            '/tmp': {'kind': 'tmp'},
                             '/var/spool/cwl': {'kind': 'tmp'}
                         },
                         'state': 'Committed',
                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
                         'output_path': '/var/spool/cwl',
-                        'container_image': '99999999999999999999999999999993+99',
+                        'container_image': 'arvados/jobs',
                         'command': ['ls', '/var/spool/cwl'],
                         'cwd': '/var/spool/cwl',
                         'scheduling_parameters': {},
@@ -107,7 +109,8 @@ class TestContainer(unittest.TestCase):
             }],
             "baseCommand": "ls"
         })
-        make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
+        make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
+                                         collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers",
                                                  avsc_names=avsc_names, make_fs_access=make_fs_access,
                                                  loader=Loader({}))
@@ -133,12 +136,13 @@ class TestContainer(unittest.TestCase):
                 'use_existing': True,
                 'priority': 1,
                 'mounts': {
+                    '/tmp': {'kind': 'tmp'},
                     '/var/spool/cwl': {'kind': 'tmp'}
                 },
                 'state': 'Committed',
                 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
                 'output_path': '/var/spool/cwl',
-                'container_image': '99999999999999999999999999999993+99',
+                'container_image': 'arvados/jobs',
                 'command': ['ls'],
                 'cwd': '/var/spool/cwl',
                 'scheduling_parameters': {
@@ -169,7 +173,12 @@ class TestContainer(unittest.TestCase):
             "portable_data_hash": "99999999999999999999999999999993+99"}
 
         sourcemock = mock.MagicMock()
-        runner.fs_access.get_collection.return_value = (sourcemock, "bar")
+        def get_collection_mock(p):
+            if "/" in p:
+                return (sourcemock, p.split("/", 1)[1])
+            else:
+                return (sourcemock, "")
+        runner.fs_access.get_collection.side_effect = get_collection_mock
 
         vwdmock = mock.MagicMock()
         collection_mock.return_value = vwdmock
@@ -188,12 +197,23 @@ class TestContainer(unittest.TestCase):
                 {
                     "class": "Directory",
                     "basename": "foo2",
-                    "location": "keep:99999999999999999999999999999997+99"
-                }]
+                    "location": "keep:99999999999999999999999999999995+99"
+                },
+                {
+                    "class": "File",
+                    "basename": "filename",
+                    "location": "keep:99999999999999999999999999999995+99/baz/filename"
+                },
+                {
+                    "class": "Directory",
+                    "basename": "subdir",
+                    "location": "keep:99999999999999999999999999999995+99/subdir"
+                }                        ]
             }],
             "baseCommand": "ls"
         })
-        make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
+        make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
+                                         collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers",
                                                  avsc_names=avsc_names, make_fs_access=make_fs_access,
                                                  loader=Loader({}))
@@ -204,8 +224,10 @@ class TestContainer(unittest.TestCase):
 
         call_args, call_kwargs = runner.api.container_requests().create.call_args
 
-        vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)],
-                                      [mock.call('', 'foo2', source_collection=sourcemock)])
+        vwdmock.copy.assert_has_calls([mock.call('bar', 'foo', source_collection=sourcemock)])
+        vwdmock.copy.assert_has_calls([mock.call('', 'foo2', source_collection=sourcemock)])
+        vwdmock.copy.assert_has_calls([mock.call('baz/filename', 'filename', source_collection=sourcemock)])
+        vwdmock.copy.assert_has_calls([mock.call('subdir', 'subdir', source_collection=sourcemock)])
 
         call_body_expected = {
                 'environment': {
@@ -220,6 +242,7 @@ class TestContainer(unittest.TestCase):
                 'use_existing': True,
                 'priority': 1,
                 'mounts': {
+                    '/tmp': {'kind': 'tmp'},
                     '/var/spool/cwl': {'kind': 'tmp'},
                     '/var/spool/cwl/foo': {
                         'kind': 'collection',
@@ -230,12 +253,22 @@ class TestContainer(unittest.TestCase):
                         'kind': 'collection',
                         'path': 'foo2',
                         'portable_data_hash': '99999999999999999999999999999996+99'
+                    },
+                    '/var/spool/cwl/filename': {
+                        'kind': 'collection',
+                        'path': 'filename',
+                        'portable_data_hash': '99999999999999999999999999999996+99'
+                    },
+                    '/var/spool/cwl/subdir': {
+                        'kind': 'collection',
+                        'path': 'subdir',
+                        'portable_data_hash': '99999999999999999999999999999996+99'
                     }
                 },
                 'state': 'Committed',
                 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
                 'output_path': '/var/spool/cwl',
-                'container_image': '99999999999999999999999999999993+99',
+                'container_image': 'arvados/jobs',
                 'command': ['ls'],
                 'cwd': '/var/spool/cwl',
                 'scheduling_parameters': {
@@ -248,6 +281,79 @@ class TestContainer(unittest.TestCase):
         for key in call_body:
             self.assertEqual(call_body_expected.get(key), call_body.get(key))
 
+
+    # Test redirecting stdin/stdout/stderr
+    @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
+    def test_redirects(self, keepdocker):
+        arv_docker_clear_cache()
+
+        runner = mock.MagicMock()
+        runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
+        runner.ignore_docker_for_reuse = False
+
+        keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
+        runner.api.collections().get().execute.return_value = {
+            "portable_data_hash": "99999999999999999999999999999993+99"}
+
+        document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
+
+        tool = cmap({
+            "inputs": [],
+            "outputs": [],
+            "baseCommand": "ls",
+            "stdout": "stdout.txt",
+            "stderr": "stderr.txt",
+            "stdin": "/keep/99999999999999999999999999999996+99/file.txt",
+            "arguments": [{"valueFrom": "$(runtime.outdir)"}]
+        })
+        make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
+                                         collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
+        arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
+                                                 basedir="", make_fs_access=make_fs_access, loader=Loader({}))
+        arvtool.formatgraph = None
+        for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run_redirect",
+                             make_fs_access=make_fs_access, tmpdir="/tmp"):
+            j.run()
+            runner.api.container_requests().create.assert_called_with(
+                body=JsonDiffMatcher({
+                    'environment': {
+                        'HOME': '/var/spool/cwl',
+                        'TMPDIR': '/tmp'
+                    },
+                    'name': 'test_run_redirect',
+                    'runtime_constraints': {
+                        'vcpus': 1,
+                        'ram': 1073741824
+                    },
+                    'use_existing': True,
+                    'priority': 1,
+                    'mounts': {
+                        '/tmp': {'kind': 'tmp'},
+                        '/var/spool/cwl': {'kind': 'tmp'},
+                        "stderr": {
+                            "kind": "file",
+                            "path": "/var/spool/cwl/stderr.txt"
+                        },
+                        "stdin": {
+                            "kind": "collection",
+                            "path": "file.txt",
+                            "portable_data_hash": "99999999999999999999999999999996+99"
+                        },
+                        "stdout": {
+                            "kind": "file",
+                            "path": "/var/spool/cwl/stdout.txt"
+                        },
+                    },
+                    'state': 'Committed',
+                    'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
+                    'output_path': '/var/spool/cwl',
+                    'container_image': 'arvados/jobs',
+                    'command': ['ls', '/var/spool/cwl'],
+                    'cwd': '/var/spool/cwl',
+                    'scheduling_parameters': {},
+                    'properties': {},
+                }))
+
     @mock.patch("arvados.collection.Collection")
     def test_done(self, col):
         api = mock.MagicMock()
@@ -286,3 +392,83 @@ class TestContainer(unittest.TestCase):
 
         arvjob.collect_outputs.assert_called_with("keep:abc+123")
         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
+
+    # The test passes no builder.resources
+    # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
+    @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
+    def test_mounts(self, keepdocker):
+        arv_docker_clear_cache()
+
+        runner = mock.MagicMock()
+        runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
+        runner.ignore_docker_for_reuse = False
+
+        keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
+        runner.api.collections().get().execute.return_value = {
+            "portable_data_hash": "99999999999999999999999999999993+99"}
+
+        document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
+
+        tool = cmap({
+            "inputs": [
+                {"id": "p1",
+                 "type": "Directory"}
+            ],
+            "outputs": [],
+            "baseCommand": "ls",
+            "arguments": [{"valueFrom": "$(runtime.outdir)"}]
+        })
+        make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
+                                     collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
+        arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
+                                                 basedir="", make_fs_access=make_fs_access, loader=Loader({}))
+        arvtool.formatgraph = None
+        job_order = {
+            "p1": {
+                "class": "Directory",
+                "location": "keep:99999999999999999999999999999994+44",
+                "listing": [
+                    {
+                        "class": "File",
+                        "location": "keep:99999999999999999999999999999994+44/file1",
+                    },
+                    {
+                        "class": "File",
+                        "location": "keep:99999999999999999999999999999994+44/file2",
+                    }
+                ]
+            }
+        }
+        for j in arvtool.job(job_order, mock.MagicMock(), basedir="", name="test_run_mounts",
+                             make_fs_access=make_fs_access, tmpdir="/tmp"):
+            j.run()
+            runner.api.container_requests().create.assert_called_with(
+                body=JsonDiffMatcher({
+                    'environment': {
+                        'HOME': '/var/spool/cwl',
+                        'TMPDIR': '/tmp'
+                    },
+                    'name': 'test_run_mounts',
+                    'runtime_constraints': {
+                        'vcpus': 1,
+                        'ram': 1073741824
+                    },
+                    'use_existing': True,
+                    'priority': 1,
+                    'mounts': {
+                        "/keep/99999999999999999999999999999994+44": {
+                            "kind": "collection",
+                            "portable_data_hash": "99999999999999999999999999999994+44"
+                        },
+                        '/tmp': {'kind': 'tmp'},
+                        '/var/spool/cwl': {'kind': 'tmp'}
+                    },
+                    'state': 'Committed',
+                    'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
+                    'output_path': '/var/spool/cwl',
+                    'container_image': 'arvados/jobs',
+                    'command': ['ls', '/var/spool/cwl'],
+                    'cwd': '/var/spool/cwl',
+                    'scheduling_parameters': {},
+                    'properties': {},
+                }))