Merge branch '10534-go-systemd-sdnotify-v14' of https://github.com/wtsi-hgi/arvados

[arvados.git] / sdk / cwl / tests / test_container.py

diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index 7d941cf8fc7c5d5f1caf04d5b60a8aa2b67c1019..bb4bac31dd1767081cdc12a313496a4bb13b4546 100644 (file)
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -1,4 +1,5 @@
 import arvados_cwl
+from arvados_cwl.arvdocker import arv_docker_clear_cache
 import logging
 import mock
 import unittest
@@ -20,59 +21,66 @@ class TestContainer(unittest.TestCase):
     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
     def test_run(self, keepdocker):
-        runner = mock.MagicMock()
-        runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
-        runner.ignore_docker_for_reuse = False
-
-        keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
-        runner.api.collections().get().execute.return_value = {
-            "portable_data_hash": "99999999999999999999999999999993+99"}
-
-        document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("draft-3")
-
-        tool = {
-            "inputs": [],
-            "outputs": [],
-            "baseCommand": "ls",
-            "arguments": [{"valueFrom": "$(runtime.outdir)"}]
-        }
-        make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
-        arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
-                                                 basedir="", make_fs_access=make_fs_access, loader=Loader({}))
-        arvtool.formatgraph = None
-        for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run",
-                             make_fs_access=make_fs_access, tmpdir="/tmp"):
-            j.run()
-            runner.api.container_requests().create.assert_called_with(
-                body={
-                    'environment': {
-                        'HOME': '/var/spool/cwl',
-                        'TMPDIR': '/tmp'
-                    },
-                    'name': 'test_run',
-                    'runtime_constraints': {
-                        'vcpus': 1,
-                        'ram': 1073741824
-                    }, 'priority': 1,
-                    'mounts': {
-                        '/var/spool/cwl': {'kind': 'tmp'}
-                    },
-                    'state': 'Committed',
-                    'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
-                    'output_path': '/var/spool/cwl',
-                    'container_image': '99999999999999999999999999999993+99',
-                    'command': ['ls', '/var/spool/cwl'],
-                    'cwd': '/var/spool/cwl'
-                })
+        for enable_reuse in (True, False):
+            arv_docker_clear_cache()
+
+            runner = mock.MagicMock()
+            runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
+            runner.ignore_docker_for_reuse = False
+
+            keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
+            runner.api.collections().get().execute.return_value = {
+                "portable_data_hash": "99999999999999999999999999999993+99"}
+
+            document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
+
+            tool = {
+                "inputs": [],
+                "outputs": [],
+                "baseCommand": "ls",
+                "arguments": [{"valueFrom": "$(runtime.outdir)"}]
+            }
+            make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, api_client=runner.api)
+            arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names,
+                                                     basedir="", make_fs_access=make_fs_access, loader=Loader({}))
+            arvtool.formatgraph = None
+            for j in arvtool.job({}, mock.MagicMock(), basedir="", name="test_run_"+str(enable_reuse),
+                                 make_fs_access=make_fs_access, tmpdir="/tmp"):
+                j.run(enable_reuse=enable_reuse)
+                runner.api.container_requests().create.assert_called_with(
+                    body={
+                        'environment': {
+                            'HOME': '/var/spool/cwl',
+                            'TMPDIR': '/tmp'
+                        },
+                        'name': 'test_run_'+str(enable_reuse),
+                        'runtime_constraints': {
+                            'vcpus': 1,
+                            'ram': 1073741824
+                        },
+                        'use_existing': enable_reuse,
+                        'priority': 1,
+                        'mounts': {
+                            '/var/spool/cwl': {'kind': 'tmp'}
+                        },
+                        'state': 'Committed',
+                        'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
+                        'output_path': '/var/spool/cwl',
+                        'container_image': '99999999999999999999999999999993+99',
+                        'command': ['ls', '/var/spool/cwl'],
+                        'cwd': '/var/spool/cwl',
+                        'scheduling_parameters': {}
+                    })
 
     # The test passes some fields in builder.resources
     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
     def test_resource_requirements(self, keepdocker):
+        arv_docker_clear_cache()
         runner = mock.MagicMock()
         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
         runner.ignore_docker_for_reuse = False
-        document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("draft-3")
+        document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 
         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
         runner.api.collections().get().execute.return_value = {
@@ -86,6 +94,14 @@ class TestContainer(unittest.TestCase):
                 "coresMin": 3,
                 "ramMin": 3000,
                 "tmpdirMin": 4000
+            }, {
+                "class": "http://arvados.org/cwl#RuntimeConstraints",
+                "keep_cache": 512
+            }, {
+                "class": "http://arvados.org/cwl#APIRequirement",
+            }, {
+                "class": "http://arvados.org/cwl#PartitionRequirement",
+                "partition": "blurb"
             }],
             "baseCommand": "ls"
         }
@@ -98,8 +114,9 @@ class TestContainer(unittest.TestCase):
                              make_fs_access=make_fs_access, tmpdir="/tmp"):
             j.run()
 
-        runner.api.container_requests().create.assert_called_with(
-            body={
+        call_args, call_kwargs = runner.api.container_requests().create.call_args
+
+        call_body_expected = {
                 'environment': {
                     'HOME': '/var/spool/cwl',
                     'TMPDIR': '/tmp'
@@ -107,8 +124,12 @@ class TestContainer(unittest.TestCase):
                 'name': 'test_resource_requirements',
                 'runtime_constraints': {
                     'vcpus': 3,
-                    'ram': 3145728000
-                }, 'priority': 1,
+                    'ram': 3145728000,
+                    'keep_cache_ram': 512,
+                    'API': True
+                },
+                'use_existing': True,
+                'priority': 1,
                 'mounts': {
                     '/var/spool/cwl': {'kind': 'tmp'}
                 },
@@ -117,8 +138,16 @@ class TestContainer(unittest.TestCase):
                 'output_path': '/var/spool/cwl',
                 'container_image': '99999999999999999999999999999993+99',
                 'command': ['ls'],
-                'cwd': '/var/spool/cwl'
-            })
+                'cwd': '/var/spool/cwl',
+                'scheduling_parameters': {
+                    'partitions': ['blurb']
+                }
+        }
+
+        call_body = call_kwargs.get('body', None)
+        self.assertNotEqual(None, call_body)
+        for key in call_body:
+            self.assertEqual(call_body_expected.get(key), call_body.get(key))
 
     @mock.patch("arvados.collection.Collection")
     def test_done(self, col):