Merge branch '19385-cwl-fast-pack' refs #19385

[arvados.git] / sdk / cwl / tests / test_container.py

diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index cb57b446da5877b4e74ef0b30ae7f930e37b0c12..a2f404d7ebe9b24b3d726d6057be14848b088cdd 100644 (file)
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -8,7 +8,7 @@ from builtins import object
 import arvados_cwl
 import arvados_cwl.context
 import arvados_cwl.util
-from arvados_cwl.arvdocker import arv_docker_clear_cache
+#from arvados_cwl.arvdocker import arv_docker_clear_cache
 import copy
 import arvados.config
 import logging
@@ -61,7 +61,7 @@ class TestContainer(unittest.TestCase):
 
     def setUp(self):
         cwltool.process._names = set()
-        arv_docker_clear_cache()
+        #arv_docker_clear_cache()
 
     def tearDown(self):
         root_logger = logging.getLogger('')
@@ -128,13 +128,14 @@ class TestContainer(unittest.TestCase):
     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
     def test_run(self, keepdocker):
         for enable_reuse in (True, False):
-            arv_docker_clear_cache()
+            #arv_docker_clear_cache()
 
             runner = mock.MagicMock()
             runner.ignore_docker_for_reuse = False
             runner.intermediate_output_ttl = 0
             runner.secret_store = cwltool.secrets.SecretStore()
             runner.api._rootDesc = {"revision": "20210628"}
+            runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 
             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
             runner.api.collections().get().execute.return_value = {
@@ -186,7 +187,7 @@ class TestContainer(unittest.TestCase):
                         'command': ['ls', '/var/spool/cwl'],
                         'cwd': '/var/spool/cwl',
                         'scheduling_parameters': {},
-                        'properties': {},
+                        'properties': {'cwl_input': {}},
                         'secret_mounts': {},
                         'output_storage_classes': ["default"]
                     }))
@@ -201,6 +202,7 @@ class TestContainer(unittest.TestCase):
         runner.intermediate_output_ttl = 3600
         runner.secret_store = cwltool.secrets.SecretStore()
         runner.api._rootDesc = {"revision": "20210628"}
+        runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 
         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
         runner.api.collections().get().execute.return_value = {
@@ -277,7 +279,7 @@ class TestContainer(unittest.TestCase):
             'scheduling_parameters': {
                 'partitions': ['blurb']
             },
-            'properties': {},
+            'properties': {'cwl_input': {}},
             'secret_mounts': {},
             'output_storage_classes': ["default"]
         }
@@ -298,6 +300,7 @@ class TestContainer(unittest.TestCase):
         runner.intermediate_output_ttl = 0
         runner.secret_store = cwltool.secrets.SecretStore()
         runner.api._rootDesc = {"revision": "20210628"}
+        runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 
         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
         runner.api.collections().get().execute.return_value = {
@@ -410,7 +413,7 @@ class TestContainer(unittest.TestCase):
             'cwd': '/var/spool/cwl',
             'scheduling_parameters': {
             },
-            'properties': {},
+            'properties': {'cwl_input': {}},
             'secret_mounts': {},
             'output_storage_classes': ["default"]
         }
@@ -429,6 +432,7 @@ class TestContainer(unittest.TestCase):
         runner.intermediate_output_ttl = 0
         runner.secret_store = cwltool.secrets.SecretStore()
         runner.api._rootDesc = {"revision": "20210628"}
+        runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 
         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
         runner.api.collections().get().execute.return_value = {
@@ -497,7 +501,7 @@ class TestContainer(unittest.TestCase):
                     'command': ['ls', '/var/spool/cwl'],
                     'cwd': '/var/spool/cwl',
                     'scheduling_parameters': {},
-                    'properties': {},
+                    'properties': {'cwl_input': {}},
                     'secret_mounts': {},
                     'output_storage_classes': ["default"]
                 }))
@@ -534,6 +538,7 @@ class TestContainer(unittest.TestCase):
         arvjob.successCodes = [0]
         arvjob.outdir = "/var/spool/cwl"
         arvjob.output_ttl = 3600
+        arvjob.uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzz1"
 
         arvjob.collect_outputs.return_value = {"out": "stuff"}
 
@@ -543,7 +548,8 @@ class TestContainer(unittest.TestCase):
             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
-            "modified_at": "2017-05-26T12:01:22Z"
+            "modified_at": "2017-05-26T12:01:22Z",
+            "properties": {}
         })
 
         self.assertFalse(api.collections().create.called)
@@ -553,6 +559,10 @@ class TestContainer(unittest.TestCase):
         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 
+        runner.api.container_requests().update.assert_called_with(uuid="zzzzz-xvhdp-zzzzzzzzzzzzzz1",
+                                                                  body={'container_request': {'properties': {'cwl_output': {'out': 'stuff'}}}})
+
+
     # Test to make sure we dont call runtime_status_update if we already did
     # some where higher up in the call stack
     @mock.patch("arvados_cwl.util.get_current_container")
@@ -636,7 +646,8 @@ class TestContainer(unittest.TestCase):
             "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
             "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
             "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
-            "modified_at": "2017-05-26T12:01:22Z"
+            "modified_at": "2017-05-26T12:01:22Z",
+            "properties": {}
         })
 
         rts_mock.assert_called_with(
@@ -655,6 +666,7 @@ class TestContainer(unittest.TestCase):
         runner.intermediate_output_ttl = 0
         runner.secret_store = cwltool.secrets.SecretStore()
         runner.api._rootDesc = {"revision": "20210628"}
+        runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 
         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
         runner.api.collections().get().execute.return_value = {
@@ -733,7 +745,38 @@ class TestContainer(unittest.TestCase):
                     'command': ['ls', '/var/spool/cwl'],
                     'cwd': '/var/spool/cwl',
                     'scheduling_parameters': {},
-                    'properties': {},
+                    'properties': {'cwl_input': {
+                        "p1": {
+                            "basename": "99999999999999999999999999999994+44",
+                            "class": "Directory",
+                            "dirname": "/keep",
+                            "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
+                            "listing": [
+                                {
+                                    "basename": "file1",
+                                    "class": "File",
+                                    "dirname": "/keep/99999999999999999999999999999994+44",
+                                    "location": "keep:99999999999999999999999999999994+44/file1",
+                                    "nameext": "",
+                                    "nameroot": "file1",
+                                    "path": "/keep/99999999999999999999999999999994+44/file1",
+                                    "size": 0
+                                },
+                                {
+                                    "basename": "file2",
+                                    "class": "File",
+                                    "dirname": "/keep/99999999999999999999999999999994+44",
+                                    "location": "keep:99999999999999999999999999999994+44/file2",
+                                    "nameext": "",
+                                    "nameroot": "file2",
+                                    "path": "/keep/99999999999999999999999999999994+44/file2",
+                                    "size": 0
+                                }
+                            ],
+                            "location": "keep:99999999999999999999999999999994+44",
+                            "path": "/keep/99999999999999999999999999999994+44"
+                        }
+                    }},
                     'secret_mounts': {},
                     'output_storage_classes': ["default"]
                 }))
@@ -748,6 +791,7 @@ class TestContainer(unittest.TestCase):
         runner.intermediate_output_ttl = 0
         runner.secret_store = cwltool.secrets.SecretStore()
         runner.api._rootDesc = {"revision": "20210628"}
+        runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 
         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
         runner.api.collections().get().execute.return_value = {
@@ -827,7 +871,7 @@ class TestContainer(unittest.TestCase):
                     'command': ['md5sum', 'example.conf'],
                     'cwd': '/var/spool/cwl',
                     'scheduling_parameters': {},
-                    'properties': {},
+                    'properties': {'cwl_input': job_order},
                     "secret_mounts": {
                         "/var/spool/cwl/example.conf": {
                             "content": "username: user\npassword: blorp\n",
@@ -846,6 +890,7 @@ class TestContainer(unittest.TestCase):
         runner.intermediate_output_ttl = 0
         runner.secret_store = cwltool.secrets.SecretStore()
         runner.api._rootDesc = {"revision": "20210628"}
+        runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 
         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
         runner.api.collections().get().execute.return_value = {
@@ -884,13 +929,14 @@ class TestContainer(unittest.TestCase):
     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
     def test_setting_storage_class(self, keepdocker):
-        arv_docker_clear_cache()
+        #arv_docker_clear_cache()
 
         runner = mock.MagicMock()
         runner.ignore_docker_for_reuse = False
         runner.intermediate_output_ttl = 0
         runner.secret_store = cwltool.secrets.SecretStore()
         runner.api._rootDesc = {"revision": "20210628"}
+        runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 
         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
         runner.api.collections().get().execute.return_value = {
@@ -949,7 +995,7 @@ class TestContainer(unittest.TestCase):
                     'command': ['ls', '/var/spool/cwl'],
                     'cwd': '/var/spool/cwl',
                     'scheduling_parameters': {},
-                    'properties': {},
+                    'properties': {'cwl_input': {}},
                     'secret_mounts': {},
                     'output_storage_classes': ["foo_sc", "bar_sc"]
                 }))
@@ -959,13 +1005,14 @@ class TestContainer(unittest.TestCase):
     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
     def test_setting_process_properties(self, keepdocker):
-        arv_docker_clear_cache()
+        #arv_docker_clear_cache()
 
         runner = mock.MagicMock()
         runner.ignore_docker_for_reuse = False
         runner.intermediate_output_ttl = 0
         runner.secret_store = cwltool.secrets.SecretStore()
         runner.api._rootDesc = {"revision": "20210628"}
+        runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 
         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
         runner.api.collections().get().execute.return_value = {
@@ -1037,6 +1084,7 @@ class TestContainer(unittest.TestCase):
                     'scheduling_parameters': {},
                     'properties': {
                         "baz": "blorp",
+                        "cwl_input": {"x": "blorp"},
                         "foo": "bar",
                         "quux": {
                             "q1": 1,
@@ -1053,13 +1101,14 @@ class TestContainer(unittest.TestCase):
     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
     def test_cuda_requirement(self, keepdocker):
         arvados_cwl.add_arv_hints()
-        arv_docker_clear_cache()
+        #arv_docker_clear_cache()
 
         runner = mock.MagicMock()
         runner.ignore_docker_for_reuse = False
         runner.intermediate_output_ttl = 0
         runner.secret_store = cwltool.secrets.SecretStore()
         runner.api._rootDesc = {"revision": "20210628"}
+        runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 
         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
         runner.api.collections().get().execute.return_value = {
@@ -1145,7 +1194,7 @@ class TestContainer(unittest.TestCase):
                         'command': ['nvidia-smi'],
                         'cwd': '/var/spool/cwl',
                         'scheduling_parameters': {},
-                        'properties': {},
+                        'properties': {'cwl_input': {}},
                         'secret_mounts': {},
                         'output_storage_classes': ["default"]
                     }))
@@ -1157,13 +1206,13 @@ class TestContainer(unittest.TestCase):
     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
     def test_match_local_docker(self, keepdocker, determine_image_id):
         arvados_cwl.add_arv_hints()
-        arv_docker_clear_cache()
 
         runner = mock.MagicMock()
         runner.ignore_docker_for_reuse = False
         runner.intermediate_output_ttl = 0
         runner.secret_store = cwltool.secrets.SecretStore()
         runner.api._rootDesc = {"revision": "20210628"}
+        runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 
         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz4", {"dockerhash": "456"}),
                                    ("zzzzz-4zz18-zzzzzzzzzzzzzz3", {"dockerhash": "123"})]
@@ -1219,7 +1268,7 @@ class TestContainer(unittest.TestCase):
             'command': ['echo'],
             'cwd': '/var/spool/cwl',
             'scheduling_parameters': {},
-            'properties': {},
+            'properties': {'cwl_input': {}},
             'secret_mounts': {},
             'output_storage_classes': ["default"]
         }
@@ -1230,7 +1279,7 @@ class TestContainer(unittest.TestCase):
             runner.api.container_requests().create.assert_called_with(
                 body=JsonDiffMatcher(container_request))
 
-        arv_docker_clear_cache()
+        runtimeContext.cached_docker_lookups.clear()
         runtimeContext.match_local_docker = True
         container_request['container_image'] = '99999999999999999999999999999993+99'
         container_request['name'] = 'test_run_True_2'
@@ -1248,13 +1297,14 @@ class TestContainer(unittest.TestCase):
         arvados_cwl.add_arv_hints()
         for enable_preemptible in (None, True, False):
             for preemptible_hint in (None, True, False):
-                arv_docker_clear_cache()
+                #arv_docker_clear_cache()
 
                 runner = mock.MagicMock()
                 runner.ignore_docker_for_reuse = False
                 runner.intermediate_output_ttl = 0
                 runner.secret_store = cwltool.secrets.SecretStore()
                 runner.api._rootDesc = {"revision": "20210628"}
+                runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 
                 keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
                 runner.api.collections().get().execute.return_value = {
@@ -1332,7 +1382,7 @@ class TestContainer(unittest.TestCase):
                             'command': ['ls', '/var/spool/cwl'],
                             'cwd': '/var/spool/cwl',
                             'scheduling_parameters': sched,
-                            'properties': {},
+                            'properties': {'cwl_input': {}},
                             'secret_mounts': {},
                             'output_storage_classes': ["default"]
                         }))
@@ -1347,6 +1397,7 @@ class TestContainer(unittest.TestCase):
             runner.intermediate_output_ttl = 0
             runner.secret_store = cwltool.secrets.SecretStore()
             runner.api._rootDesc = {"revision": rev}
+            runner.api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 
             keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
             runner.api.collections().get().execute.return_value = {
@@ -1393,7 +1444,7 @@ class TestContainer(unittest.TestCase):
 class TestWorkflow(unittest.TestCase):
     def setUp(self):
         cwltool.process._names = set()
-        arv_docker_clear_cache()
+        #arv_docker_clear_cache()
 
     def helper(self, runner, enable_reuse=True):
         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
@@ -1434,6 +1485,7 @@ class TestWorkflow(unittest.TestCase):
 
         api = mock.MagicMock()
         api._rootDesc = get_rootDesc()
+        api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 
         runner = arvados_cwl.executor.ArvCwlExecutor(api)
         self.assertEqual(runner.work_api, 'containers')
@@ -1521,7 +1573,19 @@ class TestWorkflow(unittest.TestCase):
                 "output_path": "/var/spool/cwl",
                 "output_ttl": 0,
                 "priority": 500,
-                "properties": {},
+                "properties": {'cwl_input': {
+                        "fileblub": {
+                            "basename": "token.txt",
+                            "class": "File",
+                            "dirname": "/keep/99999999999999999999999999999999+118",
+                            "location": "keep:99999999999999999999999999999999+118/token.txt",
+                            "nameext": ".txt",
+                            "nameroot": "token",
+                            "path": "/keep/99999999999999999999999999999999+118/token.txt",
+                            "size": 0
+                        },
+                        "sleeptime": 5
+                }},
                 "runtime_constraints": {
                     "ram": 1073741824,
                     "vcpus": 1
@@ -1554,6 +1618,7 @@ class TestWorkflow(unittest.TestCase):
 
         api = mock.MagicMock()
         api._rootDesc = get_rootDesc()
+        api.config.return_value = {"Containers": {"DefaultKeepCacheRAM": 256<<20}}
 
         runner = arvados_cwl.executor.ArvCwlExecutor(api)
         self.assertEqual(runner.work_api, 'containers')
@@ -1594,7 +1659,7 @@ class TestWorkflow(unittest.TestCase):
                 'name': u'echo-subwf',
                 'secret_mounts': {},
                 'runtime_constraints': {'API': True, 'vcpus': 3, 'ram': 1073741824},
-                'properties': {},
+                'properties': {'cwl_input': {}},
                 'priority': 500,
                 'mounts': {
                     '/var/spool/cwl/cwl.input.yml': {