15227: Fixes type checking on json attributes.

[arvados.git] / sdk / cwl / tests / test_container.py

diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index 46184325f187d18436b28bf6dfbcb162757c0654..07d962bf9b55e2d33efab5b470b3146cab8c49f9 100644 (file)
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -2,8 +2,12 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+from builtins import str
+from builtins import object
+
 import arvados_cwl
 import arvados_cwl.context
+import arvados_cwl.util
 from arvados_cwl.arvdocker import arv_docker_clear_cache
 import copy
 import arvados.config
@@ -70,10 +74,29 @@ class TestContainer(unittest.TestCase):
              "make_fs_access": make_fs_access,
              "tmpdir": "/tmp",
              "enable_reuse": enable_reuse,
-             "priority": 500})
+             "priority": 500,
+             "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
+            })
 
         return loadingContext, runtimeContext
 
+    # Helper function to set up the ArvCwlExecutor to use the containers api
+    # and test that the RuntimeStatusLoggingHandler is set up correctly
+    def setup_and_test_container_executor_and_logging(self, gcc_mock) :
+        api = mock.MagicMock()
+        api._rootDesc = copy.deepcopy(get_rootDesc())
+        del api._rootDesc.get('resources')['jobs']['methods']['create']
+
+        # Make sure ArvCwlExecutor thinks it's running inside a container so it
+        # adds the logging handler that will call runtime_status_update() mock
+        self.assertFalse(gcc_mock.called)
+        runner = arvados_cwl.ArvCwlExecutor(api)
+        self.assertEqual(runner.work_api, 'containers')
+        root_logger = logging.getLogger('')
+        handlerClasses = [h.__class__ for h in root_logger.handlers]
+        self.assertTrue(arvados_cwl.RuntimeStatusLoggingHandler in handlerClasses)
+        return runner
+
     # The test passes no builder.resources
     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
@@ -82,7 +105,6 @@ class TestContainer(unittest.TestCase):
             arv_docker_clear_cache()
 
             runner = mock.MagicMock()
-            runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
             runner.ignore_docker_for_reuse = False
             runner.intermediate_output_ttl = 0
             runner.secret_store = cwltool.secrets.SecretStore()
@@ -132,7 +154,7 @@ class TestContainer(unittest.TestCase):
                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
                         'output_path': '/var/spool/cwl',
                         'output_ttl': 0,
-                        'container_image': 'arvados/jobs',
+                        'container_image': '99999999999999999999999999999993+99',
                         'command': ['ls', '/var/spool/cwl'],
                         'cwd': '/var/spool/cwl',
                         'scheduling_parameters': {},
@@ -146,7 +168,6 @@ class TestContainer(unittest.TestCase):
     def test_resource_requirements(self, keepdocker):
         arv_docker_clear_cache()
         runner = mock.MagicMock()
-        runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
         runner.ignore_docker_for_reuse = False
         runner.intermediate_output_ttl = 3600
         runner.secret_store = cwltool.secrets.SecretStore()
@@ -219,7 +240,7 @@ class TestContainer(unittest.TestCase):
             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
             'output_path': '/var/spool/cwl',
             'output_ttl': 7200,
-            'container_image': 'arvados/jobs',
+            'container_image': '99999999999999999999999999999993+99',
             'command': ['ls'],
             'cwd': '/var/spool/cwl',
             'scheduling_parameters': {
@@ -242,7 +263,6 @@ class TestContainer(unittest.TestCase):
     def test_initial_work_dir(self, collection_mock, keepdocker):
         arv_docker_clear_cache()
         runner = mock.MagicMock()
-        runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
         runner.ignore_docker_for_reuse = False
         runner.intermediate_output_ttl = 0
         runner.secret_store = cwltool.secrets.SecretStore()
@@ -351,7 +371,7 @@ class TestContainer(unittest.TestCase):
             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
             'output_path': '/var/spool/cwl',
             'output_ttl': 0,
-            'container_image': 'arvados/jobs',
+            'container_image': '99999999999999999999999999999993+99',
             'command': ['ls'],
             'cwd': '/var/spool/cwl',
             'scheduling_parameters': {
@@ -372,7 +392,6 @@ class TestContainer(unittest.TestCase):
         arv_docker_clear_cache()
 
         runner = mock.MagicMock()
-        runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
         runner.ignore_docker_for_reuse = False
         runner.intermediate_output_ttl = 0
         runner.secret_store = cwltool.secrets.SecretStore()
@@ -439,7 +458,7 @@ class TestContainer(unittest.TestCase):
                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
                     'output_path': '/var/spool/cwl',
                     'output_ttl': 0,
-                    'container_image': 'arvados/jobs',
+                    'container_image': '99999999999999999999999999999993+99',
                     'command': ['ls', '/var/spool/cwl'],
                     'cwd': '/var/spool/cwl',
                     'scheduling_parameters': {},
@@ -453,7 +472,6 @@ class TestContainer(unittest.TestCase):
 
         runner = mock.MagicMock()
         runner.api = api
-        runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
         runner.num_retries = 0
         runner.ignore_docker_for_reuse = False
         runner.intermediate_output_ttl = 0
@@ -465,7 +483,10 @@ class TestContainer(unittest.TestCase):
 
         col().open.return_value = []
 
+        loadingContext, runtimeContext = self.helper(runner)
+
         arvjob = arvados_cwl.ArvadosContainer(runner,
+                                              runtimeContext,
                                               mock.MagicMock(),
                                               {},
                                               None,
@@ -496,30 +517,31 @@ class TestContainer(unittest.TestCase):
         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 
-    @mock.patch("arvados_cwl.get_current_container")
+    # Test to make sure we dont call runtime_status_update if we already did
+    # some where higher up in the call stack
+    @mock.patch("arvados_cwl.util.get_current_container")
+    def test_recursive_runtime_status_update(self, gcc_mock):
+        self.setup_and_test_container_executor_and_logging(gcc_mock)
+        root_logger = logging.getLogger('')
+
+        # get_current_container is invoked when we call runtime_status_update
+        # so try and log again!
+        gcc_mock.side_effect = lambda *args: root_logger.error("Second Error")
+        try:
+            root_logger.error("First Error")
+        except RuntimeError:
+            self.fail("RuntimeStatusLoggingHandler should not be called recursively")
+
+    @mock.patch("arvados_cwl.ArvCwlExecutor.runtime_status_update")
+    @mock.patch("arvados_cwl.util.get_current_container")
     @mock.patch("arvados.collection.CollectionReader")
     @mock.patch("arvados.collection.Collection")
-    def test_child_failure(self, col, reader, gcc_mock):
-        api = mock.MagicMock()
-        api._rootDesc = copy.deepcopy(get_rootDesc())
-        del api._rootDesc.get('resources')['jobs']['methods']['create']
+    def test_child_failure(self, col, reader, gcc_mock, rts_mock):
+        runner = self.setup_and_test_container_executor_and_logging(gcc_mock)
 
-        # Set up runner with mocked runtime_status_update()
-        self.assertFalse(gcc_mock.called)
-        runtime_status_update = mock.MagicMock()
-        arvados_cwl.ArvCwlRunner.runtime_status_update = runtime_status_update
-        runner = arvados_cwl.ArvCwlRunner(api)
-        self.assertEqual(runner.work_api, 'containers')
-
-        # Make sure ArvCwlRunner thinks it's running inside a container so it
-        # adds the logging handler that will call runtime_status_update() mock
         gcc_mock.return_value = {"uuid" : "zzzzz-dz642-zzzzzzzzzzzzzzz"}
         self.assertTrue(gcc_mock.called)
-        root_logger = logging.getLogger('')
-        handlerClasses = [h.__class__ for h in root_logger.handlers]
-        self.assertTrue(arvados_cwl.RuntimeStatusLoggingHandler in handlerClasses)
 
-        runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
         runner.num_retries = 0
         runner.ignore_docker_for_reuse = False
         runner.intermediate_output_ttl = 0
@@ -536,7 +558,10 @@ class TestContainer(unittest.TestCase):
 
         col().open.return_value = []
 
+        loadingContext, runtimeContext = self.helper(runner)
+
         arvjob = arvados_cwl.ArvadosContainer(runner,
+                                              runtimeContext,
                                               mock.MagicMock(),
                                               {},
                                               None,
@@ -559,7 +584,7 @@ class TestContainer(unittest.TestCase):
             "modified_at": "2017-05-26T12:01:22Z"
         })
 
-        runtime_status_update.assert_called_with(
+        rts_mock.assert_called_with(
             'error',
             'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:',
             '  ** log is empty **'
@@ -573,7 +598,6 @@ class TestContainer(unittest.TestCase):
         arv_docker_clear_cache()
 
         runner = mock.MagicMock()
-        runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
         runner.ignore_docker_for_reuse = False
         runner.intermediate_output_ttl = 0
         runner.secret_store = cwltool.secrets.SecretStore()
@@ -606,6 +630,7 @@ class TestContainer(unittest.TestCase):
             "p1": {
                 "class": "Directory",
                 "location": "keep:99999999999999999999999999999994+44",
+                "http://arvados.org/cwl#collectionUUID": "zzzzz-4zz18-zzzzzzzzzzzzzzz",
                 "listing": [
                     {
                         "class": "File",
@@ -636,7 +661,8 @@ class TestContainer(unittest.TestCase):
                     'mounts': {
                         "/keep/99999999999999999999999999999994+44": {
                             "kind": "collection",
-                            "portable_data_hash": "99999999999999999999999999999994+44"
+                            "portable_data_hash": "99999999999999999999999999999994+44",
+                            "uuid": "zzzzz-4zz18-zzzzzzzzzzzzzzz"
                         },
                         '/tmp': {'kind': 'tmp',
                                  "capacity": 1073741824 },
@@ -648,7 +674,7 @@ class TestContainer(unittest.TestCase):
                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
                     'output_path': '/var/spool/cwl',
                     'output_ttl': 0,
-                    'container_image': 'arvados/jobs',
+                    'container_image': '99999999999999999999999999999994+99',
                     'command': ['ls', '/var/spool/cwl'],
                     'cwd': '/var/spool/cwl',
                     'scheduling_parameters': {},
@@ -663,7 +689,6 @@ class TestContainer(unittest.TestCase):
         arv_docker_clear_cache()
 
         runner = mock.MagicMock()
-        runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
         runner.ignore_docker_for_reuse = False
         runner.intermediate_output_ttl = 0
         runner.secret_store = cwltool.secrets.SecretStore()
@@ -741,7 +766,7 @@ class TestContainer(unittest.TestCase):
                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
                     'output_path': '/var/spool/cwl',
                     'output_ttl': 0,
-                    'container_image': 'arvados/jobs',
+                    'container_image': '99999999999999999999999999999993+99',
                     'command': ['md5sum', 'example.conf'],
                     'cwd': '/var/spool/cwl',
                     'scheduling_parameters': {},
@@ -761,7 +786,6 @@ class TestContainer(unittest.TestCase):
         arv_docker_clear_cache()
 
         runner = mock.MagicMock()
-        runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
         runner.ignore_docker_for_reuse = False
         runner.intermediate_output_ttl = 0
         runner.secret_store = cwltool.secrets.SecretStore()