Merge branch '17756-dispatch-lsf' into main

[arvados.git] / sdk / cwl / tests / test_container.py

diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index 3374e1c13f8004100c2f3c114edbfba2db26dec6..09983f87a2cea8f46b8790e6749c05a5bc57d8f8 100644 (file)
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -18,6 +18,7 @@ import os
 import functools
 import cwltool.process
 import cwltool.secrets
+from cwltool.update import INTERNAL_VERSION
 from schema_salad.ref_resolver import Loader
 from schema_salad.sourceline import cmap
 
@@ -58,9 +59,10 @@ class TestContainer(unittest.TestCase):
 
     def setUp(self):
         cwltool.process._names = set()
+        arv_docker_clear_cache()
 
     def helper(self, runner, enable_reuse=True):
-        document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
+        document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
 
         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
@@ -69,7 +71,7 @@ class TestContainer(unittest.TestCase):
              "basedir": "",
              "make_fs_access": make_fs_access,
              "loader": Loader({}),
-             "metadata": {"cwlVersion": "v1.1", "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}})
+             "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}})
         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
             {"work_api": "containers",
              "basedir": "",
@@ -168,7 +170,6 @@ class TestContainer(unittest.TestCase):
     # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
     def test_resource_requirements(self, keepdocker):
-        arv_docker_clear_cache()
         runner = mock.MagicMock()
         runner.ignore_docker_for_reuse = False
         runner.intermediate_output_ttl = 3600
@@ -263,7 +264,6 @@ class TestContainer(unittest.TestCase):
     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
     @mock.patch("arvados.collection.Collection")
     def test_initial_work_dir(self, collection_mock, keepdocker):
-        arv_docker_clear_cache()
         runner = mock.MagicMock()
         runner.ignore_docker_for_reuse = False
         runner.intermediate_output_ttl = 0
@@ -391,8 +391,6 @@ class TestContainer(unittest.TestCase):
     # Test redirecting stdin/stdout/stderr
     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
     def test_redirects(self, keepdocker):
-        arv_docker_clear_cache()
-
         runner = mock.MagicMock()
         runner.ignore_docker_for_reuse = False
         runner.intermediate_output_ttl = 0
@@ -402,7 +400,7 @@ class TestContainer(unittest.TestCase):
         runner.api.collections().get().execute.return_value = {
             "portable_data_hash": "99999999999999999999999999999993+99"}
 
-        document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
+        document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema(INTERNAL_VERSION)
 
         tool = cmap({
             "inputs": [],
@@ -534,6 +532,25 @@ class TestContainer(unittest.TestCase):
         except RuntimeError:
             self.fail("RuntimeStatusLoggingHandler should not be called recursively")
 
+
+    # Test to make sure that an exception raised from
+    # get_current_container doesn't cause the logger to raise an
+    # exception
+    @mock.patch("arvados_cwl.util.get_current_container")
+    def test_runtime_status_get_current_container_exception(self, gcc_mock):
+        self.setup_and_test_container_executor_and_logging(gcc_mock)
+        root_logger = logging.getLogger('')
+
+        # get_current_container is invoked when we call
+        # runtime_status_update, it is going to also raise an
+        # exception.
+        gcc_mock.side_effect = Exception("Second Error")
+        try:
+            root_logger.error("First Error")
+        except Exception:
+            self.fail("Exception in logger should not propagate")
+        self.assertTrue(gcc_mock.called)
+
     @mock.patch("arvados_cwl.ArvCwlExecutor.runtime_status_update")
     @mock.patch("arvados_cwl.util.get_current_container")
     @mock.patch("arvados.collection.CollectionReader")
@@ -597,8 +614,6 @@ class TestContainer(unittest.TestCase):
     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
     def test_mounts(self, keepdocker):
-        arv_docker_clear_cache()
-
         runner = mock.MagicMock()
         runner.ignore_docker_for_reuse = False
         runner.intermediate_output_ttl = 0
@@ -688,8 +703,6 @@ class TestContainer(unittest.TestCase):
     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
     def test_secrets(self, keepdocker):
-        arv_docker_clear_cache()
-
         runner = mock.MagicMock()
         runner.ignore_docker_for_reuse = False
         runner.intermediate_output_ttl = 0
@@ -785,8 +798,6 @@ class TestContainer(unittest.TestCase):
     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
     def test_timelimit(self, keepdocker):
-        arv_docker_clear_cache()
-
         runner = mock.MagicMock()
         runner.ignore_docker_for_reuse = False
         runner.intermediate_output_ttl = 0
@@ -827,9 +838,10 @@ class TestContainer(unittest.TestCase):
 class TestWorkflow(unittest.TestCase):
     def setUp(self):
         cwltool.process._names = set()
+        arv_docker_clear_cache()
 
     def helper(self, runner, enable_reuse=True):
-        document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
+        document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
 
         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
@@ -844,7 +856,7 @@ class TestWorkflow(unittest.TestCase):
              "basedir": "",
              "make_fs_access": make_fs_access,
              "loader": document_loader,
-             "metadata": {"cwlVersion": "v1.1", "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
+             "metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
              "construct_tool_object": runner.arv_make_tool})
         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
             {"work_api": "containers",
@@ -863,7 +875,6 @@ class TestWorkflow(unittest.TestCase):
     @mock.patch("arvados.collection.Collection")
     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
     def test_run(self, list_images_in_arv, mockcollection, mockcollectionreader):
-        arv_docker_clear_cache()
         arvados_cwl.add_arv_hints()
 
         api = mock.MagicMock()
@@ -885,6 +896,8 @@ class TestWorkflow(unittest.TestCase):
         loadingContext, runtimeContext = self.helper(runner)
         runner.fs_access = runtimeContext.make_fs_access(runtimeContext.basedir)
 
+        mockcollectionreader().exists.return_value = True
+
         tool, metadata = loadingContext.loader.resolve_ref("tests/wf/scatter2.cwl")
         metadata["cwlVersion"] = tool["cwlVersion"]
 
@@ -909,7 +922,7 @@ class TestWorkflow(unittest.TestCase):
                     "--no-container",
                     "--move-outputs",
                     "--preserve-entire-environment",
-                    "workflow.cwl#main",
+                    "workflow.cwl",
                     "cwl.input.yml"
                 ],
                 "container_image": "99999999999999999999999999999993+99",
@@ -979,7 +992,6 @@ class TestWorkflow(unittest.TestCase):
     @mock.patch("arvados.collection.Collection")
     @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
     def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader):
-        arv_docker_clear_cache()
         arvados_cwl.add_arv_hints()
 
         api = mock.MagicMock()
@@ -1057,7 +1069,7 @@ class TestWorkflow(unittest.TestCase):
                     u'--no-container',
                     u'--move-outputs',
                     u'--preserve-entire-environment',
-                    u'workflow.cwl#main',
+                    u'workflow.cwl',
                     u'cwl.input.yml'
                 ],
                 'use_existing': True,