14360: Merge branch 'master' into 14360-dispatch-cloud

[arvados.git] / sdk / cwl / tests / test_container.py

diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index 69f3ae046e31ca3e02aa52be7967e43da2394d00..a34d550f62ad2b24a1ff156d7b740377c811cab5 100644 (file)
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -4,7 +4,9 @@
 
 import arvados_cwl
 import arvados_cwl.context
+import arvados_cwl.util
 from arvados_cwl.arvdocker import arv_docker_clear_cache
+import copy
 import arvados.config
 import logging
 import mock
@@ -17,6 +19,7 @@ from schema_salad.ref_resolver import Loader
 from schema_salad.sourceline import cmap
 
 from .matcher import JsonDiffMatcher
+from .mock_discovery import get_rootDesc
 
 if not os.getenv('ARVADOS_DEBUG'):
     logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
@@ -68,7 +71,9 @@ class TestContainer(unittest.TestCase):
              "make_fs_access": make_fs_access,
              "tmpdir": "/tmp",
              "enable_reuse": enable_reuse,
-             "priority": 500})
+             "priority": 500,
+             "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
+            })
 
         return loadingContext, runtimeContext
 
@@ -80,7 +85,6 @@ class TestContainer(unittest.TestCase):
             arv_docker_clear_cache()
 
             runner = mock.MagicMock()
-            runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
             runner.ignore_docker_for_reuse = False
             runner.intermediate_output_ttl = 0
             runner.secret_store = cwltool.secrets.SecretStore()
@@ -130,7 +134,7 @@ class TestContainer(unittest.TestCase):
                         'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
                         'output_path': '/var/spool/cwl',
                         'output_ttl': 0,
-                        'container_image': 'arvados/jobs',
+                        'container_image': '99999999999999999999999999999993+99',
                         'command': ['ls', '/var/spool/cwl'],
                         'cwd': '/var/spool/cwl',
                         'scheduling_parameters': {},
@@ -144,7 +148,6 @@ class TestContainer(unittest.TestCase):
     def test_resource_requirements(self, keepdocker):
         arv_docker_clear_cache()
         runner = mock.MagicMock()
-        runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
         runner.ignore_docker_for_reuse = False
         runner.intermediate_output_ttl = 3600
         runner.secret_store = cwltool.secrets.SecretStore()
@@ -217,7 +220,7 @@ class TestContainer(unittest.TestCase):
             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
             'output_path': '/var/spool/cwl',
             'output_ttl': 7200,
-            'container_image': 'arvados/jobs',
+            'container_image': '99999999999999999999999999999993+99',
             'command': ['ls'],
             'cwd': '/var/spool/cwl',
             'scheduling_parameters': {
@@ -240,7 +243,6 @@ class TestContainer(unittest.TestCase):
     def test_initial_work_dir(self, collection_mock, keepdocker):
         arv_docker_clear_cache()
         runner = mock.MagicMock()
-        runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
         runner.ignore_docker_for_reuse = False
         runner.intermediate_output_ttl = 0
         runner.secret_store = cwltool.secrets.SecretStore()
@@ -349,7 +351,7 @@ class TestContainer(unittest.TestCase):
             'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
             'output_path': '/var/spool/cwl',
             'output_ttl': 0,
-            'container_image': 'arvados/jobs',
+            'container_image': '99999999999999999999999999999993+99',
             'command': ['ls'],
             'cwd': '/var/spool/cwl',
             'scheduling_parameters': {
@@ -370,7 +372,6 @@ class TestContainer(unittest.TestCase):
         arv_docker_clear_cache()
 
         runner = mock.MagicMock()
-        runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
         runner.ignore_docker_for_reuse = False
         runner.intermediate_output_ttl = 0
         runner.secret_store = cwltool.secrets.SecretStore()
@@ -437,7 +438,7 @@ class TestContainer(unittest.TestCase):
                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
                     'output_path': '/var/spool/cwl',
                     'output_ttl': 0,
-                    'container_image': 'arvados/jobs',
+                    'container_image': '99999999999999999999999999999993+99',
                     'command': ['ls', '/var/spool/cwl'],
                     'cwd': '/var/spool/cwl',
                     'scheduling_parameters': {},
@@ -451,7 +452,6 @@ class TestContainer(unittest.TestCase):
 
         runner = mock.MagicMock()
         runner.api = api
-        runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
         runner.num_retries = 0
         runner.ignore_docker_for_reuse = False
         runner.intermediate_output_ttl = 0
@@ -463,7 +463,10 @@ class TestContainer(unittest.TestCase):
 
         col().open.return_value = []
 
+        loadingContext, runtimeContext = self.helper(runner)
+
         arvjob = arvados_cwl.ArvadosContainer(runner,
+                                              runtimeContext,
                                               mock.MagicMock(),
                                               {},
                                               None,
@@ -488,11 +491,84 @@ class TestContainer(unittest.TestCase):
         })
 
         self.assertFalse(api.collections().create.called)
+        self.assertFalse(runner.runtime_status_error.called)
 
         arvjob.collect_outputs.assert_called_with("keep:abc+123")
         arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
         runner.add_intermediate_output.assert_called_with("zzzzz-4zz18-zzzzzzzzzzzzzz2")
 
+    @mock.patch("arvados_cwl.util.get_current_container")
+    @mock.patch("arvados.collection.CollectionReader")
+    @mock.patch("arvados.collection.Collection")
+    def test_child_failure(self, col, reader, gcc_mock):
+        api = mock.MagicMock()
+        api._rootDesc = copy.deepcopy(get_rootDesc())
+        del api._rootDesc.get('resources')['jobs']['methods']['create']
+
+        # Set up runner with mocked runtime_status_update()
+        self.assertFalse(gcc_mock.called)
+        runtime_status_update = mock.MagicMock()
+        arvados_cwl.ArvCwlExecutor.runtime_status_update = runtime_status_update
+        runner = arvados_cwl.ArvCwlExecutor(api)
+        self.assertEqual(runner.work_api, 'containers')
+
+        # Make sure ArvCwlExecutor thinks it's running inside a container so it
+        # adds the logging handler that will call runtime_status_update() mock
+        gcc_mock.return_value = {"uuid" : "zzzzz-dz642-zzzzzzzzzzzzzzz"}
+        self.assertTrue(gcc_mock.called)
+        root_logger = logging.getLogger('')
+        handlerClasses = [h.__class__ for h in root_logger.handlers]
+        self.assertTrue(arvados_cwl.RuntimeStatusLoggingHandler in handlerClasses)
+
+        runner.num_retries = 0
+        runner.ignore_docker_for_reuse = False
+        runner.intermediate_output_ttl = 0
+        runner.secret_store = cwltool.secrets.SecretStore()
+        runner.label = mock.MagicMock()
+        runner.label.return_value = '[container testjob]'
+
+        runner.api.containers().get().execute.return_value = {
+            "state":"Complete",
+            "output": "abc+123",
+            "exit_code": 1,
+            "log": "def+234"
+        }
+
+        col().open.return_value = []
+
+        loadingContext, runtimeContext = self.helper(runner)
+
+        arvjob = arvados_cwl.ArvadosContainer(runner,
+                                              runtimeContext,
+                                              mock.MagicMock(),
+                                              {},
+                                              None,
+                                              [],
+                                              [],
+                                              "testjob")
+        arvjob.output_callback = mock.MagicMock()
+        arvjob.collect_outputs = mock.MagicMock()
+        arvjob.successCodes = [0]
+        arvjob.outdir = "/var/spool/cwl"
+        arvjob.output_ttl = 3600
+        arvjob.collect_outputs.return_value = {"out": "stuff"}
+
+        arvjob.done({
+            "state": "Final",
+            "log_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz1",
+            "output_uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2",
+            "uuid": "zzzzz-xvhdp-zzzzzzzzzzzzzzz",
+            "container_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
+            "modified_at": "2017-05-26T12:01:22Z"
+        })
+
+        runtime_status_update.assert_called_with(
+            'error',
+            'arvados.cwl-runner: [container testjob] (zzzzz-xvhdp-zzzzzzzzzzzzzzz) error log:',
+            '  ** log is empty **'
+        )
+        arvjob.output_callback.assert_called_with({"out": "stuff"}, "permanentFail")
+
     # The test passes no builder.resources
     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
@@ -500,7 +576,6 @@ class TestContainer(unittest.TestCase):
         arv_docker_clear_cache()
 
         runner = mock.MagicMock()
-        runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
         runner.ignore_docker_for_reuse = False
         runner.intermediate_output_ttl = 0
         runner.secret_store = cwltool.secrets.SecretStore()
@@ -575,7 +650,7 @@ class TestContainer(unittest.TestCase):
                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
                     'output_path': '/var/spool/cwl',
                     'output_ttl': 0,
-                    'container_image': 'arvados/jobs',
+                    'container_image': '99999999999999999999999999999994+99',
                     'command': ['ls', '/var/spool/cwl'],
                     'cwd': '/var/spool/cwl',
                     'scheduling_parameters': {},
@@ -590,7 +665,6 @@ class TestContainer(unittest.TestCase):
         arv_docker_clear_cache()
 
         runner = mock.MagicMock()
-        runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
         runner.ignore_docker_for_reuse = False
         runner.intermediate_output_ttl = 0
         runner.secret_store = cwltool.secrets.SecretStore()
@@ -668,7 +742,7 @@ class TestContainer(unittest.TestCase):
                     'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
                     'output_path': '/var/spool/cwl',
                     'output_ttl': 0,
-                    'container_image': 'arvados/jobs',
+                    'container_image': '99999999999999999999999999999993+99',
                     'command': ['md5sum', 'example.conf'],
                     'cwd': '/var/spool/cwl',
                     'scheduling_parameters': {},
@@ -688,7 +762,6 @@ class TestContainer(unittest.TestCase):
         arv_docker_clear_cache()
 
         runner = mock.MagicMock()
-        runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
         runner.ignore_docker_for_reuse = False
         runner.intermediate_output_ttl = 0
         runner.secret_store = cwltool.secrets.SecretStore()