10812: Add --submit-runner-image to override default Docker image when submitting...

Conflicts:
	sdk/cwl/arvados_cwl/__init__.py
	sdk/cwl/arvados_cwl/runner.py

diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 3ffc4c7254a0b8a850716e82aebe6bbd1afb3c26..5d83300b418e56ea998ad0eba3a4169ca9fdefa7 100644 (file)
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -384,13 +384,21 @@ class ArvCwlRunner(object):
                                          self.output_callback,
                                          **kwargs).next()
                 else:
-                    runnerjob = RunnerContainer(self, tool, job_order, kwargs.get("enable_reuse"), self.output_name,
-                                                self.output_tags, submit_runner_ram=kwargs.get("submit_runner_ram"),
-                                                name=kwargs.get("name"), on_error=kwargs.get("on_error"))
-            else:
-                runnerjob = RunnerJob(self, tool, job_order, kwargs.get("enable_reuse"), self.output_name,
-                                      self.output_tags, submit_runner_ram=kwargs.get("submit_runner_ram"),
-                                      name=kwargs.get("name"), on_error=kwargs.get("on_error"))
+                    runnerjob = RunnerContainer(self, tool, job_order, kwargs.get("enable_reuse"),
+                                                self.output_name,
+                                                self.output_tags,
+                                                submit_runner_ram=kwargs.get("submit_runner_ram"),
+                                                name=kwargs["name"],
+                                                on_error=kwargs.get("on_error"),
+                                                submit_runner_image=kwargs.get("submit_runner_image"))
+            elif self.work_api == "jobs":
+                runnerjob = RunnerJob(self, tool, job_order, kwargs.get("enable_reuse"),
+                                      self.output_name,
+                                      self.output_tags,
+                                      submit_runner_ram=kwargs.get("submit_runner_ram"),
+                                      name=kwargs["name"],
+                                      on_error=kwargs.get("on_error"),
+                                      submit_runner_image=kwargs.get("submit_runner_image"))
 
         if not kwargs.get("submit") and "cwl_runner_job" not in kwargs and not self.work_api == "containers":
             # Create pipeline for local run
@@ -573,6 +581,10 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
                         help="RAM (in MiB) required for the workflow runner job (default 1024)",
                         default=1024)
 
+    parser.add_argument("--submit-runner-image", type=str,
+                        help="Docker image for workflow runner job, default arvados/jobs:%s" % __version__,
+                        default=None)
+
     parser.add_argument("--name", type=str,
                         help="Name to use for workflow execution instance.",
                         default=None)

diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 987b0d64bbc821bc199456b4ab317ab252841ee6..a3a26aae0a4bf6ab180429cad98193f47f3a0113 100644 (file)
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -84,7 +84,7 @@ class ArvadosContainer(object):
 
         (docker_req, docker_is_req) = get_feature(self, "DockerRequirement")
         if not docker_req:
-            docker_req = {"dockerImageId": arvados_jobs_image(self.arvrunner)}
+            docker_req = {"dockerImageId": "arvados/jobs"}
 
         container_request["container_image"] = arv_docker_get_image(self.arvrunner.api,
                                                                      docker_req,
@@ -199,7 +199,7 @@ class RunnerContainer(Runner):
             "cwd": "/var/spool/cwl",
             "priority": 1,
             "state": "Committed",
-            "container_image": arvados_jobs_image(self.arvrunner),
+            "container_image": arvados_jobs_image(self.arvrunner, self.jobs_image),
             "mounts": {
                 "/var/lib/cwl/cwl.input.json": {
                     "kind": "json",

diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index 11ef653a3d76c5862148493dc1736c0d72fadd0d..4bbbda26fb692f40551cce7f88a4396d7a5871ae 100644 (file)
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -92,7 +92,7 @@ class ArvadosJob(object):
                         "Option 'dockerOutputDirectory' of DockerRequirement not supported.")
                 runtime_constraints["docker_image"] = arv_docker_get_image(self.arvrunner.api, docker_req, pull_image, self.arvrunner.project_uuid)
             else:
-                runtime_constraints["docker_image"] = arvados_jobs_image(self.arvrunner)
+                runtime_constraints["docker_image"] = "arvados/jobs"
 
         resources = self.builder.resources
         if resources is not None:
@@ -261,7 +261,7 @@ class RunnerJob(Runner):
             "repository": "arvados",
             "script_parameters": self.job_order,
             "runtime_constraints": {
-                "docker_image": arvados_jobs_image(self.arvrunner),
+                "docker_image": arvados_jobs_image(self.arvrunner, self.jobs_image),
                 "min_ram_mb_per_node": self.submit_runner_ram
             }
         }

diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 1c3625e26bb1345673b31af5377d7b9d5282a10b..09ca63b1a1b6fce6b00b9d68b073cfc14cc83e33 100644 (file)
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -156,8 +156,9 @@ def upload_instance(arvrunner, name, tool, job_order):
 
         return workflowmapper
 
-def arvados_jobs_image(arvrunner):
-    img = "arvados/jobs:"+__version__
+def arvados_jobs_image(arvrunner, img):
+    """Determine if the right arvados/jobs image version is available.  If not, try to pull and upload it."""
+
     try:
         arv_docker_get_image(arvrunner.api, {"dockerPull": img}, True, arvrunner.project_uuid)
     except Exception as e:
@@ -167,7 +168,7 @@ def arvados_jobs_image(arvrunner):
 class Runner(object):
     def __init__(self, runner, tool, job_order, enable_reuse,
                  output_name, output_tags, submit_runner_ram=0,
-                 name=None, on_error=None):
+                 name=None, on_error=None, submit_runner_image=None):
         self.arvrunner = runner
         self.tool = tool
         self.job_order = job_order
@@ -179,6 +180,7 @@ class Runner(object):
         self.output_tags = output_tags
         self.name = name
         self.on_error = on_error
+        self.jobs_image = submit_runner_image or "arvados/jobs:"+__version__
 
         if submit_runner_ram:
             self.submit_runner_ram = submit_runner_ram

diff --git a/sdk/cwl/tests/test_job.py b/sdk/cwl/tests/test_job.py
index 7675e3d4bc45bd64b55a690c405597fa6b23a615..8aafb4a3c87c05cc0b7b08d4574852157e40701e 100644 (file)
--- a/sdk/cwl/tests/test_job.py
+++ b/sdk/cwl/tests/test_job.py
@@ -62,7 +62,7 @@ class TestJob(unittest.TestCase):
                         'repository': 'arvados',
                         'script': 'crunchrunner',
                         'runtime_constraints': {
-                            'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
+                            'docker_image': 'arvados/jobs',
                             'min_cores_per_node': 1,
                             'min_ram_mb_per_node': 1024,
                             'min_scratch_mb_per_node': 2048 # tmpdirSize + outdirSize
@@ -72,7 +72,7 @@ class TestJob(unittest.TestCase):
                     filters=[['repository', '=', 'arvados'],
                              ['script', '=', 'crunchrunner'],
                              ['script_version', 'in git', 'a3f2cb186e437bfce0031b024b2157b73ed2717d'],
-                             ['docker_image_locator', 'in docker', 'arvados/jobs:'+arvados_cwl.__version__]]
+                             ['docker_image_locator', 'in docker', 'arvados/jobs']]
                 )
 
     # The test passes some fields in builder.resources
@@ -130,7 +130,7 @@ class TestJob(unittest.TestCase):
                 'repository': 'arvados',
                 'script': 'crunchrunner',
                 'runtime_constraints': {
-                    'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
+                    'docker_image': 'arvados/jobs',
                     'min_cores_per_node': 3,
                     'min_ram_mb_per_node': 3000,
                     'min_scratch_mb_per_node': 5024, # tmpdirSize + outdirSize
@@ -141,7 +141,7 @@ class TestJob(unittest.TestCase):
             filters=[['repository', '=', 'arvados'],
                      ['script', '=', 'crunchrunner'],
                      ['script_version', 'in git', 'a3f2cb186e437bfce0031b024b2157b73ed2717d'],
-                     ['docker_image_locator', 'in docker', 'arvados/jobs:'+arvados_cwl.__version__]])
+                     ['docker_image_locator', 'in docker', 'arvados/jobs']])
 
     @mock.patch("arvados.collection.CollectionReader")
     def test_done(self, reader):
@@ -295,14 +295,14 @@ class TestWorkflow(unittest.TestCase):
                 'runtime_constraints': {
                     'min_scratch_mb_per_node': 2048,
                     'min_cores_per_node': 1,
-                    'docker_image': 'arvados/jobs:'+arvados_cwl.__version__,
+                    'docker_image': 'arvados/jobs',
                     'min_ram_mb_per_node': 1024
                 },
                 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'}),
             filters=[['repository', '=', 'arvados'],
                      ['script', '=', 'crunchrunner'],
                      ['script_version', 'in git', 'a3f2cb186e437bfce0031b024b2157b73ed2717d'],
-                     ['docker_image_locator', 'in docker', 'arvados/jobs:'+arvados_cwl.__version__]],
+                     ['docker_image_locator', 'in docker', 'arvados/jobs']],
             find_or_create=True)
 
         mockcollection().open().__enter__().write.assert_has_calls([mock.call(subwf)])

diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 0a01fb4cc37dd99809ae306d24fd087a3d09195d..13cc4831490e6165678b11ca519ed16772cb91a9 100644 (file)
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -793,7 +793,7 @@ class TestSubmit(unittest.TestCase):
                                                                           }], "items_available": 1, "offset": 0},
                                                                   {"items": [{"uuid": ""}], "items_available": 1, "offset": 0})
         arvrunner.api.collections().create().execute.return_value = {"uuid": ""}
-        self.assertEqual("arvados/jobs:"+arvados_cwl.__version__, arvados_cwl.runner.arvados_jobs_image(arvrunner))
+        self.assertEqual("arvados/jobs:"+arvados_cwl.__version__, arvados_cwl.runner.arvados_jobs_image(arvrunner, "arvados/jobs:"+arvados_cwl.__version__))
 
 class TestCreateTemplate(unittest.TestCase):
     existing_template_uuid = "zzzzz-d1hrv-validworkfloyml"