16602: Embed the desired workflow runner version in WorkflowRunnerResources

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz@curii.com>

diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml
index dce1bd4d0247d2f56af8902f844814633b739b25..8a3fa3173a9dd98baa5d8aa1ab74e19fe4bceb6d 100644 (file)
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml
@@ -240,6 +240,12 @@ $graph:
         MiB.  Default 256 MiB.  Will be added on to the RAM request
         when determining node size to request.
       jsonldPredicate: "http://arvados.org/cwl#RuntimeConstraints/keep_cache"
+    acrContainerImage:
+      type: string?
+      doc: |
+        The container image containing the correct version of
+        arvados-cwl-runner to use when invoking the workflow on
+        Arvados.
 
 - name: ClusterTarget
   type: record

diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml
index b9b9e616510817a3731ce308c74d2cf8771ccb6c..95ed0a75bc69bfe94929ce0e2ee80adf6dcec7e6 100644 (file)
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml
@@ -184,6 +184,12 @@ $graph:
         MiB.  Default 256 MiB.  Will be added on to the RAM request
         when determining node size to request.
       jsonldPredicate: "http://arvados.org/cwl#RuntimeConstraints/keep_cache"
+    acrContainerImage:
+      type: string?
+      doc: |
+        The container image containing the correct version of
+        arvados-cwl-runner to use when invoking the workflow on
+        Arvados.
 
 - name: ClusterTarget
   type: record

diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml
index b9b9e616510817a3731ce308c74d2cf8771ccb6c..95ed0a75bc69bfe94929ce0e2ee80adf6dcec7e6 100644 (file)
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml
@@ -184,6 +184,12 @@ $graph:
         MiB.  Default 256 MiB.  Will be added on to the RAM request
         when determining node size to request.
       jsonldPredicate: "http://arvados.org/cwl#RuntimeConstraints/keep_cache"
+    acrContainerImage:
+      type: string?
+      doc: |
+        The container image containing the correct version of
+        arvados-cwl-runner to use when invoking the workflow on
+        Arvados.
 
 - name: ClusterTarget
   type: record

diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 97c5fafe792fc06ce099e6a9bc6934671ace580d..56c6f39e9d76654816e60b30595841f80b106d34 100644 (file)
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -24,9 +24,10 @@ import ruamel.yaml as yaml
 
 from .runner import (upload_dependencies, packed_workflow, upload_workflow_collection,
                      trim_anonymous_location, remove_redundant_fields, discover_secondary_files,
-                     make_builder)
+                     make_builder, arvados_jobs_image)
 from .pathmapper import ArvPathMapper, trim_listing
 from .arvtool import ArvadosCommandTool, set_cluster_target
+from ._version import __version__
 
 from .perf import Perf
 
@@ -37,7 +38,8 @@ max_res_pars = ("coresMin", "coresMax", "ramMin", "ramMax", "tmpdirMin", "tmpdir
 sum_res_pars = ("outdirMin", "outdirMax")
 
 def upload_workflow(arvRunner, tool, job_order, project_uuid, uuid=None,
-                    submit_runner_ram=0, name=None, merged_map=None):
+                    submit_runner_ram=0, name=None, merged_map=None,
+                    submit_runner_image=None):
 
     packed = packed_workflow(arvRunner, tool, merged_map)
 
@@ -57,18 +59,25 @@ def upload_workflow(arvRunner, tool, job_order, project_uuid, uuid=None,
     upload_dependencies(arvRunner, name, tool.doc_loader,
                         packed, tool.tool["id"], False)
 
+    wf_runner_resources = None
+
+    hints = main.get("hints", [])
+    found = False
+    for h in hints:
+        if h["class"] == "http://arvados.org/cwl#WorkflowRunnerResources":
+            wf_runner_resources = h
+            found = True
+            break
+    if not found:
+        wf_runner_resources = {"class": "http://arvados.org/cwl#WorkflowRunnerResources"}
+        hints.append(wf_runner_resources)
+
+    wf_runner_resources["acrContainerImage"] = arvados_jobs_image(arvRunner, submit_runner_image or "arvados/jobs:"+__version__)
+
     if submit_runner_ram:
-        hints = main.get("hints", [])
-        found = False
-        for h in hints:
-            if h["class"] == "http://arvados.org/cwl#WorkflowRunnerResources":
-                h["ramMin"] = submit_runner_ram
-                found = True
-                break
-        if not found:
-            hints.append({"class": "http://arvados.org/cwl#WorkflowRunnerResources",
-                          "ramMin": submit_runner_ram})
-        main["hints"] = hints
+        wf_runner_resources["ramMin"] = submit_runner_ram
+
+    main["hints"] = hints
 
     body = {
         "workflow": {

diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index e8d1347ddfeec7545b8ab9740de38b78b55b4e75..9ba798ec640f3d0caaecdaf114bafbdb36a1b4ad 100644 (file)
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -596,7 +596,8 @@ The 'jobs' API is no longer supported.
                                         uuid=existing_uuid,
                                         submit_runner_ram=runtimeContext.submit_runner_ram,
                                         name=runtimeContext.name,
-                                        merged_map=merged_map),
+                                        merged_map=merged_map,
+                                        submit_runner_image=runtimeContext.submit_runner_image),
                         "success")
 
         self.apply_reqs(job_order, tool)

diff --git a/sdk/cwl/tests/collection_per_tool/collection_per_tool_packed.cwl b/sdk/cwl/tests/collection_per_tool/collection_per_tool_packed.cwl
index 9bf1c20aabc6591a4b1d00282e9c871456fca219..1054d8f29bdb627c6b8710429534dada68edddea 100644 (file)
--- a/sdk/cwl/tests/collection_per_tool/collection_per_tool_packed.cwl
+++ b/sdk/cwl/tests/collection_per_tool/collection_per_tool_packed.cwl
@@ -6,6 +6,12 @@
     "$graph": [
         {
             "class": "Workflow",
+            "hints": [
+                {
+                    "acrContainerImage": "999999999999999999999999999999d3+99",
+                    "class": "http://arvados.org/cwl#WorkflowRunnerResources"
+                }
+            ],
             "id": "#main",
             "inputs": [],
             "outputs": [],
@@ -82,4 +88,4 @@
         }
     ],
     "cwlVersion": "v1.0"
-}
\ No newline at end of file
+}

diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 0698db70ff68534ba70aa4176c5487f308cf2559..4da545bf36b91c6f96ba3ec8657e3d0a2a4db265 100644 (file)
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -1354,7 +1354,7 @@ class TestSubmit(unittest.TestCase):
 class TestCreateWorkflow(unittest.TestCase):
     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
     expect_workflow = StripYAMLComments(
-        open("tests/wf/expect_packed.cwl").read())
+        open("tests/wf/expect_upload_packed.cwl").read().rstrip())
 
     @stubs
     def test_create(self, stubs):
@@ -1472,7 +1472,7 @@ class TestCreateWorkflow(unittest.TestCase):
             stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 
         toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
-        expect_workflow = StripYAMLComments(open(toolfile).read())
+        expect_workflow = StripYAMLComments(open(toolfile).read().rstrip())
 
         body = {
             "workflow": {