From: Peter Amstutz Date: Wed, 26 Aug 2020 19:40:33 +0000 (-0400) Subject: 16602: Embed the desired workflow runner version in WorkflowRunnerResources X-Git-Tag: 2.1.0~96^2~5 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/5ce67dcb90e196227d920c52fc1a7256e39ede92 16602: Embed the desired workflow runner version in WorkflowRunnerResources Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml index dce1bd4d02..8a3fa3173a 100644 --- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml +++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml @@ -240,6 +240,12 @@ $graph: MiB. Default 256 MiB. Will be added on to the RAM request when determining node size to request. jsonldPredicate: "http://arvados.org/cwl#RuntimeConstraints/keep_cache" + acrContainerImage: + type: string? + doc: | + The container image containing the correct version of + arvados-cwl-runner to use when invoking the workflow on + Arvados. - name: ClusterTarget type: record diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml index b9b9e61651..95ed0a75bc 100644 --- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml +++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml @@ -184,6 +184,12 @@ $graph: MiB. Default 256 MiB. Will be added on to the RAM request when determining node size to request. jsonldPredicate: "http://arvados.org/cwl#RuntimeConstraints/keep_cache" + acrContainerImage: + type: string? + doc: | + The container image containing the correct version of + arvados-cwl-runner to use when invoking the workflow on + Arvados. - name: ClusterTarget type: record diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml index b9b9e61651..95ed0a75bc 100644 --- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml +++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml @@ -184,6 +184,12 @@ $graph: MiB. Default 256 MiB. Will be added on to the RAM request when determining node size to request. jsonldPredicate: "http://arvados.org/cwl#RuntimeConstraints/keep_cache" + acrContainerImage: + type: string? + doc: | + The container image containing the correct version of + arvados-cwl-runner to use when invoking the workflow on + Arvados. - name: ClusterTarget type: record diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py index 97c5fafe79..56c6f39e9d 100644 --- a/sdk/cwl/arvados_cwl/arvworkflow.py +++ b/sdk/cwl/arvados_cwl/arvworkflow.py @@ -24,9 +24,10 @@ import ruamel.yaml as yaml from .runner import (upload_dependencies, packed_workflow, upload_workflow_collection, trim_anonymous_location, remove_redundant_fields, discover_secondary_files, - make_builder) + make_builder, arvados_jobs_image) from .pathmapper import ArvPathMapper, trim_listing from .arvtool import ArvadosCommandTool, set_cluster_target +from ._version import __version__ from .perf import Perf @@ -37,7 +38,8 @@ max_res_pars = ("coresMin", "coresMax", "ramMin", "ramMax", "tmpdirMin", "tmpdir sum_res_pars = ("outdirMin", "outdirMax") def upload_workflow(arvRunner, tool, job_order, project_uuid, uuid=None, - submit_runner_ram=0, name=None, merged_map=None): + submit_runner_ram=0, name=None, merged_map=None, + submit_runner_image=None): packed = packed_workflow(arvRunner, tool, merged_map) @@ -57,18 +59,25 @@ def upload_workflow(arvRunner, tool, job_order, project_uuid, uuid=None, upload_dependencies(arvRunner, name, tool.doc_loader, packed, tool.tool["id"], False) + wf_runner_resources = None + + hints = main.get("hints", []) + found = False + for h in hints: + if h["class"] == "http://arvados.org/cwl#WorkflowRunnerResources": + wf_runner_resources = h + found = True + break + if not found: + wf_runner_resources = {"class": "http://arvados.org/cwl#WorkflowRunnerResources"} + hints.append(wf_runner_resources) + + wf_runner_resources["acrContainerImage"] = arvados_jobs_image(arvRunner, submit_runner_image or "arvados/jobs:"+__version__) + if submit_runner_ram: - hints = main.get("hints", []) - found = False - for h in hints: - if h["class"] == "http://arvados.org/cwl#WorkflowRunnerResources": - h["ramMin"] = submit_runner_ram - found = True - break - if not found: - hints.append({"class": "http://arvados.org/cwl#WorkflowRunnerResources", - "ramMin": submit_runner_ram}) - main["hints"] = hints + wf_runner_resources["ramMin"] = submit_runner_ram + + main["hints"] = hints body = { "workflow": { diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py index e8d1347ddf..9ba798ec64 100644 --- a/sdk/cwl/arvados_cwl/executor.py +++ b/sdk/cwl/arvados_cwl/executor.py @@ -596,7 +596,8 @@ The 'jobs' API is no longer supported. uuid=existing_uuid, submit_runner_ram=runtimeContext.submit_runner_ram, name=runtimeContext.name, - merged_map=merged_map), + merged_map=merged_map, + submit_runner_image=runtimeContext.submit_runner_image), "success") self.apply_reqs(job_order, tool) diff --git a/sdk/cwl/tests/collection_per_tool/collection_per_tool_packed.cwl b/sdk/cwl/tests/collection_per_tool/collection_per_tool_packed.cwl index 9bf1c20aab..1054d8f29b 100644 --- a/sdk/cwl/tests/collection_per_tool/collection_per_tool_packed.cwl +++ b/sdk/cwl/tests/collection_per_tool/collection_per_tool_packed.cwl @@ -6,6 +6,12 @@ "$graph": [ { "class": "Workflow", + "hints": [ + { + "acrContainerImage": "999999999999999999999999999999d3+99", + "class": "http://arvados.org/cwl#WorkflowRunnerResources" + } + ], "id": "#main", "inputs": [], "outputs": [], @@ -82,4 +88,4 @@ } ], "cwlVersion": "v1.0" -} \ No newline at end of file +} diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py index 0698db70ff..4da545bf36 100644 --- a/sdk/cwl/tests/test_submit.py +++ b/sdk/cwl/tests/test_submit.py @@ -1354,7 +1354,7 @@ class TestSubmit(unittest.TestCase): class TestCreateWorkflow(unittest.TestCase): existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml" expect_workflow = StripYAMLComments( - open("tests/wf/expect_packed.cwl").read()) + open("tests/wf/expect_upload_packed.cwl").read().rstrip()) @stubs def test_create(self, stubs): @@ -1472,7 +1472,7 @@ class TestCreateWorkflow(unittest.TestCase): stubs.capture_stdout, sys.stderr, api_client=stubs.api) toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl" - expect_workflow = StripYAMLComments(open(toolfile).read()) + expect_workflow = StripYAMLComments(open(toolfile).read().rstrip()) body = { "workflow": {