X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/e8d1a643cdbc3a5f4c0e5c745da58d9f7e1248d8..c22d90571a1fcb4b52e5387a791e3aefff5be6af:/sdk/cwl/arvados_cwl/arvworkflow.py diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py index 97c5fafe79..4fe82a6fe1 100644 --- a/sdk/cwl/arvados_cwl/arvworkflow.py +++ b/sdk/cwl/arvados_cwl/arvworkflow.py @@ -17,16 +17,17 @@ from cwltool.pack import pack from cwltool.load_tool import fetch_document, resolve_and_validate_document from cwltool.process import shortname from cwltool.workflow import Workflow, WorkflowException, WorkflowStep -from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class +from cwltool.utils import adjustFileObjs, adjustDirObjs, visit_class, normalizeFilesDirs from cwltool.context import LoadingContext import ruamel.yaml as yaml from .runner import (upload_dependencies, packed_workflow, upload_workflow_collection, trim_anonymous_location, remove_redundant_fields, discover_secondary_files, - make_builder) + make_builder, arvados_jobs_image) from .pathmapper import ArvPathMapper, trim_listing from .arvtool import ArvadosCommandTool, set_cluster_target +from ._version import __version__ from .perf import Perf @@ -37,7 +38,8 @@ max_res_pars = ("coresMin", "coresMax", "ramMin", "ramMax", "tmpdirMin", "tmpdir sum_res_pars = ("outdirMin", "outdirMax") def upload_workflow(arvRunner, tool, job_order, project_uuid, uuid=None, - submit_runner_ram=0, name=None, merged_map=None): + submit_runner_ram=0, name=None, merged_map=None, + submit_runner_image=None): packed = packed_workflow(arvRunner, tool, merged_map) @@ -57,18 +59,25 @@ def upload_workflow(arvRunner, tool, job_order, project_uuid, uuid=None, upload_dependencies(arvRunner, name, tool.doc_loader, packed, tool.tool["id"], False) + wf_runner_resources = None + + hints = main.get("hints", []) + found = False + for h in hints: + if h["class"] == "http://arvados.org/cwl#WorkflowRunnerResources": + wf_runner_resources = h + found = True + break + if not found: + wf_runner_resources = {"class": "http://arvados.org/cwl#WorkflowRunnerResources"} + hints.append(wf_runner_resources) + + wf_runner_resources["acrContainerImage"] = arvados_jobs_image(arvRunner, submit_runner_image or "arvados/jobs:"+__version__) + if submit_runner_ram: - hints = main.get("hints", []) - found = False - for h in hints: - if h["class"] == "http://arvados.org/cwl#WorkflowRunnerResources": - h["ramMin"] = submit_runner_ram - found = True - break - if not found: - hints.append({"class": "http://arvados.org/cwl#WorkflowRunnerResources", - "ramMin": submit_runner_ram}) - main["hints"] = hints + wf_runner_resources["ramMin"] = submit_runner_ram + + main["hints"] = hints body = { "workflow": { @@ -177,6 +186,7 @@ class ArvadosWorkflow(Workflow): discover_secondary_files(self.arvrunner.fs_access, builder, self.tool["inputs"], joborder) + normalizeFilesDirs(joborder) with Perf(metrics, "subworkflow upload_deps"): upload_dependencies(self.arvrunner,