From 1cfcb49baf325386a409a1fe549dfb61e4982496 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Wed, 25 Nov 2020 16:45:53 -0500 Subject: [PATCH] 17072: Fix for added args on cwltool.docker.get_image Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- sdk/cwl/arvados_cwl/arvcontainer.py | 4 +++- sdk/cwl/arvados_cwl/arvdocker.py | 6 ++++-- sdk/cwl/arvados_cwl/runner.py | 18 ++++++++++++++---- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py index d3521099c4..7b81bfb447 100644 --- a/sdk/cwl/arvados_cwl/arvcontainer.py +++ b/sdk/cwl/arvados_cwl/arvcontainer.py @@ -234,7 +234,9 @@ class ArvadosContainer(JobBase): container_request["container_image"] = arv_docker_get_image(self.arvrunner.api, docker_req, runtimeContext.pull_image, - runtimeContext.project_uuid) + runtimeContext.project_uuid, + runtimeContext.force_docker_pull, + runtimeContext.tmp_outdir_prefix) network_req, _ = self.get_requirement("NetworkAccess") if network_req: diff --git a/sdk/cwl/arvados_cwl/arvdocker.py b/sdk/cwl/arvados_cwl/arvdocker.py index a8f56ad1d4..3c82082713 100644 --- a/sdk/cwl/arvados_cwl/arvdocker.py +++ b/sdk/cwl/arvados_cwl/arvdocker.py @@ -18,7 +18,8 @@ logger = logging.getLogger('arvados.cwl-runner') cached_lookups = {} cached_lookups_lock = threading.Lock() -def arv_docker_get_image(api_client, dockerRequirement, pull_image, project_uuid): +def arv_docker_get_image(api_client, dockerRequirement, pull_image, project_uuid, + force_pull, tmp_outdir_prefix): """Check if a Docker image is available in Keep, if not, upload it using arv-keepdocker.""" if "http://arvados.org/cwl#dockerCollectionPDH" in dockerRequirement: @@ -48,7 +49,8 @@ def arv_docker_get_image(api_client, dockerRequirement, pull_image, project_uuid if not images: # Fetch Docker image if necessary. try: - cwltool.docker.DockerCommandLineJob.get_image(dockerRequirement, pull_image) + cwltool.docker.DockerCommandLineJob.get_image(dockerRequirement, pull_image, + force_pull, tmp_outdir_prefix) except OSError as e: raise WorkflowException("While trying to get Docker image '%s', failed to execute 'docker': %s" % (dockerRequirement["dockerImageId"], e)) diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py index bad8f1e40c..42d4b552ac 100644 --- a/sdk/cwl/arvados_cwl/runner.py +++ b/sdk/cwl/arvados_cwl/runner.py @@ -443,9 +443,14 @@ def upload_docker(arvrunner, tool): # TODO: can be supported by containers API, but not jobs API. raise SourceLine(docker_req, "dockerOutputDirectory", UnsupportedRequirement).makeError( "Option 'dockerOutputDirectory' of DockerRequirement not supported.") - arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, docker_req, True, arvrunner.project_uuid) + arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, docker_req, True, arvrunner.project_uuid, + arvrunner.runtimeContext.force_docker_pull, + arvrunner.runtimeContext.tmp_outdir_prefix) else: - arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, {"dockerPull": "arvados/jobs:"+__version__}, True, arvrunner.project_uuid) + arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, {"dockerPull": "arvados/jobs:"+__version__}, + True, arvrunner.project_uuid, + arvrunner.runtimeContext.force_docker_pull, + arvrunner.runtimeContext.tmp_outdir_prefix) elif isinstance(tool, cwltool.workflow.Workflow): for s in tool.steps: upload_docker(arvrunner, s.embedded_tool) @@ -478,7 +483,10 @@ def packed_workflow(arvrunner, tool, merged_map): if "location" in v and v["location"] in merged_map[cur_id].secondaryFiles: v["secondaryFiles"] = merged_map[cur_id].secondaryFiles[v["location"]] if v.get("class") == "DockerRequirement": - v["http://arvados.org/cwl#dockerCollectionPDH"] = arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, v, True, arvrunner.project_uuid) + v["http://arvados.org/cwl#dockerCollectionPDH"] = arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, v, True, + arvrunner.project_uuid, + arvrunner.runtimeContext.force_docker_pull, + arvrunner.runtimeContext.tmp_outdir_prefix) for l in v: visit(v[l], cur_id) if isinstance(v, list): @@ -583,7 +591,9 @@ def arvados_jobs_image(arvrunner, img): """Determine if the right arvados/jobs image version is available. If not, try to pull and upload it.""" try: - return arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, {"dockerPull": img}, True, arvrunner.project_uuid) + return arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, {"dockerPull": img}, True, arvrunner.project_uuid, + arvrunner.runtimeContext.force_docker_pull, + arvrunner.runtimeContext.tmp_outdir_prefix) except Exception as e: raise Exception("Docker image %s is not available\n%s" % (img, e) ) -- 2.30.2