1 # Copyright (C) The Arvados Authors. All rights reserved.
3 # SPDX-License-Identifier: Apache-2.0
10 from schema_salad.sourceline import SourceLine
13 from cwltool.errors import WorkflowException
14 import arvados.commands.keepdocker
16 logger = logging.getLogger('arvados.cwl-runner')
19 cached_lookups_lock = threading.Lock()
21 def arv_docker_get_image(api_client, dockerRequirement, pull_image, project_uuid,
22 force_pull, tmp_outdir_prefix):
23 """Check if a Docker image is available in Keep, if not, upload it using arv-keepdocker."""
25 if "http://arvados.org/cwl#dockerCollectionPDH" in dockerRequirement:
26 return dockerRequirement["http://arvados.org/cwl#dockerCollectionPDH"]
28 if "dockerImageId" not in dockerRequirement and "dockerPull" in dockerRequirement:
29 dockerRequirement = copy.deepcopy(dockerRequirement)
30 dockerRequirement["dockerImageId"] = dockerRequirement["dockerPull"]
31 if hasattr(dockerRequirement, 'lc'):
32 dockerRequirement.lc.data["dockerImageId"] = dockerRequirement.lc.data["dockerPull"]
35 global cached_lookups_lock
36 with cached_lookups_lock:
37 if dockerRequirement["dockerImageId"] in cached_lookups:
38 return cached_lookups[dockerRequirement["dockerImageId"]]
40 with SourceLine(dockerRequirement, "dockerImageId", WorkflowException, logger.isEnabledFor(logging.DEBUG)):
41 sp = dockerRequirement["dockerImageId"].split(":")
43 image_tag = sp[1] if len(sp) > 1 else "latest"
45 images = arvados.commands.keepdocker.list_images_in_arv(api_client, 3,
46 image_name=image_name,
50 # Fetch Docker image if necessary.
52 result = cwltool.docker.DockerCommandLineJob.get_image(dockerRequirement, pull_image,
53 force_pull, tmp_outdir_prefix)
55 raise WorkflowException("Docker image '%s' not available" % dockerRequirement["dockerImageId"])
57 raise WorkflowException("While trying to get Docker image '%s', failed to execute 'docker': %s" % (dockerRequirement["dockerImageId"], e))
59 # Upload image to Arvados
62 args.append("--project-uuid="+project_uuid)
63 args.append(image_name)
64 args.append(image_tag)
65 logger.info("Uploading Docker image %s:%s", image_name, image_tag)
67 arvados.commands.put.api_client = api_client
68 arvados.commands.keepdocker.main(args, stdout=sys.stderr, install_sig_handlers=False, api=api_client)
69 except SystemExit as e:
70 # If e.code is None or zero, then keepdocker exited normally and we can continue
72 raise WorkflowException("keepdocker exited with code %s" % e.code)
74 images = arvados.commands.keepdocker.list_images_in_arv(api_client, 3,
75 image_name=image_name,
79 raise WorkflowException("Could not find Docker image %s:%s" % (image_name, image_tag))
81 pdh = api_client.collections().get(uuid=images[0][0]).execute()["portable_data_hash"]
83 with cached_lookups_lock:
84 cached_lookups[dockerRequirement["dockerImageId"]] = pdh
88 def arv_docker_clear_cache():
90 global cached_lookups_lock
91 with cached_lookups_lock: