1 # Copyright (C) The Arvados Authors. All rights reserved.
3 # SPDX-License-Identifier: Apache-2.0
12 from schema_salad.sourceline import SourceLine
15 from cwltool.errors import WorkflowException
16 import arvados.commands.keepdocker
18 logger = logging.getLogger('arvados.cwl-runner')
21 cached_lookups_lock = threading.Lock()
23 def determine_image_id(dockerImageId):
25 subprocess.check_output( # nosec
26 ["docker", "images", "--no-trunc", "--all"]
32 match = re.match(r"^([^ ]+)\s+([^ ]+)\s+([^ ]+)", line)
33 split = dockerImageId.split(":")
35 split.append("latest")
37 # if split[1] doesn't match valid tag names, it is a part of repository
38 if not re.match(r"[\w][\w.-]{0,127}", split[1]):
39 split[0] = split[0] + ":" + split[1]
42 if re.match(r"[\w][\w.-]{0,127}", split[2]):
43 split[0] = split[0] + ":" + split[1]
47 # check for repository:tag match or image id match
49 (split[0] == match.group(1) and split[1] == match.group(2))
50 or dockerImageId == match.group(3)
59 def arv_docker_get_image(api_client, dockerRequirement, pull_image, project_uuid,
60 force_pull, tmp_outdir_prefix, match_local_docker):
61 """Check if a Docker image is available in Keep, if not, upload it using arv-keepdocker."""
63 if "http://arvados.org/cwl#dockerCollectionPDH" in dockerRequirement:
64 return dockerRequirement["http://arvados.org/cwl#dockerCollectionPDH"]
66 if "dockerImageId" not in dockerRequirement and "dockerPull" in dockerRequirement:
67 dockerRequirement = copy.deepcopy(dockerRequirement)
68 dockerRequirement["dockerImageId"] = dockerRequirement["dockerPull"]
69 if hasattr(dockerRequirement, 'lc'):
70 dockerRequirement.lc.data["dockerImageId"] = dockerRequirement.lc.data["dockerPull"]
73 global cached_lookups_lock
74 with cached_lookups_lock:
75 if dockerRequirement["dockerImageId"] in cached_lookups:
76 return cached_lookups[dockerRequirement["dockerImageId"]]
78 with SourceLine(dockerRequirement, "dockerImageId", WorkflowException, logger.isEnabledFor(logging.DEBUG)):
79 sp = dockerRequirement["dockerImageId"].split(":")
81 image_tag = sp[1] if len(sp) > 1 else "latest"
83 out_of_project_images = arvados.commands.keepdocker.list_images_in_arv(api_client, 3,
84 image_name=image_name,
88 images = arvados.commands.keepdocker.list_images_in_arv(api_client, 3,
89 image_name=image_name,
91 project_uuid=project_uuid)
93 if match_local_docker:
94 local_image_id = determine_image_id(dockerRequirement["dockerImageId"])
99 if i[1]["dockerhash"] == local_image_id:
107 for i in out_of_project_images:
108 if i[1]["dockerhash"] == local_image_id:
110 out_of_project_images = [i]
114 out_of_project_images = []
117 if not out_of_project_images:
118 # Fetch Docker image if necessary.
120 result = cwltool.docker.DockerCommandLineJob.get_image(dockerRequirement, pull_image,
121 force_pull, tmp_outdir_prefix)
123 raise WorkflowException("Docker image '%s' not available" % dockerRequirement["dockerImageId"])
125 raise WorkflowException("While trying to get Docker image '%s', failed to execute 'docker': %s" % (dockerRequirement["dockerImageId"], e))
127 # Upload image to Arvados
130 args.append("--project-uuid="+project_uuid)
131 args.append(image_name)
132 args.append(image_tag)
133 logger.info("Uploading Docker image %s:%s", image_name, image_tag)
135 arvados.commands.put.api_client = api_client
136 arvados.commands.keepdocker.main(args, stdout=sys.stderr, install_sig_handlers=False, api=api_client)
137 except SystemExit as e:
138 # If e.code is None or zero, then keepdocker exited normally and we can continue
140 raise WorkflowException("keepdocker exited with code %s" % e.code)
142 images = arvados.commands.keepdocker.list_images_in_arv(api_client, 3,
143 image_name=image_name,
145 project_uuid=project_uuid)
148 raise WorkflowException("Could not find Docker image %s:%s" % (image_name, image_tag))
150 pdh = api_client.collections().get(uuid=images[0][0]).execute()["portable_data_hash"]
152 with cached_lookups_lock:
153 cached_lookups[dockerRequirement["dockerImageId"]] = pdh
157 def arv_docker_clear_cache():
158 global cached_lookups
159 global cached_lookups_lock
160 with cached_lookups_lock: