projects
/
arvados.git
/ blobdiff
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
18336: Pins upper version limit for httplib2 dependency on PySDK.
[arvados.git]
/
sdk
/
cwl
/
arvados_cwl
/
arvdocker.py
diff --git
a/sdk/cwl/arvados_cwl/arvdocker.py
b/sdk/cwl/arvados_cwl/arvdocker.py
index 7508febb08cc8bd704d251cc0490ea045a75053b..26408317cbe6d0cdb5382f38c1455b0bd7b5db2a 100644
(file)
--- a/
sdk/cwl/arvados_cwl/arvdocker.py
+++ b/
sdk/cwl/arvados_cwl/arvdocker.py
@@
-18,9
+18,13
@@
logger = logging.getLogger('arvados.cwl-runner')
cached_lookups = {}
cached_lookups_lock = threading.Lock()
cached_lookups = {}
cached_lookups_lock = threading.Lock()
-def arv_docker_get_image(api_client, dockerRequirement, pull_image, project_uuid):
+def arv_docker_get_image(api_client, dockerRequirement, pull_image, project_uuid,
+ force_pull, tmp_outdir_prefix):
"""Check if a Docker image is available in Keep, if not, upload it using arv-keepdocker."""
"""Check if a Docker image is available in Keep, if not, upload it using arv-keepdocker."""
+ if "http://arvados.org/cwl#dockerCollectionPDH" in dockerRequirement:
+ return dockerRequirement["http://arvados.org/cwl#dockerCollectionPDH"]
+
if "dockerImageId" not in dockerRequirement and "dockerPull" in dockerRequirement:
dockerRequirement = copy.deepcopy(dockerRequirement)
dockerRequirement["dockerImageId"] = dockerRequirement["dockerPull"]
if "dockerImageId" not in dockerRequirement and "dockerPull" in dockerRequirement:
dockerRequirement = copy.deepcopy(dockerRequirement)
dockerRequirement["dockerImageId"] = dockerRequirement["dockerPull"]
@@
-31,7
+35,7
@@
def arv_docker_get_image(api_client, dockerRequirement, pull_image, project_uuid
global cached_lookups_lock
with cached_lookups_lock:
if dockerRequirement["dockerImageId"] in cached_lookups:
global cached_lookups_lock
with cached_lookups_lock:
if dockerRequirement["dockerImageId"] in cached_lookups:
- return
dockerRequirement["dockerImageId"
]
+ return
cached_lookups[dockerRequirement["dockerImageId"]
]
with SourceLine(dockerRequirement, "dockerImageId", WorkflowException, logger.isEnabledFor(logging.DEBUG)):
sp = dockerRequirement["dockerImageId"].split(":")
with SourceLine(dockerRequirement, "dockerImageId", WorkflowException, logger.isEnabledFor(logging.DEBUG)):
sp = dockerRequirement["dockerImageId"].split(":")
@@
-45,7
+49,10
@@
def arv_docker_get_image(api_client, dockerRequirement, pull_image, project_uuid
if not images:
# Fetch Docker image if necessary.
try:
if not images:
# Fetch Docker image if necessary.
try:
- cwltool.docker.DockerCommandLineJob.get_image(dockerRequirement, pull_image)
+ result = cwltool.docker.DockerCommandLineJob.get_image(dockerRequirement, pull_image,
+ force_pull, tmp_outdir_prefix)
+ if not result:
+ raise WorkflowException("Docker image '%s' not available" % dockerRequirement["dockerImageId"])
except OSError as e:
raise WorkflowException("While trying to get Docker image '%s', failed to execute 'docker': %s" % (dockerRequirement["dockerImageId"], e))
except OSError as e:
raise WorkflowException("While trying to get Docker image '%s', failed to execute 'docker': %s" % (dockerRequirement["dockerImageId"], e))
@@
-60,6
+67,7
@@
def arv_docker_get_image(api_client, dockerRequirement, pull_image, project_uuid
arvados.commands.put.api_client = api_client
arvados.commands.keepdocker.main(args, stdout=sys.stderr, install_sig_handlers=False, api=api_client)
except SystemExit as e:
arvados.commands.put.api_client = api_client
arvados.commands.keepdocker.main(args, stdout=sys.stderr, install_sig_handlers=False, api=api_client)
except SystemExit as e:
+ # If e.code is None or zero, then keepdocker exited normally and we can continue
if e.code:
raise WorkflowException("keepdocker exited with code %s" % e.code)
if e.code:
raise WorkflowException("keepdocker exited with code %s" % e.code)
@@
-70,10
+78,12
@@
def arv_docker_get_image(api_client, dockerRequirement, pull_image, project_uuid
if not images:
raise WorkflowException("Could not find Docker image %s:%s" % (image_name, image_tag))
if not images:
raise WorkflowException("Could not find Docker image %s:%s" % (image_name, image_tag))
+ pdh = api_client.collections().get(uuid=images[0][0]).execute()["portable_data_hash"]
+
with cached_lookups_lock:
with cached_lookups_lock:
- cached_lookups[dockerRequirement["dockerImageId"]] =
True
+ cached_lookups[dockerRequirement["dockerImageId"]] =
pdh
- return
dockerRequirement["dockerImageId"]
+ return
pdh
def arv_docker_clear_cache():
global cached_lookups
def arv_docker_clear_cache():
global cached_lookups