From 8a6ca096b41dcbbb9854abddae03997afd2ddf06 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Mon, 7 Nov 2022 16:46:37 -0500 Subject: [PATCH] 19699: Report download done, don't try to stage deferred downloads Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- sdk/cwl/arvados_cwl/http.py | 1 + sdk/cwl/arvados_cwl/pathmapper.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/sdk/cwl/arvados_cwl/http.py b/sdk/cwl/arvados_cwl/http.py index 1826e13c39..d85ae30d2d 100644 --- a/sdk/cwl/arvados_cwl/http.py +++ b/sdk/cwl/arvados_cwl/http.py @@ -166,6 +166,7 @@ def http_to_keep(api, project_uuid, url, utcnow=datetime.datetime.utcnow): logger.info("%d downloaded, %3.2f MiB/s", count, (bps / (1024*1024))) checkpoint = loopnow + logger.info("Download complete") collectionname = "Downloaded from %s" % urllib.parse.quote(url, safe='') diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py index 89364a905f..a7f2103476 100644 --- a/sdk/cwl/arvados_cwl/pathmapper.py +++ b/sdk/cwl/arvados_cwl/pathmapper.py @@ -160,6 +160,9 @@ class ArvPathMapper(PathMapper): if loc.startswith("_:"): return True + if self.arvrunner.defer_downloads and (loc.startswith("http:") or loc.startswith("https:")): + return False + i = loc.rfind("/") if i > -1: loc_prefix = loc[:i+1] -- 2.30.2