19699: Report download done, don't try to stage deferred downloads

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz@curii.com>

diff --git a/sdk/cwl/arvados_cwl/http.py b/sdk/cwl/arvados_cwl/http.py
index 1826e13c3917a606bf3c0bba20c2ed5d5e898ad2..d85ae30d2d0be9fd66058161ce8264d0273eadba 100644 (file)
--- a/sdk/cwl/arvados_cwl/http.py
+++ b/sdk/cwl/arvados_cwl/http.py
@@ -166,6 +166,7 @@ def http_to_keep(api, project_uuid, url, utcnow=datetime.datetime.utcnow):
                     logger.info("%d downloaded, %3.2f MiB/s", count, (bps / (1024*1024)))
                 checkpoint = loopnow
 
+    logger.info("Download complete")
 
     collectionname = "Downloaded from %s" % urllib.parse.quote(url, safe='')
 

diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py
index 89364a905fb75d9ce5d258e98243c3a9b5ea05b1..a7f2103476718ec1dd3f07a36e370e46521783e2 100644 (file)
--- a/sdk/cwl/arvados_cwl/pathmapper.py
+++ b/sdk/cwl/arvados_cwl/pathmapper.py
@@ -160,6 +160,9 @@ class ArvPathMapper(PathMapper):
         if loc.startswith("_:"):
             return True
 
+        if self.arvrunner.defer_downloads and (loc.startswith("http:") or loc.startswith("https:")):
+            return False
+
         i = loc.rfind("/")
         if i > -1:
             loc_prefix = loc[:i+1]