projects
/
arvados.git
/ blobdiff
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
Merge branch 'main' into 19582-aws-s3v2-driver
[arvados.git]
/
sdk
/
cwl
/
arvados_cwl
/
http.py
diff --git
a/sdk/cwl/arvados_cwl/http.py
b/sdk/cwl/arvados_cwl/http.py
index ccc2e793b067f4e0853da275947a08d3630723a3..dcc2a51192dfc4d4b573da302b3373fd08d67fff 100644
(file)
--- a/
sdk/cwl/arvados_cwl/http.py
+++ b/
sdk/cwl/arvados_cwl/http.py
@@
-16,6
+16,7
@@
import arvados.collection
import urllib.parse
import logging
import calendar
import urllib.parse
import logging
import calendar
+import urllib.parse
logger = logging.getLogger('arvados.cwl-runner')
logger = logging.getLogger('arvados.cwl-runner')
@@
-132,7
+133,7
@@
def http_to_keep(api, project_uuid, url, utcnow=datetime.datetime.utcnow):
count = 0
start = time.time()
checkpoint = start
count = 0
start = time.time()
checkpoint = start
- with c.open(name, "w") as f:
+ with c.open(name, "w
b
") as f:
for chunk in req.iter_content(chunk_size=1024):
count += len(chunk)
f.write(chunk)
for chunk in req.iter_content(chunk_size=1024):
count += len(chunk)
f.write(chunk)
@@
-148,7
+149,9
@@
def http_to_keep(api, project_uuid, url, utcnow=datetime.datetime.utcnow):
logger.info("%d downloaded, %3.2f MiB/s", count, (bps / (1024*1024)))
checkpoint = loopnow
logger.info("%d downloaded, %3.2f MiB/s", count, (bps / (1024*1024)))
checkpoint = loopnow
- c.save_new(name="Downloaded from %s" % url, owner_uuid=project_uuid, ensure_unique_name=True)
+
+ collectionname = "Downloaded from %s" % urllib.parse.quote(url, safe='')
+ c.save_new(name=collectionname, owner_uuid=project_uuid, ensure_unique_name=True)
api.collections().update(uuid=c.manifest_locator(), body={"collection":{"properties": properties}}).execute()
api.collections().update(uuid=c.manifest_locator(), body={"collection":{"properties": properties}}).execute()