Merge branch '16584-with-schemas' refs #16584

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz@curii.com>

diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 6f2255b3f8b18f104a00fc2d4982171c51dbb0da..341929454ad71a87912e566dfdd86153f7f0881e 100644 (file)
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -202,6 +202,14 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
     parser.add_argument("--http-timeout", type=int,
                         default=5*60, dest="http_timeout", help="API request timeout in seconds. Default is 300 seconds (5 minutes).")
 
+    parser.add_argument(
+        "--skip-schemas",
+        action="store_true",
+        help="Skip loading of schemas",
+        default=False,
+        dest="skip_schemas",
+    )
+
     exgroup = parser.add_mutually_exclusive_group()
     exgroup.add_argument("--trash-intermediate", action="store_true",
                         default=False, dest="trash_intermediate",

diff --git a/sdk/cwl/arvados_cwl/http.py b/sdk/cwl/arvados_cwl/http.py
index 47a304372c58a27ecde8d8c13bb55d6435f9cf79..dcc2a51192dfc4d4b573da302b3373fd08d67fff 100644 (file)
--- a/sdk/cwl/arvados_cwl/http.py
+++ b/sdk/cwl/arvados_cwl/http.py
@@ -16,6 +16,7 @@ import arvados.collection
 import urllib.parse
 import logging
 import calendar
+import urllib.parse
 
 logger = logging.getLogger('arvados.cwl-runner')
 
@@ -148,7 +149,9 @@ def http_to_keep(api, project_uuid, url, utcnow=datetime.datetime.utcnow):
                     logger.info("%d downloaded, %3.2f MiB/s", count, (bps / (1024*1024)))
                 checkpoint = loopnow
 
-    c.save_new(name="Downloaded from %s" % url, owner_uuid=project_uuid, ensure_unique_name=True)
+
+    collectionname = "Downloaded from %s" % urllib.parse.quote(url, safe='')
+    c.save_new(name=collectionname, owner_uuid=project_uuid, ensure_unique_name=True)
 
     api.collections().update(uuid=c.manifest_locator(), body={"collection":{"properties": properties}}).execute()
 

diff --git a/sdk/cwl/tests/test_http.py b/sdk/cwl/tests/test_http.py
index 4119fee383e27bcfe30a97d3de754d1879c067a9..650b5f0598514bbe9fd5ea0de96ab848d2375ad0 100644 (file)
--- a/sdk/cwl/tests/test_http.py
+++ b/sdk/cwl/tests/test_http.py
@@ -61,7 +61,7 @@ class TestHttpToKeep(unittest.TestCase):
         getmock.assert_called_with("http://example.com/file1.txt", stream=True, allow_redirects=True)
 
         cm.open.assert_called_with("file1.txt", "wb")
-        cm.save_new.assert_called_with(name="Downloaded from http://example.com/file1.txt",
+        cm.save_new.assert_called_with(name="Downloaded from http%3A%2F%2Fexample.com%2Ffile1.txt",
                                        owner_uuid=None, ensure_unique_name=True)
 
         api.collections().update.assert_has_calls([
@@ -189,7 +189,7 @@ class TestHttpToKeep(unittest.TestCase):
         getmock.assert_called_with("http://example.com/file1.txt", stream=True, allow_redirects=True)
 
         cm.open.assert_called_with("file1.txt", "wb")
-        cm.save_new.assert_called_with(name="Downloaded from http://example.com/file1.txt",
+        cm.save_new.assert_called_with(name="Downloaded from http%3A%2F%2Fexample.com%2Ffile1.txt",
                                        owner_uuid=None, ensure_unique_name=True)
 
         api.collections().update.assert_has_calls([
@@ -280,7 +280,7 @@ class TestHttpToKeep(unittest.TestCase):
         getmock.assert_called_with("http://example.com/download?fn=/file1.txt", stream=True, allow_redirects=True)
 
         cm.open.assert_called_with("file1.txt", "wb")
-        cm.save_new.assert_called_with(name="Downloaded from http://example.com/download?fn=/file1.txt",
+        cm.save_new.assert_called_with(name="Downloaded from http%3A%2F%2Fexample.com%2Fdownload%3Ffn%3D%2Ffile1.txt",
                                        owner_uuid=None, ensure_unique_name=True)
 
         api.collections().update.assert_has_calls([