X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/34f3229a26dcc9d9b6c94e207eb8f58bb6555acb..2924f222d9efdb1b8776225d2d51bc8771d7b077:/sdk/cwl/arvados_cwl/__init__.py

diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index e3b8e500a1..7818ac84f4 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -221,6 +221,12 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
     parser.add_argument("--defer-downloads", action="store_true", default=False,
                         help="When submitting a workflow, defer downloading HTTP URLs to workflow launch instead of downloading to Keep before submit.")
 
+    parser.add_argument("--varying-url-params", type=str, default="",
+                        help="A comma separated list of URL query parameters that should be ignored when storing HTTP URLs in Keep.")
+
+    parser.add_argument("--prefer-cached-downloads", action="store_true", default=False,
+                        help="If a HTTP URL is found in Keep, skip upstream URL freshness check (will not notice if the upstream has changed, but also not error if upstream is unavailable).")
+
     exgroup = parser.add_mutually_exclusive_group()
     exgroup.add_argument("--enable-preemptible", dest="enable_preemptible", default=None, action="store_true", help="Use preemptible instances. Control individual steps with arv:UsePreemptible hint.")
     exgroup.add_argument("--disable-preemptible", dest="enable_preemptible", default=None, action="store_false", help="Don't use preemptible instances.")
@@ -274,6 +280,7 @@ def add_arv_hints():
         "http://commonwl.org/cwltool#CUDARequirement",
         "http://arvados.org/cwl#UsePreemptible",
         "http://arvados.org/cwl#OutputCollectionProperties",
+        "http://arvados.org/cwl#KeepCacheTypeRequirement",
     ])
 
 def exit_signal_handler(sigcode, frame):
@@ -322,12 +329,15 @@ def main(args=sys.argv[1:],
         if api_client is None:
             api_client = arvados.safeapi.ThreadSafeApiCache(
                 api_params={"model": OrderedJsonModel(), "timeout": arvargs.http_timeout},
-                keep_params={"num_retries": 4})
+                keep_params={"num_retries": 4},
+                version='v1',
+            )
             keep_client = api_client.keep
             # Make an API object now so errors are reported early.
             api_client.users().current().execute()
         if keep_client is None:
-            keep_client = arvados.keep.KeepClient(api_client=api_client, num_retries=4)
+            block_cache = arvados.keep.KeepBlockCache(disk_cache=True)
+            keep_client = arvados.keep.KeepClient(api_client=api_client, num_retries=4, block_cache=block_cache)
         executor = ArvCwlExecutor(api_client, arvargs, keep_client=keep_client, num_retries=4, stdout=stdout)
     except WorkflowException as e:
         logger.error(e, exc_info=(sys.exc_info()[1] if arvargs.debug else False))