X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/209c2259a8bf0218e585da037a04e2d9d79400ee..917c3ca20c36318578c4dfab7de076d97a2f87fe:/sdk/cwl/arvados_cwl/crunch_script.py

diff --git a/sdk/cwl/arvados_cwl/crunch_script.py b/sdk/cwl/arvados_cwl/crunch_script.py
index 821abf9266..c886550d41 100644
--- a/sdk/cwl/arvados_cwl/crunch_script.py
+++ b/sdk/cwl/arvados_cwl/crunch_script.py
@@ -1,3 +1,7 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
 # Crunch script integration for running arvados-cwl-runner (importing
 # arvados_cwl module) inside a crunch job.
 #
@@ -6,6 +10,9 @@
 # tool.  When the workflow completes, record the output object in an output
 # collection for this runner job.
 
+from past.builtins import basestring
+from future.utils import viewitems
+
 import arvados
 import arvados_cwl
 import arvados.collection
@@ -23,6 +30,7 @@ from cwltool.process import shortname
 from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, normalizeFilesDirs
 from cwltool.load_tool import load_tool
 from cwltool.errors import WorkflowException
+from arvados_cwl.context import ArvRuntimeContext
 
 from .fsaccess import CollectionFetcher, CollectionFsAccess
 
@@ -56,7 +64,7 @@ def run():
             if "location" in v:
                 v["location"] = keeppath(v["location"])
 
-        for k,v in job_order_object.items():
+        for k,v in viewitems(job_order_object):
             if isinstance(v, basestring) and arvados.util.keep_locator_pattern.match(v):
                 job_order_object[k] = {
                     "class": "File",
@@ -71,6 +79,8 @@ def run():
         output_tags = None
         enable_reuse = True
         on_error = "continue"
+        debug = False
+
         if "arv:output_name" in job_order_object:
             output_name = job_order_object["arv:output_name"]
             del job_order_object["arv:output_name"]
@@ -87,32 +97,51 @@ def run():
             on_error = job_order_object["arv:on_error"]
             del job_order_object["arv:on_error"]
 
-        runner = arvados_cwl.ArvCwlRunner(api_client=arvados.api('v1', model=OrderedJsonModel()),
-                                          output_name=output_name, output_tags=output_tags)
+        if "arv:debug" in job_order_object:
+            debug = job_order_object["arv:debug"]
+            del job_order_object["arv:debug"]
+
+        arvargs = argparse.Namespace()
+        arvargs.work_api = "jobs"
+        arvargs.output_name = output_name
+        arvargs.output_tags = output_tags
+        arvargs.thread_count = 1
+        arvargs.collection_cache_size = None
+
+        runner = arvados_cwl.ArvCwlExecutor(api_client=arvados.safeapi.ThreadSafeApiCache(
+            api_params={"model": OrderedJsonModel()}, keep_params={"num_retries": 4}),
+                                          arvargs=arvargs)
 
         make_fs_access = functools.partial(CollectionFsAccess,
                                  collection_cache=runner.collection_cache)
 
-        t = load_tool(toolpath, runner.arv_make_tool,
-                      fetcher_constructor=functools.partial(CollectionFetcher,
-                                                  api_client=runner.api,
-                                                  fs_access=make_fs_access(""),
-                                                  num_retries=runner.num_retries))
+        t = load_tool(toolpath, runner.loadingContext)
 
-        args = argparse.Namespace()
+        if debug:
+            logger.setLevel(logging.DEBUG)
+            logging.getLogger('arvados').setLevel(logging.DEBUG)
+            logging.getLogger("cwltool").setLevel(logging.DEBUG)
+
+        args = ArvRuntimeContext(vars(arvargs))
         args.project_uuid = arvados.current_job()["owner_uuid"]
         args.enable_reuse = enable_reuse
         args.on_error = on_error
         args.submit = False
-        args.debug = False
+        args.debug = debug
         args.quiet = False
         args.ignore_docker_for_reuse = False
         args.basedir = os.getcwd()
         args.name = None
         args.cwl_runner_job={"uuid": arvados.current_job()["uuid"], "state": arvados.current_job()["state"]}
         args.make_fs_access = make_fs_access
-
-        runner.arv_executor(t, job_order_object, **vars(args))
+        args.trash_intermediate = False
+        args.intermediate_output_ttl = 0
+        args.priority = arvados_cwl.DEFAULT_PRIORITY
+        args.do_validate = True
+        args.disable_js_validation = False
+        args.tmp_outdir_prefix = "tmp"
+
+        runner.arv_executor(t, job_order_object, args, logger=logger)
     except Exception as e:
         if isinstance(e, WorkflowException):
             logging.info("Workflow error %s", e)