Merge branch '9187-crunchv2-dispatching' closes #9187

[arvados.git] / crunch_scripts / cwl-runner

diff --git a/crunch_scripts/cwl-runner b/crunch_scripts/cwl-runner
index fbcb87326f7672827a9a19e30477ff1998062dbb..c786fc10b1715e1c5912e13ee98011aa94dceab3 100755 (executable)
--- a/crunch_scripts/cwl-runner
+++ b/crunch_scripts/cwl-runner
@@ -1,5 +1,13 @@
 #!/usr/bin/env python
 
+# Crunch script integration for running arvados-cwl-runner (importing
+# arvados_cwl module) inside a crunch job.
+#
+# This gets the job record, transforms the script parameters into a valid CWL
+# input object, then executes the CWL runner to run the underlying workflow or
+# tool.  When the workflow completes, record the output object in an output
+# collection for this runner job.
+
 import arvados
 import arvados_cwl
 import arvados.collection
@@ -12,31 +20,47 @@ import json
 import argparse
 from arvados.api import OrderedJsonModel
 from cwltool.process import adjustFiles
+from cwltool.load_tool import load_tool
+
+# Print package versions
+logging.info(cwltool.main.versionstring())
 
 api = arvados.api("v1")
 
 try:
     job_order_object = arvados.current_job()['script_parameters']
 
-    print job_order_object
-
     def keeppath(v):
         if arvados.util.keep_locator_pattern.match(v):
             return "file://%s/%s" % (os.environ['TASK_KEEPMOUNT'], v)
+        else:
+            return v
 
     job_order_object["cwl:tool"] = keeppath(job_order_object["cwl:tool"])
 
+    for k,v in job_order_object.items():
+        if isinstance(v, basestring) and arvados.util.keep_locator_pattern.match(v):
+            job_order_object[k] = {
+                "class": "File",
+                "path": keeppath(v)
+            }
+
     adjustFiles(job_order_object, keeppath)
 
     runner = arvados_cwl.ArvCwlRunner(api_client=arvados.api('v1', model=OrderedJsonModel()))
 
-    t = cwltool.main.load_tool(job_order_object, False, True, runner.arvMakeTool, True)
-
-    np = argparse.Namespace()
-    np.project_uuid = arvados.current_job()["owner_uuid"]
-    np.enable_reuse = True
-    np.submit = False
-    outputObj = runner.arvExecutor(t, job_order_object, "", np, cwl_runner_job=arvados.current_job())
+    t = load_tool(job_order_object, runner.arvMakeTool)
+
+    args = argparse.Namespace()
+    args.project_uuid = arvados.current_job()["owner_uuid"]
+    args.enable_reuse = True
+    args.submit = False
+    args.debug = True
+    args.quiet = False
+    args.ignore_docker_for_reuse = False
+    args.basedir = os.getcwd()
+    args.cwl_runner_job={"uuid": arvados.current_job()["uuid"], "state": arvados.current_job()["state"]}
+    outputObj = runner.arvExecutor(t, job_order_object, **vars(args))
 
     files = {}
     def capture(path):