9570: Update cwl-runner for latest cwltool. Ensure use_container,

tmpdir_prefix and on_error are set.

diff --git a/crunch_scripts/cwl-runner b/crunch_scripts/cwl-runner
index c786fc10b1715e1c5912e13ee98011aa94dceab3..4b6ff44a2c6ff69d7d87d674642c3189dda530be 100755 (executable)
--- a/crunch_scripts/cwl-runner
+++ b/crunch_scripts/cwl-runner
@@ -19,7 +19,7 @@ import os
 import json
 import argparse
 from arvados.api import OrderedJsonModel
-from cwltool.process import adjustFiles
+from cwltool.process import adjustFileObjs
 from cwltool.load_tool import load_tool
 
 # Print package versions
@@ -36,6 +36,9 @@ try:
         else:
             return v
 
+    def keeppathObj(v):
+        v["location"] = keeppath(v["location"])
+
     job_order_object["cwl:tool"] = keeppath(job_order_object["cwl:tool"])
 
     for k,v in job_order_object.items():
@@ -45,7 +48,7 @@ try:
                 "path": keeppath(v)
             }
 
-    adjustFiles(job_order_object, keeppath)
+    adjustFileObjs(job_order_object, keeppathObj)
 
     runner = arvados_cwl.ArvCwlRunner(api_client=arvados.api('v1', model=OrderedJsonModel()))
 
@@ -63,15 +66,16 @@ try:
     outputObj = runner.arvExecutor(t, job_order_object, **vars(args))
 
     files = {}
-    def capture(path):
+    def capture(fileobj):
+        path = fileobj["location"]
         sp = path.split("/")
         col = sp[0][5:]
         if col not in files:
             files[col] = set()
         files[col].add("/".join(sp[1:]))
-        return path
+        fileobj["location"] = path
 
-    adjustFiles(outputObj, capture)
+    adjustFileObjs(outputObj, capture)
 
     final = arvados.collection.Collection()
 
@@ -80,10 +84,10 @@ try:
             for f in c:
                 final.copy(f, f, c, True)
 
-    def makeRelative(path):
-        return "/".join(path.split("/")[1:])
+    def makeRelative(fileobj):
+        fileobj["location"] = "/".join(fileobj["location"].split("/")[1:])
 
-    adjustFiles(outputObj, makeRelative)
+    adjustFileObjs(outputObj, makeRelative)
 
     with final.open("cwl.output.json", "w") as f:
         json.dump(outputObj, f, indent=4)

diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 847da90a7549c855bdb3b43dd336d78b283d3f20..61736ccb808327e50943028ea1248dca340b624b 100644 (file)
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -123,6 +123,9 @@ class ArvCwlRunner(object):
 
         kwargs["fs_access"] = self.fs_access
         kwargs["enable_reuse"] = kwargs.get("enable_reuse")
+        kwargs["use_container"] = True
+        kwargs["tmpdir_prefix"] = "tmp"
+        kwargs["on_error"] = "continue"
 
         if self.work_api == "containers":
             kwargs["outdir"] = "/var/spool/cwl"

diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 19cb7eae37dec25daefd0759670cf9151e1a2add..1b5d8a4ea90e629155c837c6223e5f65ae68d55d 100644 (file)
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -115,12 +115,11 @@ class Runner(object):
                 outc = arvados.collection.Collection(record["output"])
                 with outc.open("cwl.output.json") as f:
                     outputs = json.load(f)
-                def keepify(path):
+                def keepify(fileobj):
+                    path = fileobj["location"]
                     if not path.startswith("keep:"):
-                        return "keep:%s/%s" % (record["output"], path)
-                    else:
-                        return path
-                adjustFiles(outputs, keepify)
+                        fileobj["location"] = "keep:%s/%s" % (record["output"], path)
+                adjustFileObjs(outputs, keepify)
             except Exception as e:
                 logger.error("While getting final output object: %s", e)
             self.arvrunner.output_callback(outputs, processStatus)