9751: Provide make_fs_access=CollectionFsAccess and bump cwltool pin.

[arvados.git] / crunch_scripts / cwl-runner

diff --git a/crunch_scripts/cwl-runner b/crunch_scripts/cwl-runner
index 71ba68d4232699efc540af50b23758a2bf6e6bef..2a1873a84e6925f9c1edf6057dc2b01a392d25ff 100755 (executable)
--- a/crunch_scripts/cwl-runner
+++ b/crunch_scripts/cwl-runner
@@ -19,7 +19,7 @@ import os
 import json
 import argparse
 from arvados.api import OrderedJsonModel
-from cwltool.process import adjustFiles
+from cwltool.process import adjustFileObjs
 from cwltool.load_tool import load_tool
 
 # Print package versions
@@ -32,22 +32,25 @@ try:
 
     def keeppath(v):
         if arvados.util.keep_locator_pattern.match(v):
-            return "file://%s/%s" % (os.environ['TASK_KEEPMOUNT'], v)
+            return "keep:%s" % v
         else:
             return v
 
-    job_order_object["cwl:tool"] = keeppath(job_order_object["cwl:tool"])
+    def keeppathObj(v):
+        v["location"] = keeppath(v["location"])
+
+    job_order_object["cwl:tool"] = "file://%s/%s" % (os.environ['TASK_KEEPMOUNT'], job_order_object["cwl:tool"])
 
     for k,v in job_order_object.items():
         if isinstance(v, basestring) and arvados.util.keep_locator_pattern.match(v):
             job_order_object[k] = {
                 "class": "File",
-                "path": keeppath(v)
+                "location": "keep:%s" % v
             }
 
-    adjustFiles(job_order_object, keeppath)
+    adjustFileObjs(job_order_object, keeppathObj)
 
-    runner = arvados_cwl.ArvCwlRunner(api_client=arvados.api('v1', model=OrderedJsonModel()), work_api="jobs")
+    runner = arvados_cwl.ArvCwlRunner(api_client=arvados.api('v1', model=OrderedJsonModel()))
 
     t = load_tool(job_order_object, runner.arvMakeTool)
 
@@ -63,15 +66,16 @@ try:
     outputObj = runner.arvExecutor(t, job_order_object, **vars(args))
 
     files = {}
-    def capture(path):
+    def capture(fileobj):
+        path = fileobj["location"]
         sp = path.split("/")
         col = sp[0][5:]
         if col not in files:
             files[col] = set()
         files[col].add("/".join(sp[1:]))
-        return path
+        fileobj["location"] = path
 
-    adjustFiles(outputObj, capture)
+    adjustFileObjs(outputObj, capture)
 
     final = arvados.collection.Collection()
 
@@ -80,10 +84,10 @@ try:
             for f in c:
                 final.copy(f, f, c, True)
 
-    def makeRelative(path):
-        return "/".join(path.split("/")[1:])
+    def makeRelative(fileobj):
+        fileobj["location"] = "/".join(fileobj["location"].split("/")[1:])
 
-    adjustFiles(outputObj, makeRelative)
+    adjustFileObjs(outputObj, makeRelative)
 
     with final.open("cwl.output.json", "w") as f:
         json.dump(outputObj, f, indent=4)