X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/cdcddca613c896fd8395a4045c858945451c3fa0..50d2969bdaf2fbbfbe00d35ec8339b166eee8001:/crunch_scripts/cwl-runner

diff --git a/crunch_scripts/cwl-runner b/crunch_scripts/cwl-runner
index d628f1c12d..89699f5eaf 100755
--- a/crunch_scripts/cwl-runner
+++ b/crunch_scripts/cwl-runner
@@ -12,14 +12,17 @@ import arvados
 import arvados_cwl
 import arvados.collection
 import arvados.util
-from cwltool.process import shortname
 import cwltool.main
 import logging
 import os
 import json
 import argparse
+import re
+import functools
+
 from arvados.api import OrderedJsonModel
-from cwltool.process import adjustFiles
+from cwltool.process import shortname, adjustFileObjs, adjustDirObjs, getListing, normalizeFilesDirs
+from cwltool.load_tool import load_tool
 
 # Print package versions
 logging.info(cwltool.main.versionstring())
@@ -29,17 +32,34 @@ api = arvados.api("v1")
 try:
     job_order_object = arvados.current_job()['script_parameters']
 
+    pdh_path = re.compile(r'^[0-9a-f]{32}\+\d+(/.+)?$')
+
     def keeppath(v):
-        if arvados.util.keep_locator_pattern.match(v):
-            return "file://%s/%s" % (os.environ['TASK_KEEPMOUNT'], v)
+        if pdh_path.match(v):
+            return "keep:%s" % v
+        else:
+            return v
+
+    def keeppathObj(v):
+        v["location"] = keeppath(v["location"])
+
+    job_order_object["cwl:tool"] = "file://%s/%s" % (os.environ['TASK_KEEPMOUNT'], job_order_object["cwl:tool"])
 
-    job_order_object["cwl:tool"] = keeppath(job_order_object["cwl:tool"])
+    for k,v in job_order_object.items():
+        if isinstance(v, basestring) and arvados.util.keep_locator_pattern.match(v):
+            job_order_object[k] = {
+                "class": "File",
+                "location": "keep:%s" % v
+            }
 
-    adjustFiles(job_order_object, keeppath)
+    adjustFileObjs(job_order_object, keeppathObj)
+    adjustDirObjs(job_order_object, keeppathObj)
+    normalizeFilesDirs(job_order_object)
+    adjustDirObjs(job_order_object, functools.partial(getListing, arvados_cwl.fsaccess.CollectionFsAccess("", api_client=api)))
 
     runner = arvados_cwl.ArvCwlRunner(api_client=arvados.api('v1', model=OrderedJsonModel()))
 
-    t = cwltool.main.load_tool(job_order_object, False, True, runner.arvMakeTool, True)
+    t = load_tool(job_order_object, runner.arv_make_tool)
 
     args = argparse.Namespace()
     args.project_uuid = arvados.current_job()["owner_uuid"]
@@ -47,18 +67,22 @@ try:
     args.submit = False
     args.debug = True
     args.quiet = False
-    outputObj = runner.arvExecutor(t, job_order_object, "", args, cwl_runner_job={"uuid": arvados.current_job()["uuid"], "state": arvados.current_job()["state"]})
+    args.ignore_docker_for_reuse = False
+    args.basedir = os.getcwd()
+    args.cwl_runner_job={"uuid": arvados.current_job()["uuid"], "state": arvados.current_job()["state"]}
+    outputObj = runner.arv_executor(t, job_order_object, **vars(args))
 
     files = {}
-    def capture(path):
+    def capture(fileobj):
+        path = fileobj["location"]
         sp = path.split("/")
         col = sp[0][5:]
         if col not in files:
             files[col] = set()
         files[col].add("/".join(sp[1:]))
-        return path
+        fileobj["location"] = path
 
-    adjustFiles(outputObj, capture)
+    adjustFileObjs(outputObj, capture)
 
     final = arvados.collection.Collection()
 
@@ -67,10 +91,10 @@ try:
             for f in c:
                 final.copy(f, f, c, True)
 
-    def makeRelative(path):
-        return "/".join(path.split("/")[1:])
+    def makeRelative(fileobj):
+        fileobj["location"] = "/".join(fileobj["location"].split("/")[1:])
 
-    adjustFiles(outputObj, makeRelative)
+    adjustFileObjs(outputObj, makeRelative)
 
     with final.open("cwl.output.json", "w") as f:
         json.dump(outputObj, f, indent=4)