Merge branch '8018-container-retry' closes #8018

[arvados.git] / crunch_scripts / cwl-runner

diff --git a/crunch_scripts/cwl-runner b/crunch_scripts/cwl-runner
index 4b6ff44a2c6ff69d7d87d674642c3189dda530be..5d7f553fecad9d4383e7326cabfd1419612dd9e2 100755 (executable)
--- a/crunch_scripts/cwl-runner
+++ b/crunch_scripts/cwl-runner
@@ -12,14 +12,16 @@ import arvados
 import arvados_cwl
 import arvados.collection
 import arvados.util
-from cwltool.process import shortname
 import cwltool.main
 import logging
 import os
 import json
 import argparse
+import re
+import functools
+
 from arvados.api import OrderedJsonModel
-from cwltool.process import adjustFileObjs
+from cwltool.process import shortname, adjustFileObjs, adjustDirObjs, getListing, normalizeFilesDirs
 from cwltool.load_tool import load_tool
 
 # Print package versions
@@ -30,29 +32,40 @@ api = arvados.api("v1")
 try:
     job_order_object = arvados.current_job()['script_parameters']
 
+    pdh_path = re.compile(r'^[0-9a-f]{32}\+\d+(/.+)?$')
+
     def keeppath(v):
-        if arvados.util.keep_locator_pattern.match(v):
-            return "file://%s/%s" % (os.environ['TASK_KEEPMOUNT'], v)
+        if pdh_path.match(v):
+            return "keep:%s" % v
         else:
             return v
 
     def keeppathObj(v):
         v["location"] = keeppath(v["location"])
 
-    job_order_object["cwl:tool"] = keeppath(job_order_object["cwl:tool"])
+    job_order_object["cwl:tool"] = "file://%s/%s" % (os.environ['TASK_KEEPMOUNT'], job_order_object["cwl:tool"])
 
     for k,v in job_order_object.items():
         if isinstance(v, basestring) and arvados.util.keep_locator_pattern.match(v):
             job_order_object[k] = {
                 "class": "File",
-                "path": keeppath(v)
+                "location": "keep:%s" % v
             }
 
     adjustFileObjs(job_order_object, keeppathObj)
+    adjustDirObjs(job_order_object, keeppathObj)
+    normalizeFilesDirs(job_order_object)
+    adjustDirObjs(job_order_object, functools.partial(getListing, arvados_cwl.fsaccess.CollectionFsAccess("", api_client=api)))
+
+    output_name = None
+    if "arv:output_name" in job_order_object:
+        output_name = job_order_object["arv:output_name"]
+        del job_order_object["arv:output_name"]
 
-    runner = arvados_cwl.ArvCwlRunner(api_client=arvados.api('v1', model=OrderedJsonModel()))
+    runner = arvados_cwl.ArvCwlRunner(api_client=arvados.api('v1', model=OrderedJsonModel()),
+                                      output_name=output_name)
 
-    t = load_tool(job_order_object, runner.arvMakeTool)
+    t = load_tool(job_order_object, runner.arv_make_tool)
 
     args = argparse.Namespace()
     args.project_uuid = arvados.current_job()["owner_uuid"]
@@ -63,38 +76,16 @@ try:
     args.ignore_docker_for_reuse = False
     args.basedir = os.getcwd()
     args.cwl_runner_job={"uuid": arvados.current_job()["uuid"], "state": arvados.current_job()["state"]}
-    outputObj = runner.arvExecutor(t, job_order_object, **vars(args))
-
-    files = {}
-    def capture(fileobj):
-        path = fileobj["location"]
-        sp = path.split("/")
-        col = sp[0][5:]
-        if col not in files:
-            files[col] = set()
-        files[col].add("/".join(sp[1:]))
-        fileobj["location"] = path
-
-    adjustFileObjs(outputObj, capture)
-
-    final = arvados.collection.Collection()
-
-    for k,v in files.iteritems():
-        with arvados.collection.Collection(k) as c:
-            for f in c:
-                final.copy(f, f, c, True)
-
-    def makeRelative(fileobj):
-        fileobj["location"] = "/".join(fileobj["location"].split("/")[1:])
-
-    adjustFileObjs(outputObj, makeRelative)
+    outputObj = runner.arv_executor(t, job_order_object, **vars(args))
 
-    with final.open("cwl.output.json", "w") as f:
-        json.dump(outputObj, f, indent=4)
+    if runner.final_output_collection:
+        outputCollection = runner.final_output_collection.portable_data_hash()
+    else:
+        outputCollection = None
 
     api.job_tasks().update(uuid=arvados.current_task()['uuid'],
                                          body={
-                                             'output': final.save_new(create_collection_record=False),
+                                             'output': outputCollection,
                                              'success': True,
                                              'progress':1.0
                                          }).execute()