8654: Passes 100% CWL conformance tests using running cwl-runner in a crunch job!

diff --git a/crunch_scripts/cwl-runner b/crunch_scripts/cwl-runner
index eeb9d78026ae845729a471044f218bafaf849ce9..d4d61d43acb101e10c20cc3f9e9a01268ec8107a 100755 (executable)
--- a/crunch_scripts/cwl-runner
+++ b/crunch_scripts/cwl-runner
@@ -18,8 +18,6 @@ api = arvados.api("v1")
 try:
     job_order_object = arvados.current_job()['script_parameters']
 
-    print job_order_object
-
     def keeppath(v):
         if arvados.util.keep_locator_pattern.match(v):
             return "file://%s/%s" % (os.environ['TASK_KEEPMOUNT'], v)

diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 61c065f5fb05e970ad70d718cc626c6aff80aeba..eab5d4aedd6bad7f66d213ed8062d336409def52 100644 (file)
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -372,9 +372,12 @@ class RunnerJob(object):
 
         outputs = None
         try:
-            outc = arvados.collection.Collection(record["output"])
-            with outc.open("cwl.output.json") as f:
-                outputs = json.load(f)
+            try:
+                outc = arvados.collection.Collection(record["output"])
+                with outc.open("cwl.output.json") as f:
+                    outputs = json.load(f)
+            except Exception as e:
+                logger.error("While getting final output object: %s", e)
             self.arvrunner.output_callback(outputs, processStatus)
         finally:
             del self.arvrunner.jobs[record["uuid"]]
@@ -383,20 +386,22 @@ class ArvPathMapper(cwltool.pathmapper.PathMapper):
     def __init__(self, arvrunner, referenced_files, basedir,
                  collection_pattern, file_pattern, name=None, **kwargs):
         self._pathmap = arvrunner.get_uploaded()
-        uploadfiles = []
+        uploadfiles = set()
 
         pdh_path = re.compile(r'^keep:[0-9a-f]{32}\+\d+/.+')
 
         for src in referenced_files:
             if isinstance(src, basestring) and pdh_path.match(src):
                 self._pathmap[src] = (src, collection_pattern % src[5:])
+            if "#" in src:
+                src = src[:src.index("#")]
             if src not in self._pathmap:
                 ab = cwltool.pathmapper.abspath(src, basedir)
                 st = arvados.commands.run.statfile("", ab, fnPattern=file_pattern)
                 if kwargs.get("conformance_test"):
                     self._pathmap[src] = (src, ab)
                 elif isinstance(st, arvados.commands.run.UploadFile):
-                    uploadfiles.append((src, ab, st))
+                    uploadfiles.add((src, ab, st))
                 elif isinstance(st, arvados.commands.run.ArvFile):
                     self._pathmap[src] = (ab, st.fn)
                 else:
@@ -609,7 +614,7 @@ def versionstring():
     arvpkg = pkg_resources.require("arvados-python-client")
     arvcwlpkg = pkg_resources.require("arvados-cwl-runner")
 
-    return "%s\n%s %s\n%s %s\n%s %s" % (sys.argv[0],
+    return "%s %s, %s %s, %s %s" % (sys.argv[0],
                                         "arvados-cwl-runner", arvcwlpkg[0].version,
                                         "arvados-python-client", cwlpkg[0].version,
                                         "cwltool", arvpkg[0].version)

diff --git a/sdk/python/arvados/commands/run.py b/sdk/python/arvados/commands/run.py
index ad442d5d9d8b2f778e415d7e73c70071427b105a..7a298b4c1723ae2f5c3c1836d897a22c08547249 100644 (file)
--- a/sdk/python/arvados/commands/run.py
+++ b/sdk/python/arvados/commands/run.py
@@ -34,6 +34,12 @@ class ArvFile(object):
         self.prefix = prefix
         self.fn = fn
 
+    def __hash__(self):
+        return (self.prefix+self.fn).__hash__()
+
+    def __eq__(self, other):
+        return (self.prefix == other.prefix) and (self.fn == other.fn)
+
 class UploadFile(ArvFile):
     pass