Merge branch '12278-cwl-debug-flag' closes #12278

[arvados.git] / sdk / cwl / arvados_cwl / pathmapper.py

diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py
index e39c7d23ce22fe71e94d7956e2f5abe4a8e323e3..213fd21789283610a56c7f3cfe1c1267ef53c5d7 100644 (file)
--- a/sdk/cwl/arvados_cwl/pathmapper.py
+++ b/sdk/cwl/arvados_cwl/pathmapper.py
@@ -58,6 +58,8 @@ class ArvPathMapper(PathMapper):
         if isinstance(src, basestring) and ArvPathMapper.pdh_dirpath.match(src):
             self._pathmap[src] = MapperEnt(src, self.collection_pattern % urllib.unquote(src[5:]), srcobj["class"], True)
 
+        debug = logger.isEnabledFor(logging.DEBUG)
+
         if src not in self._pathmap:
             if src.startswith("file:"):
                 # Local FS ref, may need to be uploaded or may be on keep
@@ -67,7 +69,7 @@ class ArvPathMapper(PathMapper):
                                                    fnPattern="keep:%s/%s",
                                                    dirPattern="keep:%s/%s",
                                                    raiseOSError=True)
-                with SourceLine(srcobj, "location", WorkflowException):
+                with SourceLine(srcobj, "location", WorkflowException, debug):
                     if isinstance(st, arvados.commands.run.UploadFile):
                         uploadfiles.add((src, ab, st))
                     elif isinstance(st, arvados.commands.run.ArvFile):
@@ -82,10 +84,10 @@ class ArvPathMapper(PathMapper):
             else:
                 self._pathmap[src] = MapperEnt(src, src, srcobj["class"], True)
 
-        with SourceLine(srcobj, "secondaryFiles", WorkflowException):
+        with SourceLine(srcobj, "secondaryFiles", WorkflowException, debug):
             for l in srcobj.get("secondaryFiles", []):
                 self.visit(l, uploadfiles)
-        with SourceLine(srcobj, "listing", WorkflowException):
+        with SourceLine(srcobj, "listing", WorkflowException, debug):
             for l in srcobj.get("listing", []):
                 self.visit(l, uploadfiles)
 
@@ -195,20 +197,34 @@ class ArvPathMapper(PathMapper):
         self.keepdir = None
 
     def reversemap(self, target):
-        if target.startswith("keep:"):
+        p = super(ArvPathMapper, self).reversemap(target)
+        if p:
+            return p
+        elif target.startswith("keep:"):
             return (target, target)
         elif self.keepdir and target.startswith(self.keepdir):
-            return (target, "keep:" + target[len(self.keepdir)+1:])
+            kp = "keep:" + target[len(self.keepdir)+1:]
+            return (kp, kp)
         else:
-            return super(ArvPathMapper, self).reversemap(target)
+            return None
 
 class StagingPathMapper(PathMapper):
     _follow_dirs = True
 
+    def __init__(self, referenced_files, basedir, stagedir, separateDirs=True):
+        self.targets = set()
+        super(StagingPathMapper, self).__init__(referenced_files, basedir, stagedir, separateDirs)
+
     def visit(self, obj, stagedir, basedir, copy=False, staged=False):
         # type: (Dict[unicode, Any], unicode, unicode, bool) -> None
         loc = obj["location"]
         tgt = os.path.join(stagedir, obj["basename"])
+        basetgt, baseext = os.path.splitext(tgt)
+        n = 1
+        while tgt in self.targets:
+            n += 1
+            tgt = "%s_%i%s" % (basetgt, n, baseext)
+        self.targets.add(tgt)
         if obj["class"] == "Directory":
             self._pathmap[loc] = MapperEnt(loc, tgt, "Directory", staged)
             if loc.startswith("_:") or self._follow_dirs: