Merge branch '21535-multi-wf-delete'

[arvados.git] / sdk / cwl / arvados_cwl / fsaccess.py

diff --git a/sdk/cwl/arvados_cwl/fsaccess.py b/sdk/cwl/arvados_cwl/fsaccess.py
index 252ca57d47bb30ad1834e7070b3cfdf1e0ffdbb1..a88380b468cfa3100725b79d248a370b8c136e38 100644 (file)
--- a/sdk/cwl/arvados_cwl/fsaccess.py
+++ b/sdk/cwl/arvados_cwl/fsaccess.py
@@ -2,12 +2,6 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-from future import standard_library
-standard_library.install_aliases()
-from builtins import object
-from builtins import str
-from future.utils import viewvalues
-
 import fnmatch
 import os
 import errno
@@ -16,8 +10,9 @@ import re
 import logging
 import threading
 from collections import OrderedDict
+from io import StringIO
 
-import ruamel.yaml as yaml
+import ruamel.yaml
 
 import cwltool.stdfsaccess
 from cwltool.pathmapper import abspath
@@ -100,7 +95,8 @@ class CollectionFsAccess(cwltool.stdfsaccess.StdFsAccess):
         if p.startswith("keep:") and (arvados.util.keep_locator_pattern.match(p[5:]) or
                                       arvados.util.collection_uuid_pattern.match(p[5:])):
             locator = p[5:]
-            return (self.collection_cache.get(locator), urllib.parse.unquote(sp[1]) if len(sp) == 2 else None)
+            rest = os.path.normpath(urllib.parse.unquote(sp[1])) if len(sp) == 2 else None
+            return (self.collection_cache.get(locator), rest)
         else:
             return (None, path)
 
@@ -128,15 +124,15 @@ class CollectionFsAccess(cwltool.stdfsaccess.StdFsAccess):
 
     def glob(self, pattern):
         collection, rest = self.get_collection(pattern)
-        if collection is not None and not rest:
+        if collection is not None and rest in (None, "", "."):
             return [pattern]
         patternsegments = rest.split("/")
         return sorted(self._match(collection, patternsegments, "keep:" + collection.manifest_locator()))
 
-    def open(self, fn, mode):
+    def open(self, fn, mode, encoding=None):
         collection, rest = self.get_collection(fn)
         if collection is not None:
-            return collection.open(rest, mode)
+            return collection.open(rest, mode, encoding=encoding)
         else:
             return super(CollectionFsAccess, self).open(self._abs(fn), mode)
 
@@ -148,6 +144,11 @@ class CollectionFsAccess(cwltool.stdfsaccess.StdFsAccess):
                 return False
             else:
                 raise
+        except IOError as err:
+            if err.errno == errno.ENOENT:
+                return False
+            else:
+                raise
         if collection is not None:
             if rest:
                 return collection.exists(rest)
@@ -223,24 +224,29 @@ class CollectionFetcher(DefaultFetcher):
         self.fsaccess = fs_access
         self.num_retries = num_retries
 
-    def fetch_text(self, url):
+    def fetch_text(self, url, content_types=None):
         if url.startswith("keep:"):
-            with self.fsaccess.open(url, "r") as f:
+            with self.fsaccess.open(url, "r", encoding="utf-8") as f:
                 return f.read()
         if url.startswith("arvwf:"):
             record = self.api_client.workflows().get(uuid=url[6:]).execute(num_retries=self.num_retries)
-            definition = record["definition"] + ('\nlabel: "%s"\n' % record["name"].replace('"', '\\"'))
-            return definition
+            yaml = ruamel.yaml.YAML(typ='rt', pure=True)
+            definition = yaml.load(record["definition"])
+            definition["label"] = record["name"]
+            stream = StringIO()
+            yaml.dump(definition, stream)
+            return stream.getvalue()
         return super(CollectionFetcher, self).fetch_text(url)
 
     def check_exists(self, url):
         try:
             if url.startswith("http://arvados.org/cwl"):
                 return True
-            if url.startswith("keep:"):
-                return self.fsaccess.exists(url)
-            if url.startswith("arvwf:"):
-                if self.fetch_text(url):
+            urld, _ = urllib.parse.urldefrag(url)
+            if urld.startswith("keep:"):
+                return self.fsaccess.exists(urld)
+            if urld.startswith("arvwf:"):
+                if self.fetch_text(urld):
                     return True
         except arvados.errors.NotFoundError:
             return False
@@ -302,7 +308,7 @@ def collectionResolver(api_client, document_loader, uri, num_retries=4):
 
     if pipeline_template_uuid_pattern.match(uri):
         pt = api_client.pipeline_templates().get(uuid=uri).execute(num_retries=num_retries)
-        return u"keep:" + viewvalues(pt["components"])[0]["script_parameters"]["cwl:tool"]
+        return u"keep:" + next(pt["components"].values())["script_parameters"]["cwl:tool"]
 
     p = uri.split("/")
     if arvados.util.keep_locator_pattern.match(p[0]):