sdk/cwl/arvados_cwl/fsaccess.py

   1 import fnmatch
   2
   3 class CollectionFsAccess(cwltool.process.StdFsAccess):
   4     """Implement the cwltool FsAccess interface for Arvados Collections."""
   5
   6     def __init__(self, basedir):
   7         super(CollectionFsAccess, self).__init__(basedir)
   8         self.collections = {}
   9
  10     def get_collection(self, path):
  11         p = path.split("/")
  12         if p[0].startswith("keep:") and arvados.util.keep_locator_pattern.match(p[0][5:]):
  13             pdh = p[0][5:]
  14             if pdh not in self.collections:
  15                 self.collections[pdh] = arvados.collection.CollectionReader(pdh)
  16             return (self.collections[pdh], "/".join(p[1:]))
  17         else:
  18             return (None, path)
  19
  20     def _match(self, collection, patternsegments, parent):
  21         if not patternsegments:
  22             return []
  23
  24         if not isinstance(collection, arvados.collection.RichCollectionBase):
  25             return []
  26
  27         ret = []
  28         # iterate over the files and subcollections in 'collection'
  29         for filename in collection:
  30             if patternsegments[0] == '.':
  31                 # Pattern contains something like "./foo" so just shift
  32                 # past the "./"
  33                 ret.extend(self._match(collection, patternsegments[1:], parent))
  34             elif fnmatch.fnmatch(filename, patternsegments[0]):
  35                 cur = os.path.join(parent, filename)
  36                 if len(patternsegments) == 1:
  37                     ret.append(cur)
  38                 else:
  39                     ret.extend(self._match(collection[filename], patternsegments[1:], cur))
  40         return ret
  41
  42     def glob(self, pattern):
  43         collection, rest = self.get_collection(pattern)
  44         patternsegments = rest.split("/")
  45         return self._match(collection, patternsegments, "keep:" + collection.manifest_locator())
  46
  47     def open(self, fn, mode):
  48         collection, rest = self.get_collection(fn)
  49         if collection:
  50             return collection.open(rest, mode)
  51         else:
  52             return open(self._abs(fn), mode)
  53
  54     def exists(self, fn):
  55         collection, rest = self.get_collection(fn)
  56         if collection:
  57             return collection.exists(rest)
  58         else:
  59             return os.path.exists(self._abs(fn))