sdk/cwl/arvados_cwl/fsaccess.py

   1 import fnmatch
   2 import os
   3
   4 import cwltool.process
   5
   6 import arvados.util
   7 import arvados.collection
   8
   9 class CollectionFsAccess(cwltool.process.StdFsAccess):
  10     """Implement the cwltool FsAccess interface for Arvados Collections."""
  11
  12     def __init__(self, basedir):
  13         super(CollectionFsAccess, self).__init__(basedir)
  14         self.collections = {}
  15
  16     def get_collection(self, path):
  17         p = path.split("/")
  18         if p[0].startswith("keep:") and arvados.util.keep_locator_pattern.match(p[0][5:]):
  19             pdh = p[0][5:]
  20             if pdh not in self.collections:
  21                 self.collections[pdh] = arvados.collection.CollectionReader(pdh)
  22             return (self.collections[pdh], "/".join(p[1:]))
  23         else:
  24             return (None, path)
  25
  26     def _match(self, collection, patternsegments, parent):
  27         if not patternsegments:
  28             return []
  29
  30         if not isinstance(collection, arvados.collection.RichCollectionBase):
  31             return []
  32
  33         ret = []
  34         # iterate over the files and subcollections in 'collection'
  35         for filename in collection:
  36             if patternsegments[0] == '.':
  37                 # Pattern contains something like "./foo" so just shift
  38                 # past the "./"
  39                 ret.extend(self._match(collection, patternsegments[1:], parent))
  40             elif fnmatch.fnmatch(filename, patternsegments[0]):
  41                 cur = os.path.join(parent, filename)
  42                 if len(patternsegments) == 1:
  43                     ret.append(cur)
  44                 else:
  45                     ret.extend(self._match(collection[filename], patternsegments[1:], cur))
  46         return ret
  47
  48     def glob(self, pattern):
  49         collection, rest = self.get_collection(pattern)
  50         patternsegments = rest.split("/")
  51         return self._match(collection, patternsegments, "keep:" + collection.manifest_locator())
  52
  53     def open(self, fn, mode):
  54         collection, rest = self.get_collection(fn)
  55         if collection:
  56             return collection.open(rest, mode)
  57         else:
  58             return open(self._abs(fn), mode)
  59
  60     def exists(self, fn):
  61         collection, rest = self.get_collection(fn)
  62         if collection:
  63             return collection.exists(rest)
  64         else:
  65             return os.path.exists(self._abs(fn))