sdk/cwl/arvados_cwl/fsaccess.py

   1 import fnmatch
   2 import os
   3
   4 import cwltool.process
   5 from cwltool.pathmapper import abspath
   6
   7 import arvados.util
   8 import arvados.collection
   9 import arvados.arvfile
  10
  11 class CollectionFsAccess(cwltool.process.StdFsAccess):
  12     """Implement the cwltool FsAccess interface for Arvados Collections."""
  13
  14     def __init__(self, basedir, api_client=None):
  15         super(CollectionFsAccess, self).__init__(basedir)
  16         self.api_client = api_client
  17         self.collections = {}
  18
  19     def get_collection(self, path):
  20         p = path.split("/")
  21         if p[0].startswith("keep:") and arvados.util.keep_locator_pattern.match(p[0][5:]):
  22             pdh = p[0][5:]
  23             if pdh not in self.collections:
  24                 self.collections[pdh] = arvados.collection.CollectionReader(pdh, api_client=self.api_client)
  25             return (self.collections[pdh], "/".join(p[1:]))
  26         else:
  27             return (None, path)
  28
  29     def _match(self, collection, patternsegments, parent):
  30         if not patternsegments:
  31             return []
  32
  33         if not isinstance(collection, arvados.collection.RichCollectionBase):
  34             return []
  35
  36         ret = []
  37         # iterate over the files and subcollections in 'collection'
  38         for filename in collection:
  39             if patternsegments[0] == '.':
  40                 # Pattern contains something like "./foo" so just shift
  41                 # past the "./"
  42                 ret.extend(self._match(collection, patternsegments[1:], parent))
  43             elif fnmatch.fnmatch(filename, patternsegments[0]):
  44                 cur = os.path.join(parent, filename)
  45                 if len(patternsegments) == 1:
  46                     ret.append(cur)
  47                 else:
  48                     ret.extend(self._match(collection[filename], patternsegments[1:], cur))
  49         return ret
  50
  51     def glob(self, pattern):
  52         collection, rest = self.get_collection(pattern)
  53         if collection and not rest:
  54             return [pattern]
  55         patternsegments = rest.split("/")
  56         return self._match(collection, patternsegments, "keep:" + collection.manifest_locator())
  57
  58     def open(self, fn, mode):
  59         collection, rest = self.get_collection(fn)
  60         if collection:
  61             return collection.open(rest, mode)
  62         else:
  63             return super(CollectionFsAccess, self).open(self._abs(fn), mode)
  64
  65     def exists(self, fn):
  66         collection, rest = self.get_collection(fn)
  67         if collection:
  68             return collection.exists(rest)
  69         else:
  70             return super(CollectionFsAccess, self).exists(fn)
  71
  72     def isfile(self, fn):  # type: (unicode) -> bool
  73         collection, rest = self.get_collection(fn)
  74         if collection:
  75             if rest:
  76                 return isinstance(collection.find(rest), arvados.arvfile.ArvadosFile)
  77             else:
  78                 return False
  79         else:
  80             return super(CollectionFsAccess, self).isfile(fn)
  81
  82     def isdir(self, fn):  # type: (unicode) -> bool
  83         collection, rest = self.get_collection(fn)
  84         if collection:
  85             if rest:
  86                 return isinstance(collection.find(rest), arvados.collection.Collection)
  87             else:
  88                 return True
  89         else:
  90             return super(CollectionFsAccess, self).isdir(fn)
  91
  92     def listdir(self, fn):  # type: (unicode) -> List[unicode]
  93         collection, rest = self.get_collection(fn)
  94         if rest:
  95             dir = collection.find(rest)
  96         else:
  97             dir = collection
  98         if collection:
  99             return [abspath(l, fn) for l in dir.keys()]
 100         else:
 101             return super(CollectionFsAccess, self).listdir(fn)
 102
 103     def join(self, path, *paths): # type: (unicode, *unicode) -> unicode
 104         if paths and paths[-1].startswith("keep:") and arvados.util.keep_locator_pattern.match(paths[-1][5:]):
 105             return paths[-1]
 106         return os.path.join(path, *paths)