sdk/cwl/arvados_cwl/fsaccess.py

   1 import fnmatch
   2 import os
   3 import errno
   4
   5 import cwltool.stdfsaccess
   6 from cwltool.pathmapper import abspath
   7
   8 import arvados.util
   9 import arvados.collection
  10 import arvados.arvfile
  11
  12 class CollectionFsAccess(cwltool.stdfsaccess.StdFsAccess):
  13     """Implement the cwltool FsAccess interface for Arvados Collections."""
  14
  15     def __init__(self, basedir, api_client=None, keep_client=None):
  16         super(CollectionFsAccess, self).__init__(basedir)
  17         self.api_client = api_client
  18         self.keep_client = keep_client
  19         self.collections = {}
  20
  21     def get_collection(self, path):
  22         p = path.split("/")
  23         if p[0].startswith("keep:") and arvados.util.keep_locator_pattern.match(p[0][5:]):
  24             pdh = p[0][5:]
  25             if pdh not in self.collections:
  26                 self.collections[pdh] = arvados.collection.CollectionReader(pdh, api_client=self.api_client,
  27                                                                             keep_client=self.keep_client)
  28             return (self.collections[pdh], "/".join(p[1:]))
  29         else:
  30             return (None, path)
  31
  32     def _match(self, collection, patternsegments, parent):
  33         if not patternsegments:
  34             return []
  35
  36         if not isinstance(collection, arvados.collection.RichCollectionBase):
  37             return []
  38
  39         ret = []
  40         # iterate over the files and subcollections in 'collection'
  41         for filename in collection:
  42             if patternsegments[0] == '.':
  43                 # Pattern contains something like "./foo" so just shift
  44                 # past the "./"
  45                 ret.extend(self._match(collection, patternsegments[1:], parent))
  46             elif fnmatch.fnmatch(filename, patternsegments[0]):
  47                 cur = os.path.join(parent, filename)
  48                 if len(patternsegments) == 1:
  49                     ret.append(cur)
  50                 else:
  51                     ret.extend(self._match(collection[filename], patternsegments[1:], cur))
  52         return ret
  53
  54     def glob(self, pattern):
  55         collection, rest = self.get_collection(pattern)
  56         if collection and not rest:
  57             return [pattern]
  58         patternsegments = rest.split("/")
  59         return self._match(collection, patternsegments, "keep:" + collection.manifest_locator())
  60
  61     def open(self, fn, mode):
  62         collection, rest = self.get_collection(fn)
  63         if collection:
  64             return collection.open(rest, mode)
  65         else:
  66             return super(CollectionFsAccess, self).open(self._abs(fn), mode)
  67
  68     def exists(self, fn):
  69         collection, rest = self.get_collection(fn)
  70         if collection:
  71             return collection.exists(rest)
  72         else:
  73             return super(CollectionFsAccess, self).exists(fn)
  74
  75     def isfile(self, fn):  # type: (unicode) -> bool
  76         collection, rest = self.get_collection(fn)
  77         if collection:
  78             if rest:
  79                 return isinstance(collection.find(rest), arvados.arvfile.ArvadosFile)
  80             else:
  81                 return False
  82         else:
  83             return super(CollectionFsAccess, self).isfile(fn)
  84
  85     def isdir(self, fn):  # type: (unicode) -> bool
  86         collection, rest = self.get_collection(fn)
  87         if collection:
  88             if rest:
  89                 return isinstance(collection.find(rest), arvados.collection.RichCollectionBase)
  90             else:
  91                 return True
  92         else:
  93             return super(CollectionFsAccess, self).isdir(fn)
  94
  95     def listdir(self, fn):  # type: (unicode) -> List[unicode]
  96         collection, rest = self.get_collection(fn)
  97         if collection:
  98             if rest:
  99                 dir = collection.find(rest)
 100             else:
 101                 dir = collection
 102             if dir is None:
 103                 raise IOError(errno.ENOENT, "Directory '%s' in '%s' not found" % (rest, collection.portable_data_hash()))
 104             if not isinstance(dir, arvados.collection.RichCollectionBase):
 105                 raise IOError(errno.ENOENT, "Path '%s' in '%s' is not a Directory" % (rest, collection.portable_data_hash()))
 106             return [abspath(l, fn) for l in dir.keys()]
 107         else:
 108             return super(CollectionFsAccess, self).listdir(fn)
 109
 110     def join(self, path, *paths): # type: (unicode, *unicode) -> unicode
 111         if paths and paths[-1].startswith("keep:") and arvados.util.keep_locator_pattern.match(paths[-1][5:]):
 112             return paths[-1]
 113         return os.path.join(path, *paths)
 114
 115     def realpath(self, path):
 116         if path.startswith("$(task.tmpdir)") or path.startswith("$(task.outdir)"):
 117             return path
 118         collection, rest = self.get_collection(path)
 119         if collection:
 120             return path
 121         else:
 122             return os.path.realpath(path)