3198: Use new Collection/ArvadosFile API.

diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py
index 83b4710da818571280eb6544f3fb462f4d8ce787..f1bf9cce5f0d70faf5ee19ca4319eb408f59ce74 100644 (file)
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@ -24,7 +24,7 @@ import ciso8601
 import collections
 
 from fusedir import sanitize_filename, Directory, CollectionDirectory, MagicDirectory, TagsDirectory, ProjectDirectory, SharedDirectory
-from fusefile import StreamReaderFile, StringFile
+from fusefile import StringFile, FuseArvadosFile
 
 _logger = logging.getLogger('arvados.arvados_fuse')
 
@@ -157,7 +157,7 @@ class Operations(llfuse.Operations):
 
     """
 
-    def __init__(self, uid, gid, encoding="utf-8", inode_cache=1000):
+    def __init__(self, uid, gid, encoding="utf-8", inode_cache=1000, num_retries=7):
         super(Operations, self).__init__()
 
         self.inodes = Inodes(inode_cache)
@@ -173,6 +173,8 @@ class Operations(llfuse.Operations):
         # is fully initialized should wait() on this event object.
         self.initlock = threading.Event()
 
+        self.num_retries = num_retries
+
     def init(self):
         # Allow threads that are waiting for the driver to be finished
         # initializing to continue
@@ -196,7 +198,7 @@ class Operations(llfuse.Operations):
         entry.st_mode = stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH
         if isinstance(e, Directory):
             entry.st_mode |= stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH | stat.S_IFDIR
-        elif isinstance(e, StreamReaderFile):
+        elif isinstance(e, FuseArvadosFile):
             entry.st_mode |= stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH | stat.S_IFREG
         else:
             entry.st_mode |= stat.S_IFREG
@@ -266,12 +268,12 @@ class Operations(llfuse.Operations):
 
         try:
             with llfuse.lock_released:
-                return handle.fileobj.readfrom(off, size)
+                return handle.fileobj.readfrom(off, size, self.num_retries)
         except arvados.errors.NotFoundError as e:
             _logger.warning("Block not found: " + str(e))
             raise llfuse.FUSEError(errno.EIO)
         except Exception:
-            _logger.exception()
+            _logger.exception("Read error")
             raise llfuse.FUSEError(errno.EIO)
 
     def release(self, fh):

diff --git a/services/fuse/arvados_fuse/fusedir.py b/services/fuse/arvados_fuse/fusedir.py
index 08a5168a7c7409afab5c988e9a5e18882c0bef98..c1cfaa411e00452976a7c3aace663c0dc27817b5 100644 (file)
--- a/services/fuse/arvados_fuse/fusedir.py
+++ b/services/fuse/arvados_fuse/fusedir.py
@@ -6,9 +6,10 @@ import arvados
 import apiclient
 import functools
 
-from fusefile import StringFile, StreamReaderFile, ObjectFile
+from fusefile import StringFile, ObjectFile, FuseArvadosFile
 from fresh import FreshBase, convertTime, use_counter
 
+import arvados.collection
 from arvados.util import portable_data_hash_pattern, uuid_pattern, collection_uuid_pattern, group_uuid_pattern, user_uuid_pattern, link_uuid_pattern
 
 _logger = logging.getLogger('arvados.arvados_fuse')
@@ -184,6 +185,17 @@ class CollectionDirectory(Directory):
     def same(self, i):
         return i['uuid'] == self.collection_locator or i['portable_data_hash'] == self.collection_locator
 
+    @staticmethod
+    def populate(inodes, cwd, collection, mtime):
+        for entry, item in collection.items():
+            entry = sanitize_filename(entry)
+            if isinstance(item, arvados.collection.RichCollectionBase):
+                cwd._entries[entry] = inodes.add_entry(Directory(cwd.inode, inodes))
+                cwd._mtime = mtime
+                CollectionDirectory.populate(inodes, cwd._entries[entry], item, mtime)
+            else:
+                cwd._entries[entry] = inodes.add_entry(FuseArvadosFile(cwd.inode, item, mtime))
+
     # Used by arv-web.py to switch the contents of the CollectionDirectory
     def change_collection(self, new_locator):
         """Switch the contents of the CollectionDirectory.
@@ -205,16 +217,7 @@ class CollectionDirectory(Directory):
         if self.collection_object_file is not None:
             self.collection_object_file.update(self.collection_object)
 
-        for s in coll_reader.all_streams():
-            cwd = self
-            for part in s.name().split('/'):
-                if part != '' and part != '.':
-                    partname = sanitize_filename(part)
-                    if partname not in cwd._entries:
-                        cwd._entries[partname] = self.inodes.add_entry(Directory(cwd.inode, self.inodes))
-                    cwd = cwd._entries[partname]
-            for k, v in s.files().items():
-                cwd._entries[sanitize_filename(k)] = self.inodes.add_entry(StreamReaderFile(cwd.inode, v, self.mtime()))
+        CollectionDirectory.populate(self.inodes, self, coll_reader, self.mtime())
 
     def update(self):
         try:

diff --git a/services/fuse/arvados_fuse/fusefile.py b/services/fuse/arvados_fuse/fusefile.py
index efe31c387c09432f3905144c19b5417161d6a1a7..9af341f231337ed0732a3419784e88f676a9c032 100644 (file)
--- a/services/fuse/arvados_fuse/fusefile.py
+++ b/services/fuse/arvados_fuse/fusefile.py
@@ -18,7 +18,7 @@ class File(FreshBase):
     def size(self):
         return 0
 
-    def readfrom(self, off, size):
+    def readfrom(self, off, size, num_retries=0):
         return ''
 
     def mtime(self):
@@ -28,18 +28,18 @@ class File(FreshBase):
         return True
 
 
-class StreamReaderFile(File):
-    """Wraps a StreamFileReader as a file."""
+class FuseArvadosFile(File):
+    """Wraps a ArvadosFile."""
 
-    def __init__(self, parent_inode, reader, _mtime):
-        super(StreamReaderFile, self).__init__(parent_inode, _mtime)
-        self.reader = reader
+    def __init__(self, parent_inode, arvfile, _mtime):
+        super(FuseArvadosFile, self).__init__(parent_inode, _mtime)
+        self.arvfile = arvfile
 
     def size(self):
-        return self.reader.size()
+        return self.arvfile.size()
 
-    def readfrom(self, off, size):
-        return self.reader.readfrom(off, size)
+    def readfrom(self, off, size, num_retries=0):
+        return self.arvfile.readfrom(off, size, num_retries)
 
     def stale(self):
         return False
@@ -54,7 +54,7 @@ class StringFile(File):
     def size(self):
         return len(self.contents)
 
-    def readfrom(self, off, size):
+    def readfrom(self, off, size, num_retries=0):
         return self.contents[off:(off+size)]