9945: Merge branch 'master' into 9945-make-python-package-dependency-free

[arvados.git] / services / fuse / arvados_fuse / __init__.py

diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py
index 30770fc0152a58125495420ebed2d8836768cfd3..f1e49f5afcffff32143b9033c5f83dddcd0c7c65 100644 (file)
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@ -156,13 +156,32 @@ class InodeCache(object):
 
     def _remove(self, obj, clear):
         if clear:
+            # Kernel behavior seems to be that if a file is
+            # referenced, its parents remain referenced too. This
+            # means has_ref() exits early when a collection is not
+            # candidate for eviction.
+            #
+            # By contrast, in_use() doesn't increment references on
+            # parents, so it requires a full tree walk to determine if
+            # a collection is a candidate for eviction.  This takes
+            # .07s for 240000 files, which becomes a major drag when
+            # cap_cache is being called several times a second and
+            # there are multiple non-evictable collections in the
+            # cache.
+            #
+            # So it is important for performance that we do the
+            # has_ref() check first.
+
+            if obj.has_ref(True):
+                _logger.debug("InodeCache cannot clear inode %i, still referenced", obj.inode)
+                return
+
             if obj.in_use():
                 _logger.debug("InodeCache cannot clear inode %i, in use", obj.inode)
                 return
-            if obj.has_ref(True):
-                obj.kernel_invalidate()
-                _logger.debug("InodeCache sent kernel invalidate inode %i", obj.inode)
-                return
+
+            obj.kernel_invalidate()
+            _logger.debug("InodeCache sent kernel invalidate inode %i", obj.inode)
             obj.clear()
 
         # The llfuse lock is released in del_entry(), which is called by
@@ -201,7 +220,8 @@ class InodeCache(object):
                     if obj not in self._by_uuid[obj.cache_uuid]:
                         self._by_uuid[obj.cache_uuid].append(obj)
             self._total += obj.objsize()
-            _logger.debug("InodeCache touched inode %i (size %i) (uuid %s) total now %i", obj.inode, obj.objsize(), obj.cache_uuid, self._total)
+            _logger.debug("InodeCache touched inode %i (size %i) (uuid %s) total now %i (%i entries)",
+                          obj.inode, obj.objsize(), obj.cache_uuid, self._total, len(self._entries))
             self.cap_cache()
 
     def touch(self, obj):
@@ -266,17 +286,22 @@ class Inodes(object):
             del self._entries[entry.inode]
             with llfuse.lock_released:
                 entry.finalize()
-            self.invalidate_inode(entry.inode)
             entry.inode = None
         else:
             entry.dead = True
             _logger.debug("del_entry on inode %i with refcount %i", entry.inode, entry.ref_count)
 
-    def invalidate_inode(self, inode):
-        llfuse.invalidate_inode(inode)
+    def invalidate_inode(self, entry):
+        if entry.has_ref(False):
+            # Only necessary if the kernel has previously done a lookup on this
+            # inode and hasn't yet forgotten about it.
+            llfuse.invalidate_inode(entry.inode)
 
-    def invalidate_entry(self, inode, name):
-        llfuse.invalidate_entry(inode, name.encode(self.encoding))
+    def invalidate_entry(self, entry, name):
+        if entry.has_ref(False):
+            # Only necessary if the kernel has previously done a lookup on this
+            # inode and hasn't yet forgotten about it.
+            llfuse.invalidate_entry(entry.inode, name.encode(self.encoding))
 
     def clear(self):
         self.inode_cache.clear()
@@ -432,8 +457,8 @@ class Operations(llfuse.Operations):
         entry = llfuse.EntryAttributes()
         entry.st_ino = inode
         entry.generation = 0
-        entry.entry_timeout = 60 if e.allow_dirent_cache else 0
-        entry.attr_timeout = 60 if e.allow_attr_cache else 0
+        entry.entry_timeout = 0
+        entry.attr_timeout = e.time_to_next_poll() if e.allow_attr_cache else 0
 
         entry.st_mode = stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH
         if isinstance(e, Directory):