21541: Manage memory related to project directories

[arvados.git] / services / fuse / arvados_fuse / fusedir.py

diff --git a/services/fuse/arvados_fuse/fusedir.py b/services/fuse/arvados_fuse/fusedir.py
index e3b8dd4c2cca29616626dab55f6d440c22b58f51..0671670c9d5b4a238f7f76dbc76457fdd11e808a 100644 (file)
--- a/services/fuse/arvados_fuse/fusedir.py
+++ b/services/fuse/arvados_fuse/fusedir.py
@@ -150,6 +150,9 @@ class Directory(FreshBase):
         self.inodes.touch(self)
         super(Directory, self).fresh()
 
+    def objsize(self):
+        return len(self._entries) * 64
+
     def merge(self, items, fn, same, new_entry):
         """Helper method for updating the contents of the directory.
 
@@ -176,29 +179,45 @@ class Directory(FreshBase):
         changed = False
         for i in items:
             name = self.sanitize_filename(fn(i))
-            if name:
-                if name in oldentries and same(oldentries[name], i):
+            if not name:
+                continue
+            if name in oldentries:
+                ent = oldentries[name]
+                ent.inc_use()
+                if same(ent, i):
                     # move existing directory entry over
-                    self._entries[name] = oldentries[name]
+                    self._entries[name] = ent
                     del oldentries[name]
-                else:
-                    _logger.debug("Adding entry '%s' to inode %i", name, self.inode)
-                    # create new directory entry
-                    ent = new_entry(i)
-                    if ent is not None:
-                        self._entries[name] = self.inodes.add_entry(ent)
-                        changed = True
+
+        for i in items:
+            name = self.sanitize_filename(fn(i))
+            if not name:
+                continue
+            if name not in self._entries:
+                _logger.debug("Adding entry '%s' to inode %i", name, self.inode)
+                # create new directory entry
+                ent = new_entry(i)
+                if ent is not None:
+                    ent.inc_use()
+                    self._entries[name] = self.inodes.add_entry(ent)
+                    changed = True
 
         # delete any other directory entries that were not in found in 'items'
         for i in oldentries:
             _logger.debug("Forgetting about entry '%s' on inode %i", i, self.inode)
             self.inodes.invalidate_entry(self, i)
             self.inodes.del_entry(oldentries[i])
+            ent.dec_use()
             changed = True
 
         if changed:
             self.inodes.invalidate_inode(self)
             self._mtime = time.time()
+            self.inodes.inode_cache.update_cache_size(self)
+            self.inodes.inode_cache.cap_cache()
+
+        for ent in self._entries.values():
+           ent.dec_use()
 
         self.fresh()
 
@@ -223,7 +242,6 @@ class Directory(FreshBase):
         oldentries = self._entries
         self._entries = {}
         for n in oldentries:
-            oldentries[n].clear()
             self.inodes.del_entry(oldentries[n])
         self.invalidate()
 
@@ -515,6 +533,7 @@ class CollectionDirectory(CollectionDirectoryBase):
             self.collection_record_file.invalidate()
             self.inodes.invalidate_inode(self.collection_record_file)
             _logger.debug("%s invalidated collection record file", self)
+        self.inodes.inode_cache.update_cache_size(self)
         self.fresh()
 
     def uuid(self):
@@ -641,6 +660,7 @@ class CollectionDirectory(CollectionDirectoryBase):
             self.collection.stop_threads()
         super(CollectionDirectory, self).clear()
         self._manifest_size = 0
+        self.inodes.inode_cache.update_cache_size(self)
 
 
 class TmpCollectionDirectory(CollectionDirectoryBase):
@@ -1060,10 +1080,14 @@ class ProjectDirectory(Directory):
 
         try:
             with llfuse.lock_released:
+                _logger.debug("Getting lock to update %s", self.project_uuid)
                 self._updating_lock.acquire()
                 if not self.stale():
+                    _logger.debug("%s was updated already", self.project_uuid)
                     return
 
+                _logger.debug("Requesting update of %s", self.project_uuid)
+
                 if group_uuid_pattern.match(self.project_uuid):
                     self.project_object = self.api.groups().get(
                         uuid=self.project_uuid).execute(num_retries=self.num_retries)
@@ -1092,7 +1116,6 @@ class ProjectDirectory(Directory):
                         *self._filters_for('collections', qualified=True),
                     ],
                 ) if obj['current_version_uuid'] == obj['uuid'])
-
             # end with llfuse.lock_released, re-acquire lock
 
             self.merge(contents,