21541: Manage memory related to project directories

The existing cache management only related to collection directories.
This adds an estimate for other kinds of directories so they will also
contribute to cache size and be candidates to be removed from cache.

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz@curii.com>

diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py
index c04a880122a55690629a15fa2b67b58f22127a79..a70c99fde92aa1d0ab699b6cf94a7b2c40ee3f6b 100644 (file)
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@ -165,6 +165,8 @@ class InodeCache(object):
         return self._total
 
     def _remove(self, obj, clear):
+        if obj.inode is None:
+            return
         if clear:
             # Kernel behavior seems to be that if a file is
             # referenced, its parents remain referenced too. This
@@ -212,11 +214,13 @@ class InodeCache(object):
             _logger.debug("InodeCache cleared inode %i total now %i", obj.inode, self._total)
 
     def cap_cache(self):
+        _logger.debug("in cap_cache %i, %i", self._total, self.cap)
         if self._total > self.cap:
             for ent in listvalues(self._entries):
                 if self._total < self.cap or len(self._entries) < self.min_entries:
                     break
                 self._remove(ent, True)
+            _logger.debug("end cap_cache %i, %i", self._total, self.cap)
 
     def manage(self, obj):
         if obj.persisted():
@@ -229,10 +233,16 @@ class InodeCache(object):
                 else:
                     if obj not in self._by_uuid[obj.cache_uuid]:
                         self._by_uuid[obj.cache_uuid].append(obj)
-            self._total += obj.objsize()
+            self._total += obj.cache_size
             _logger.debug("InodeCache touched inode %i (size %i) (uuid %s) total now %i (%i entries)",
                           obj.inode, obj.objsize(), obj.cache_uuid, self._total, len(self._entries))
 
+    def update_cache_size(self, obj):
+        if obj.inode in self._entries:
+            self._total -= obj.cache_size
+            obj.cache_size = obj.objsize()
+            self._total += obj.cache_size
+
     def touch(self, obj):
         if obj.persisted():
             if obj.inode in self._entries:

diff --git a/services/fuse/arvados_fuse/command.py b/services/fuse/arvados_fuse/command.py
index 74d28a9e8a08228bf3229829370665dfa32f0d90..8c24467aebe4b2c85a8bb097e1b5cfcf1f5e9f99 100644 (file)
--- a/services/fuse/arvados_fuse/command.py
+++ b/services/fuse/arvados_fuse/command.py
@@ -472,7 +472,7 @@ From here, the following directories are available:
 
     def _llfuse_main(self):
         try:
-            llfuse.main()
+            llfuse.main(workers=1)
         except:
             llfuse.close(unmount=False)
             raise

diff --git a/services/fuse/arvados_fuse/fusedir.py b/services/fuse/arvados_fuse/fusedir.py
index 00b9d8b547dc4c03131fc98944528051496a75c6..0671670c9d5b4a238f7f76dbc76457fdd11e808a 100644 (file)
--- a/services/fuse/arvados_fuse/fusedir.py
+++ b/services/fuse/arvados_fuse/fusedir.py
@@ -150,6 +150,9 @@ class Directory(FreshBase):
         self.inodes.touch(self)
         super(Directory, self).fresh()
 
+    def objsize(self):
+        return len(self._entries) * 64
+
     def merge(self, items, fn, same, new_entry):
         """Helper method for updating the contents of the directory.
 
@@ -210,8 +213,8 @@ class Directory(FreshBase):
         if changed:
             self.inodes.invalidate_inode(self)
             self._mtime = time.time()
-
-        self.inodes.inode_cache.cap_cache()
+            self.inodes.inode_cache.update_cache_size(self)
+            self.inodes.inode_cache.cap_cache()
 
         for ent in self._entries.values():
            ent.dec_use()
@@ -239,7 +242,6 @@ class Directory(FreshBase):
         oldentries = self._entries
         self._entries = {}
         for n in oldentries:
-            oldentries[n].clear()
             self.inodes.del_entry(oldentries[n])
         self.invalidate()
 
@@ -531,6 +533,7 @@ class CollectionDirectory(CollectionDirectoryBase):
             self.collection_record_file.invalidate()
             self.inodes.invalidate_inode(self.collection_record_file)
             _logger.debug("%s invalidated collection record file", self)
+        self.inodes.inode_cache.update_cache_size(self)
         self.fresh()
 
     def uuid(self):
@@ -657,6 +660,7 @@ class CollectionDirectory(CollectionDirectoryBase):
             self.collection.stop_threads()
         super(CollectionDirectory, self).clear()
         self._manifest_size = 0
+        self.inodes.inode_cache.update_cache_size(self)
 
 
 class TmpCollectionDirectory(CollectionDirectoryBase):