From 32e56d60fd965260662b5c8d8aaafc0d793e33bd Mon Sep 17 00:00:00 2001
From: Peter Amstutz <peter.amstutz@curii.com>
Date: Thu, 7 Mar 2024 09:39:35 -0500
Subject: [PATCH] 21541: Manage memory related to project directories

The existing cache management only related to collection directories.
This adds an estimate for other kinds of directories so they will also
contribute to cache size and be candidates to be removed from cache.

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz@curii.com>
---
 services/fuse/arvados_fuse/__init__.py | 12 +++++++++++-
 services/fuse/arvados_fuse/command.py  |  2 +-
 services/fuse/arvados_fuse/fusedir.py  | 10 +++++++---
 3 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py
index c04a880122..a70c99fde9 100644
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@ -165,6 +165,8 @@ class InodeCache(object):
         return self._total
 
     def _remove(self, obj, clear):
+        if obj.inode is None:
+            return
         if clear:
             # Kernel behavior seems to be that if a file is
             # referenced, its parents remain referenced too. This
@@ -212,11 +214,13 @@ class InodeCache(object):
             _logger.debug("InodeCache cleared inode %i total now %i", obj.inode, self._total)
 
     def cap_cache(self):
+        _logger.debug("in cap_cache %i, %i", self._total, self.cap)
         if self._total > self.cap:
             for ent in listvalues(self._entries):
                 if self._total < self.cap or len(self._entries) < self.min_entries:
                     break
                 self._remove(ent, True)
+            _logger.debug("end cap_cache %i, %i", self._total, self.cap)
 
     def manage(self, obj):
         if obj.persisted():
@@ -229,10 +233,16 @@ class InodeCache(object):
                 else:
                     if obj not in self._by_uuid[obj.cache_uuid]:
                         self._by_uuid[obj.cache_uuid].append(obj)
-            self._total += obj.objsize()
+            self._total += obj.cache_size
             _logger.debug("InodeCache touched inode %i (size %i) (uuid %s) total now %i (%i entries)",
                           obj.inode, obj.objsize(), obj.cache_uuid, self._total, len(self._entries))
 
+    def update_cache_size(self, obj):
+        if obj.inode in self._entries:
+            self._total -= obj.cache_size
+            obj.cache_size = obj.objsize()
+            self._total += obj.cache_size
+
     def touch(self, obj):
         if obj.persisted():
             if obj.inode in self._entries:
diff --git a/services/fuse/arvados_fuse/command.py b/services/fuse/arvados_fuse/command.py
index 74d28a9e8a..8c24467aeb 100644
--- a/services/fuse/arvados_fuse/command.py
+++ b/services/fuse/arvados_fuse/command.py
@@ -472,7 +472,7 @@ From here, the following directories are available:
 
     def _llfuse_main(self):
         try:
-            llfuse.main()
+            llfuse.main(workers=1)
         except:
             llfuse.close(unmount=False)
             raise
diff --git a/services/fuse/arvados_fuse/fusedir.py b/services/fuse/arvados_fuse/fusedir.py
index 00b9d8b547..0671670c9d 100644
--- a/services/fuse/arvados_fuse/fusedir.py
+++ b/services/fuse/arvados_fuse/fusedir.py
@@ -150,6 +150,9 @@ class Directory(FreshBase):
         self.inodes.touch(self)
         super(Directory, self).fresh()
 
+    def objsize(self):
+        return len(self._entries) * 64
+
     def merge(self, items, fn, same, new_entry):
         """Helper method for updating the contents of the directory.
 
@@ -210,8 +213,8 @@ class Directory(FreshBase):
         if changed:
             self.inodes.invalidate_inode(self)
             self._mtime = time.time()
-
-        self.inodes.inode_cache.cap_cache()
+            self.inodes.inode_cache.update_cache_size(self)
+            self.inodes.inode_cache.cap_cache()
 
         for ent in self._entries.values():
            ent.dec_use()
@@ -239,7 +242,6 @@ class Directory(FreshBase):
         oldentries = self._entries
         self._entries = {}
         for n in oldentries:
-            oldentries[n].clear()
             self.inodes.del_entry(oldentries[n])
         self.invalidate()
 
@@ -531,6 +533,7 @@ class CollectionDirectory(CollectionDirectoryBase):
             self.collection_record_file.invalidate()
             self.inodes.invalidate_inode(self.collection_record_file)
             _logger.debug("%s invalidated collection record file", self)
+        self.inodes.inode_cache.update_cache_size(self)
         self.fresh()
 
     def uuid(self):
@@ -657,6 +660,7 @@ class CollectionDirectory(CollectionDirectoryBase):
             self.collection.stop_threads()
         super(CollectionDirectory, self).clear()
         self._manifest_size = 0
+        self.inodes.inode_cache.update_cache_size(self)
 
 
 class TmpCollectionDirectory(CollectionDirectoryBase):
-- 
2.30.2