From a773baf0d1662b7aab632bfd4f7db50b4b29a6b8 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Mon, 4 Dec 2023 14:28:30 -0500 Subject: [PATCH] 21223: Add tests for setting RLIMIT_NOFILE based on --file-cache Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- services/fuse/arvados_fuse/command.py | 13 ++++++---- services/fuse/tests/test_command_args.py | 30 ++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/services/fuse/arvados_fuse/command.py b/services/fuse/arvados_fuse/command.py index 75ff4c132e..9c607c7f0c 100644 --- a/services/fuse/arvados_fuse/command.py +++ b/services/fuse/arvados_fuse/command.py @@ -137,7 +137,7 @@ class Mount(object): try: self._setup_logging() except Exception as e: - self.logger.exception("arv-mount: exception during setup: %s", e) + self.logger.exception("exception during setup: %s", e) exit(1) try: @@ -153,16 +153,21 @@ class Mount(object): if nofile_limit[0] < minlimit: resource.setrlimit(resource.RLIMIT_NOFILE, (min(minlimit, nofile_limit[1]), nofile_limit[1])) + + if minlimit > nofile_limit[1]: + self.logger.warning("file handles required to meet --file-cache (%s) exceeds hard file handle limit (%s), cache size will be smaller than requested", minlimit, nofile_limit[1]) + except Exception as e: - self.logger.warning("arv-mount: unable to adjust file handle limit: %s", e) + self.logger.warning("unable to adjust file handle limit: %s", e) - self.logger.info("arv-mount: RLIMIT_NOFILE is %s", resource.getrlimit(resource.RLIMIT_NOFILE)) + nofile_limit = resource.getrlimit(resource.RLIMIT_NOFILE) + self.logger.info("file cache capped at %s bytes or less based on available disk (RLIMIT_NOFILE is %s)", ((nofile_limit[0]//8)*64*1024*1024), nofile_limit) try: self._setup_api() self._setup_mount() except Exception as e: - self.logger.exception("arv-mount: exception during setup: %s", e) + self.logger.exception("exception during setup: %s", e) exit(1) def __enter__(self): diff --git a/services/fuse/tests/test_command_args.py b/services/fuse/tests/test_command_args.py index 600bb0fe22..9f4aa53bdb 100644 --- a/services/fuse/tests/test_command_args.py +++ b/services/fuse/tests/test_command_args.py @@ -20,6 +20,7 @@ from . import run_test_server import sys import tempfile import unittest +import resource def noexit(func): """If argparse or arvados_fuse tries to exit, fail the test instead""" @@ -261,6 +262,35 @@ class MountArgsTest(unittest.TestCase): '--foreground', self.mntdir]) arvados_fuse.command.Mount(args) + @noexit + @mock.patch('resource.setrlimit') + @mock.patch('resource.getrlimit') + def test_file_cache(self, getrlimit, setrlimit): + args = arvados_fuse.command.ArgumentParser().parse_args([ + '--foreground', '--file-cache=256000000000', self.mntdir]) + self.assertEqual(args.mode, None) + + getrlimit.return_value = (1024, 1048576) + + self.mnt = arvados_fuse.command.Mount(args) + + setrlimit.assert_called_with(resource.RLIMIT_NOFILE, (30517, 1048576)) + + + @noexit + @mock.patch('resource.setrlimit') + @mock.patch('resource.getrlimit') + def test_file_cache_hard_limit(self, getrlimit, setrlimit): + args = arvados_fuse.command.ArgumentParser().parse_args([ + '--foreground', '--file-cache=256000000000', self.mntdir]) + self.assertEqual(args.mode, None) + + getrlimit.return_value = (1024, 2048) + + self.mnt = arvados_fuse.command.Mount(args) + + setrlimit.assert_called_with(resource.RLIMIT_NOFILE, (2048, 2048)) + class MountErrorTest(unittest.TestCase): def setUp(self): self.mntdir = tempfile.mkdtemp() -- 2.30.2