services/fuse/tests/test_command_args.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: AGPL-3.0
   4
   5 import arvados
   6 import arvados_fuse
   7 import arvados_fuse.command
   8 import contextlib
   9 import functools
  10 import io
  11 import json
  12 import llfuse
  13 import logging
  14 import os
  15 import sys
  16 import tempfile
  17 import unittest
  18 import resource
  19
  20 from unittest import mock
  21
  22 from . import run_test_server
  23
  24 def noexit(func):
  25     """If argparse or arvados_fuse tries to exit, fail the test instead"""
  26     class SystemExitCaught(Exception):
  27         pass
  28     @functools.wraps(func)
  29     def wrapper(*args, **kwargs):
  30         try:
  31             return func(*args, **kwargs)
  32         except SystemExit:
  33             raise SystemExitCaught
  34     return wrapper
  35
  36 @contextlib.contextmanager
  37 def nostderr():
  38     with open(os.devnull, 'w') as dn:
  39         orig, sys.stderr = sys.stderr, dn
  40         try:
  41             yield
  42         finally:
  43             sys.stderr = orig
  44
  45
  46 class MountArgsTest(unittest.TestCase):
  47     def setUp(self):
  48         self.mntdir = tempfile.mkdtemp()
  49         run_test_server.authorize_with('active')
  50
  51     def tearDown(self):
  52         os.rmdir(self.mntdir)
  53
  54     def lookup(self, mnt, *path):
  55         ent = mnt.operations.inodes[llfuse.ROOT_INODE]
  56         for p in path:
  57             ent = ent[p]
  58         return ent
  59
  60     @contextlib.contextmanager
  61     def stderrMatches(self, stderr):
  62         orig, sys.stderr = sys.stderr, stderr
  63         try:
  64             yield
  65         finally:
  66             sys.stderr = orig
  67
  68     def check_ent_type(self, cls, *path):
  69         ent = self.lookup(self.mnt, *path)
  70         self.assertEqual(ent.__class__, cls)
  71         return ent
  72
  73     @noexit
  74     def test_default_all(self):
  75         args = arvados_fuse.command.ArgumentParser().parse_args([
  76             '--foreground', self.mntdir])
  77         self.assertEqual(args.mode, None)
  78         self.mnt = arvados_fuse.command.Mount(args)
  79         e = self.check_ent_type(arvados_fuse.ProjectDirectory, 'home')
  80         self.assertEqual(e.project_object['uuid'],
  81                          run_test_server.fixture('users')['active']['uuid'])
  82         e = self.check_ent_type(arvados_fuse.MagicDirectory, 'by_id')
  83
  84         e = self.check_ent_type(arvados_fuse.StringFile, 'README')
  85         readme = e.readfrom(0, -1).decode()
  86         self.assertRegex(readme, r'active-user@arvados\.local')
  87         self.assertRegex(readme, r'\n$')
  88
  89         e = self.check_ent_type(arvados_fuse.StringFile, 'by_id', 'README')
  90         txt = e.readfrom(0, -1).decode()
  91         self.assertRegex(txt, r'portable data hash')
  92         self.assertRegex(txt, r'\n$')
  93
  94     @noexit
  95     def test_by_id(self):
  96         args = arvados_fuse.command.ArgumentParser().parse_args([
  97             '--by-id',
  98             '--foreground', self.mntdir])
  99         self.assertEqual(args.mode, 'by_id')
 100         self.mnt = arvados_fuse.command.Mount(args)
 101         e = self.check_ent_type(arvados_fuse.MagicDirectory)
 102         self.assertEqual(e.pdh_only, False)
 103         self.assertEqual(True, self.mnt.listen_for_events)
 104
 105     @noexit
 106     def test_by_pdh(self):
 107         args = arvados_fuse.command.ArgumentParser().parse_args([
 108             '--by-pdh',
 109             '--foreground', self.mntdir])
 110         self.assertEqual(args.mode, 'by_pdh')
 111         self.mnt = arvados_fuse.command.Mount(args)
 112         e = self.check_ent_type(arvados_fuse.MagicDirectory)
 113         self.assertEqual(e.pdh_only, True)
 114         self.assertEqual(False, self.mnt.listen_for_events)
 115
 116     @noexit
 117     def test_by_tag(self):
 118         args = arvados_fuse.command.ArgumentParser().parse_args([
 119             '--by-tag',
 120             '--foreground', self.mntdir])
 121         self.assertEqual(args.mode, 'by_tag')
 122         self.mnt = arvados_fuse.command.Mount(args)
 123         e = self.check_ent_type(arvados_fuse.TagsDirectory)
 124         self.assertEqual(True, self.mnt.listen_for_events)
 125
 126     @noexit
 127     def test_collection(self, id_type='uuid'):
 128         c = run_test_server.fixture('collections')['public_text_file']
 129         cid = c[id_type]
 130         args = arvados_fuse.command.ArgumentParser().parse_args([
 131             '--collection', cid,
 132             '--foreground', self.mntdir])
 133         self.mnt = arvados_fuse.command.Mount(args)
 134         e = self.check_ent_type(arvados_fuse.CollectionDirectory)
 135         self.assertEqual(e.collection_locator, cid)
 136         self.assertEqual(id_type == 'uuid', self.mnt.listen_for_events)
 137
 138     def test_collection_pdh(self):
 139         self.test_collection('portable_data_hash')
 140
 141     @noexit
 142     def test_home(self):
 143         args = arvados_fuse.command.ArgumentParser().parse_args([
 144             '--home',
 145             '--foreground', self.mntdir])
 146         self.assertEqual(args.mode, 'home')
 147         self.mnt = arvados_fuse.command.Mount(args)
 148         e = self.check_ent_type(arvados_fuse.ProjectDirectory)
 149         self.assertEqual(e.project_object['uuid'],
 150                          run_test_server.fixture('users')['active']['uuid'])
 151         self.assertEqual(True, self.mnt.listen_for_events)
 152
 153     def test_mutually_exclusive_args(self):
 154         cid = run_test_server.fixture('collections')['public_text_file']['uuid']
 155         gid = run_test_server.fixture('groups')['aproject']['uuid']
 156         for badargs in [
 157                 ['--mount-tmp', 'foo', '--collection', cid],
 158                 ['--mount-tmp', 'foo', '--project', gid],
 159                 ['--collection', cid, '--project', gid],
 160                 ['--by-id', '--project', gid],
 161                 ['--mount-tmp', 'foo', '--by-id'],
 162         ]:
 163             with nostderr():
 164                 with self.assertRaises(SystemExit):
 165                     args = arvados_fuse.command.ArgumentParser().parse_args(
 166                         badargs + ['--foreground', self.mntdir])
 167                     arvados_fuse.command.Mount(args)
 168     @noexit
 169     def test_project(self):
 170         uuid = run_test_server.fixture('groups')['aproject']['uuid']
 171         args = arvados_fuse.command.ArgumentParser().parse_args([
 172             '--project', uuid,
 173             '--foreground', self.mntdir])
 174         self.mnt = arvados_fuse.command.Mount(args)
 175         e = self.check_ent_type(arvados_fuse.ProjectDirectory)
 176         self.assertEqual(e.project_object['uuid'], uuid)
 177
 178     @noexit
 179     def test_shared(self):
 180         args = arvados_fuse.command.ArgumentParser().parse_args([
 181             '--shared',
 182             '--foreground', self.mntdir])
 183         self.assertEqual(args.mode, 'shared')
 184         self.mnt = arvados_fuse.command.Mount(args)
 185         e = self.check_ent_type(arvados_fuse.SharedDirectory)
 186         self.assertEqual(e.current_user['uuid'],
 187                          run_test_server.fixture('users')['active']['uuid'])
 188         self.assertEqual(True, self.mnt.listen_for_events)
 189
 190     def test_version_argument(self):
 191         # The argparse version action prints to stderr in Python 2 and stdout
 192         # in Python 3.4 and up. Write both to the same stream so the test can pass
 193         # in both cases.
 194         origerr = sys.stderr
 195         origout = sys.stdout
 196         sys.stderr = io.StringIO()
 197         sys.stdout = sys.stderr
 198
 199         with self.assertRaises(SystemExit):
 200             args = arvados_fuse.command.ArgumentParser().parse_args(['--version'])
 201         self.assertRegex(sys.stdout.getvalue(), r'[0-9]+\.[0-9]+\.[0-9]+')
 202         sys.stderr.close()
 203         sys.stderr = origerr
 204         sys.stdout = origout
 205
 206     @noexit
 207     @mock.patch('arvados.events.subscribe')
 208     def test_disable_event_listening(self, mock_subscribe):
 209         args = arvados_fuse.command.ArgumentParser().parse_args([
 210             '--disable-event-listening',
 211             '--by-id',
 212             '--foreground', self.mntdir])
 213         self.mnt = arvados_fuse.command.Mount(args)
 214         self.assertEqual(True, self.mnt.listen_for_events)
 215         self.assertEqual(True, self.mnt.args.disable_event_listening)
 216         with self.mnt:
 217             pass
 218         self.assertEqual(0, mock_subscribe.call_count)
 219
 220     @noexit
 221     @mock.patch('arvados.events.subscribe')
 222     def test_custom(self, mock_subscribe):
 223         args = arvados_fuse.command.ArgumentParser().parse_args([
 224             '--mount-tmp', 'foo',
 225             '--mount-tmp', 'bar',
 226             '--mount-home', 'my_home',
 227             '--foreground', self.mntdir])
 228         self.assertEqual(args.mode, None)
 229         self.mnt = arvados_fuse.command.Mount(args)
 230         self.check_ent_type(arvados_fuse.Directory)
 231         self.check_ent_type(arvados_fuse.TmpCollectionDirectory, 'foo')
 232         self.check_ent_type(arvados_fuse.TmpCollectionDirectory, 'bar')
 233         e = self.check_ent_type(arvados_fuse.ProjectDirectory, 'my_home')
 234         self.assertEqual(e.project_object['uuid'],
 235                          run_test_server.fixture('users')['active']['uuid'])
 236         self.assertEqual(True, self.mnt.listen_for_events)
 237         with self.mnt:
 238             pass
 239         self.assertEqual(1, mock_subscribe.call_count)
 240
 241     @noexit
 242     @mock.patch('arvados.events.subscribe')
 243     def test_custom_no_listen(self, mock_subscribe):
 244         args = arvados_fuse.command.ArgumentParser().parse_args([
 245             '--mount-by-pdh', 'pdh',
 246             '--mount-tmp', 'foo',
 247             '--mount-tmp', 'bar',
 248             '--foreground', self.mntdir])
 249         self.mnt = arvados_fuse.command.Mount(args)
 250         self.assertEqual(False, self.mnt.listen_for_events)
 251         with self.mnt:
 252             pass
 253         self.assertEqual(0, mock_subscribe.call_count)
 254
 255     def test_custom_unsupported_layouts(self):
 256         for name in ['.', '..', '', 'foo/bar', '/foo']:
 257             with nostderr():
 258                 with self.assertRaises(SystemExit):
 259                     args = arvados_fuse.command.ArgumentParser().parse_args([
 260                         '--mount-tmp', name,
 261                         '--foreground', self.mntdir])
 262                     arvados_fuse.command.Mount(args)
 263
 264     @noexit
 265     @mock.patch('resource.setrlimit')
 266     @mock.patch('resource.getrlimit')
 267     def test_default_file_cache(self, getrlimit, setrlimit):
 268         args = arvados_fuse.command.ArgumentParser().parse_args([
 269             '--foreground', self.mntdir])
 270         self.assertEqual(args.mode, None)
 271         getrlimit.return_value = (1024, 1048576)
 272         self.mnt = arvados_fuse.command.Mount(args)
 273         setrlimit.assert_called_with(resource.RLIMIT_NOFILE, (10240, 1048576))
 274
 275     @noexit
 276     @mock.patch('resource.setrlimit')
 277     @mock.patch('resource.getrlimit')
 278     def test_small_file_cache(self, getrlimit, setrlimit):
 279         args = arvados_fuse.command.ArgumentParser().parse_args([
 280             '--foreground', '--file-cache=256000000', self.mntdir])
 281         self.assertEqual(args.mode, None)
 282         getrlimit.return_value = (1024, 1048576)
 283         self.mnt = arvados_fuse.command.Mount(args)
 284         setrlimit.assert_not_called()
 285
 286     @noexit
 287     @mock.patch('resource.setrlimit')
 288     @mock.patch('resource.getrlimit')
 289     def test_large_file_cache(self, getrlimit, setrlimit):
 290         args = arvados_fuse.command.ArgumentParser().parse_args([
 291             '--foreground', '--file-cache=256000000000', self.mntdir])
 292         self.assertEqual(args.mode, None)
 293         getrlimit.return_value = (1024, 1048576)
 294         self.mnt = arvados_fuse.command.Mount(args)
 295         setrlimit.assert_called_with(resource.RLIMIT_NOFILE, (30517, 1048576))
 296
 297     @noexit
 298     @mock.patch('resource.setrlimit')
 299     @mock.patch('resource.getrlimit')
 300     def test_file_cache_hard_limit(self, getrlimit, setrlimit):
 301         args = arvados_fuse.command.ArgumentParser().parse_args([
 302             '--foreground', '--file-cache=256000000000', self.mntdir])
 303         self.assertEqual(args.mode, None)
 304         getrlimit.return_value = (1024, 2048)
 305         self.mnt = arvados_fuse.command.Mount(args)
 306         setrlimit.assert_called_with(resource.RLIMIT_NOFILE, (2048, 2048))
 307
 308 class MountErrorTest(unittest.TestCase):
 309     def setUp(self):
 310         self.mntdir = tempfile.mkdtemp()
 311         run_test_server.run()
 312         run_test_server.authorize_with("active")
 313         self.logger = logging.getLogger("null")
 314         self.logger.setLevel(logging.CRITICAL+1)
 315
 316     def tearDown(self):
 317         if os.path.exists(self.mntdir):
 318             # If the directory was not unmounted, this will raise an exception.
 319             os.rmdir(self.mntdir)
 320         run_test_server.reset()
 321
 322     def test_no_token(self):
 323         del arvados.config._settings["ARVADOS_API_TOKEN"]
 324         arvados.config._settings = {}
 325         with self.assertRaises(SystemExit) as ex:
 326             args = arvados_fuse.command.ArgumentParser().parse_args([self.mntdir])
 327             arvados_fuse.command.Mount(args, logger=self.logger).run()
 328         self.assertEqual(1, ex.exception.code)
 329
 330     def test_no_host(self):
 331         del arvados.config._settings["ARVADOS_API_HOST"]
 332         with self.assertRaises(SystemExit) as ex:
 333             args = arvados_fuse.command.ArgumentParser().parse_args([self.mntdir])
 334             arvados_fuse.command.Mount(args, logger=self.logger).run()
 335         self.assertEqual(1, ex.exception.code)
 336
 337     def test_bogus_host(self):
 338         arvados.config._settings["ARVADOS_API_HOST"] = "100::"
 339         with self.assertRaises(SystemExit) as ex, mock.patch('time.sleep'):
 340             args = arvados_fuse.command.ArgumentParser().parse_args([self.mntdir])
 341             arvados_fuse.command.Mount(args, logger=self.logger).run()
 342         self.assertEqual(1, ex.exception.code)
 343
 344     def test_bogus_token(self):
 345         arvados.config._settings["ARVADOS_API_TOKEN"] = "zzzzzzzzzzzzz"
 346         with self.assertRaises(SystemExit) as ex:
 347             args = arvados_fuse.command.ArgumentParser().parse_args([self.mntdir])
 348             arvados_fuse.command.Mount(args, logger=self.logger).run()
 349         self.assertEqual(1, ex.exception.code)
 350
 351     def test_bogus_mount_dir(self):
 352         # All FUSE errors in llfuse.init() are raised as RuntimeError
 353         # An easy error to trigger is to supply a nonexistent mount point,
 354         # so test that one.
 355         #
 356         # Other possible errors that also raise RuntimeError (but are much
 357         # harder to test automatically because they depend on operating
 358         # system configuration):
 359         #
 360         # The user doesn't have permission to use FUSE
 361         # The user specified --allow-other but user_allow_other is not set
 362         # in /etc/fuse.conf
 363         os.rmdir(self.mntdir)
 364         with self.assertRaises(SystemExit) as ex:
 365             args = arvados_fuse.command.ArgumentParser().parse_args([self.mntdir])
 366             arvados_fuse.command.Mount(args, logger=self.logger).run()
 367         self.assertEqual(1, ex.exception.code)
 368
 369     def test_unreadable_collection(self):
 370         with self.assertRaises(SystemExit) as ex:
 371             args = arvados_fuse.command.ArgumentParser().parse_args([
 372                 "--collection", "zzzzz-4zz18-zzzzzzzzzzzzzzz", self.mntdir])
 373             arvados_fuse.command.Mount(args, logger=self.logger).run()
 374         self.assertEqual(1, ex.exception.code)
 375
 376     def test_unreadable_project(self):
 377         with self.assertRaises(SystemExit) as ex:
 378             args = arvados_fuse.command.ArgumentParser().parse_args([
 379                 "--project", "zzzzz-j7d0g-zzzzzzzzzzzzzzz", self.mntdir])
 380             arvados_fuse.command.Mount(args, logger=self.logger).run()
 381         self.assertEqual(1, ex.exception.code)