services/fuse/tests/test_command_args.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: AGPL-3.0
   4
   5 from __future__ import absolute_import
   6 from __future__ import print_function
   7 from six import assertRegex
   8 import arvados
   9 import arvados_fuse
  10 import arvados_fuse.command
  11 import contextlib
  12 import functools
  13 import io
  14 import json
  15 import llfuse
  16 import logging
  17 import mock
  18 import os
  19 from . import run_test_server
  20 import sys
  21 import tempfile
  22 import unittest
  23 import resource
  24
  25 def noexit(func):
  26     """If argparse or arvados_fuse tries to exit, fail the test instead"""
  27     class SystemExitCaught(Exception):
  28         pass
  29     @functools.wraps(func)
  30     def wrapper(*args, **kwargs):
  31         try:
  32             return func(*args, **kwargs)
  33         except SystemExit:
  34             raise SystemExitCaught
  35     return wrapper
  36
  37 @contextlib.contextmanager
  38 def nostderr():
  39     with open(os.devnull, 'w') as dn:
  40         orig, sys.stderr = sys.stderr, dn
  41         try:
  42             yield
  43         finally:
  44             sys.stderr = orig
  45
  46
  47 class MountArgsTest(unittest.TestCase):
  48     def setUp(self):
  49         self.mntdir = tempfile.mkdtemp()
  50         run_test_server.authorize_with('active')
  51
  52     def tearDown(self):
  53         os.rmdir(self.mntdir)
  54
  55     def lookup(self, mnt, *path):
  56         ent = mnt.operations.inodes[llfuse.ROOT_INODE]
  57         for p in path:
  58             ent = ent[p]
  59         return ent
  60
  61     @contextlib.contextmanager
  62     def stderrMatches(self, stderr):
  63         orig, sys.stderr = sys.stderr, stderr
  64         try:
  65             yield
  66         finally:
  67             sys.stderr = orig
  68
  69     def check_ent_type(self, cls, *path):
  70         ent = self.lookup(self.mnt, *path)
  71         self.assertEqual(ent.__class__, cls)
  72         return ent
  73
  74     @noexit
  75     def test_default_all(self):
  76         args = arvados_fuse.command.ArgumentParser().parse_args([
  77             '--foreground', self.mntdir])
  78         self.assertEqual(args.mode, None)
  79         self.mnt = arvados_fuse.command.Mount(args)
  80         e = self.check_ent_type(arvados_fuse.ProjectDirectory, 'home')
  81         self.assertEqual(e.project_object['uuid'],
  82                          run_test_server.fixture('users')['active']['uuid'])
  83         e = self.check_ent_type(arvados_fuse.MagicDirectory, 'by_id')
  84
  85         e = self.check_ent_type(arvados_fuse.StringFile, 'README')
  86         readme = e.readfrom(0, -1).decode()
  87         assertRegex(self, readme, r'active-user@arvados\.local')
  88         assertRegex(self, readme, r'\n$')
  89
  90         e = self.check_ent_type(arvados_fuse.StringFile, 'by_id', 'README')
  91         txt = e.readfrom(0, -1).decode()
  92         assertRegex(self, txt, r'portable data hash')
  93         assertRegex(self, txt, r'\n$')
  94
  95     @noexit
  96     def test_by_id(self):
  97         args = arvados_fuse.command.ArgumentParser().parse_args([
  98             '--by-id',
  99             '--foreground', self.mntdir])
 100         self.assertEqual(args.mode, 'by_id')
 101         self.mnt = arvados_fuse.command.Mount(args)
 102         e = self.check_ent_type(arvados_fuse.MagicDirectory)
 103         self.assertEqual(e.pdh_only, False)
 104         self.assertEqual(True, self.mnt.listen_for_events)
 105
 106     @noexit
 107     def test_by_pdh(self):
 108         args = arvados_fuse.command.ArgumentParser().parse_args([
 109             '--by-pdh',
 110             '--foreground', self.mntdir])
 111         self.assertEqual(args.mode, 'by_pdh')
 112         self.mnt = arvados_fuse.command.Mount(args)
 113         e = self.check_ent_type(arvados_fuse.MagicDirectory)
 114         self.assertEqual(e.pdh_only, True)
 115         self.assertEqual(False, self.mnt.listen_for_events)
 116
 117     @noexit
 118     def test_by_tag(self):
 119         args = arvados_fuse.command.ArgumentParser().parse_args([
 120             '--by-tag',
 121             '--foreground', self.mntdir])
 122         self.assertEqual(args.mode, 'by_tag')
 123         self.mnt = arvados_fuse.command.Mount(args)
 124         e = self.check_ent_type(arvados_fuse.TagsDirectory)
 125         self.assertEqual(True, self.mnt.listen_for_events)
 126
 127     @noexit
 128     def test_collection(self, id_type='uuid'):
 129         c = run_test_server.fixture('collections')['public_text_file']
 130         cid = c[id_type]
 131         args = arvados_fuse.command.ArgumentParser().parse_args([
 132             '--collection', cid,
 133             '--foreground', self.mntdir])
 134         self.mnt = arvados_fuse.command.Mount(args)
 135         e = self.check_ent_type(arvados_fuse.CollectionDirectory)
 136         self.assertEqual(e.collection_locator, cid)
 137         self.assertEqual(id_type == 'uuid', self.mnt.listen_for_events)
 138
 139     def test_collection_pdh(self):
 140         self.test_collection('portable_data_hash')
 141
 142     @noexit
 143     def test_home(self):
 144         args = arvados_fuse.command.ArgumentParser().parse_args([
 145             '--home',
 146             '--foreground', self.mntdir])
 147         self.assertEqual(args.mode, 'home')
 148         self.mnt = arvados_fuse.command.Mount(args)
 149         e = self.check_ent_type(arvados_fuse.ProjectDirectory)
 150         self.assertEqual(e.project_object['uuid'],
 151                          run_test_server.fixture('users')['active']['uuid'])
 152         self.assertEqual(True, self.mnt.listen_for_events)
 153
 154     def test_mutually_exclusive_args(self):
 155         cid = run_test_server.fixture('collections')['public_text_file']['uuid']
 156         gid = run_test_server.fixture('groups')['aproject']['uuid']
 157         for badargs in [
 158                 ['--mount-tmp', 'foo', '--collection', cid],
 159                 ['--mount-tmp', 'foo', '--project', gid],
 160                 ['--collection', cid, '--project', gid],
 161                 ['--by-id', '--project', gid],
 162                 ['--mount-tmp', 'foo', '--by-id'],
 163         ]:
 164             with nostderr():
 165                 with self.assertRaises(SystemExit):
 166                     args = arvados_fuse.command.ArgumentParser().parse_args(
 167                         badargs + ['--foreground', self.mntdir])
 168                     arvados_fuse.command.Mount(args)
 169     @noexit
 170     def test_project(self):
 171         uuid = run_test_server.fixture('groups')['aproject']['uuid']
 172         args = arvados_fuse.command.ArgumentParser().parse_args([
 173             '--project', uuid,
 174             '--foreground', self.mntdir])
 175         self.mnt = arvados_fuse.command.Mount(args)
 176         e = self.check_ent_type(arvados_fuse.ProjectDirectory)
 177         self.assertEqual(e.project_object['uuid'], uuid)
 178
 179     @noexit
 180     def test_shared(self):
 181         args = arvados_fuse.command.ArgumentParser().parse_args([
 182             '--shared',
 183             '--foreground', self.mntdir])
 184         self.assertEqual(args.mode, 'shared')
 185         self.mnt = arvados_fuse.command.Mount(args)
 186         e = self.check_ent_type(arvados_fuse.SharedDirectory)
 187         self.assertEqual(e.current_user['uuid'],
 188                          run_test_server.fixture('users')['active']['uuid'])
 189         self.assertEqual(True, self.mnt.listen_for_events)
 190
 191     def test_version_argument(self):
 192         # The argparse version action prints to stderr in Python 2 and stdout
 193         # in Python 3.4 and up. Write both to the same stream so the test can pass
 194         # in both cases.
 195         origerr = sys.stderr
 196         origout = sys.stdout
 197         sys.stderr = io.StringIO()
 198         sys.stdout = sys.stderr
 199
 200         with self.assertRaises(SystemExit):
 201             args = arvados_fuse.command.ArgumentParser().parse_args(['--version'])
 202         assertRegex(self, sys.stdout.getvalue(), "[0-9]+\.[0-9]+\.[0-9]+")
 203         sys.stderr.close()
 204         sys.stderr = origerr
 205         sys.stdout = origout
 206
 207     @noexit
 208     @mock.patch('arvados.events.subscribe')
 209     def test_disable_event_listening(self, mock_subscribe):
 210         args = arvados_fuse.command.ArgumentParser().parse_args([
 211             '--disable-event-listening',
 212             '--by-id',
 213             '--foreground', self.mntdir])
 214         self.mnt = arvados_fuse.command.Mount(args)
 215         self.assertEqual(True, self.mnt.listen_for_events)
 216         self.assertEqual(True, self.mnt.args.disable_event_listening)
 217         with self.mnt:
 218             pass
 219         self.assertEqual(0, mock_subscribe.call_count)
 220
 221     @noexit
 222     @mock.patch('arvados.events.subscribe')
 223     def test_custom(self, mock_subscribe):
 224         args = arvados_fuse.command.ArgumentParser().parse_args([
 225             '--mount-tmp', 'foo',
 226             '--mount-tmp', 'bar',
 227             '--mount-home', 'my_home',
 228             '--foreground', self.mntdir])
 229         self.assertEqual(args.mode, None)
 230         self.mnt = arvados_fuse.command.Mount(args)
 231         self.check_ent_type(arvados_fuse.Directory)
 232         self.check_ent_type(arvados_fuse.TmpCollectionDirectory, 'foo')
 233         self.check_ent_type(arvados_fuse.TmpCollectionDirectory, 'bar')
 234         e = self.check_ent_type(arvados_fuse.ProjectDirectory, 'my_home')
 235         self.assertEqual(e.project_object['uuid'],
 236                          run_test_server.fixture('users')['active']['uuid'])
 237         self.assertEqual(True, self.mnt.listen_for_events)
 238         with self.mnt:
 239             pass
 240         self.assertEqual(1, mock_subscribe.call_count)
 241
 242     @noexit
 243     @mock.patch('arvados.events.subscribe')
 244     def test_custom_no_listen(self, mock_subscribe):
 245         args = arvados_fuse.command.ArgumentParser().parse_args([
 246             '--mount-by-pdh', 'pdh',
 247             '--mount-tmp', 'foo',
 248             '--mount-tmp', 'bar',
 249             '--foreground', self.mntdir])
 250         self.mnt = arvados_fuse.command.Mount(args)
 251         self.assertEqual(False, self.mnt.listen_for_events)
 252         with self.mnt:
 253             pass
 254         self.assertEqual(0, mock_subscribe.call_count)
 255
 256     def test_custom_unsupported_layouts(self):
 257         for name in ['.', '..', '', 'foo/bar', '/foo']:
 258             with nostderr():
 259                 with self.assertRaises(SystemExit):
 260                     args = arvados_fuse.command.ArgumentParser().parse_args([
 261                         '--mount-tmp', name,
 262                         '--foreground', self.mntdir])
 263                     arvados_fuse.command.Mount(args)
 264
 265     @noexit
 266     @mock.patch('resource.setrlimit')
 267     @mock.patch('resource.getrlimit')
 268     def test_default_file_cache(self, getrlimit, setrlimit):
 269         args = arvados_fuse.command.ArgumentParser().parse_args([
 270             '--foreground', self.mntdir])
 271         self.assertEqual(args.mode, None)
 272         getrlimit.return_value = (1024, 1048576)
 273         self.mnt = arvados_fuse.command.Mount(args)
 274         setrlimit.assert_called_with(resource.RLIMIT_NOFILE, (10240, 1048576))
 275
 276     @noexit
 277     @mock.patch('resource.setrlimit')
 278     @mock.patch('resource.getrlimit')
 279     def test_small_file_cache(self, getrlimit, setrlimit):
 280         args = arvados_fuse.command.ArgumentParser().parse_args([
 281             '--foreground', '--file-cache=256000000', self.mntdir])
 282         self.assertEqual(args.mode, None)
 283         getrlimit.return_value = (1024, 1048576)
 284         self.mnt = arvados_fuse.command.Mount(args)
 285         setrlimit.assert_not_called()
 286
 287     @noexit
 288     @mock.patch('resource.setrlimit')
 289     @mock.patch('resource.getrlimit')
 290     def test_large_file_cache(self, getrlimit, setrlimit):
 291         args = arvados_fuse.command.ArgumentParser().parse_args([
 292             '--foreground', '--file-cache=256000000000', self.mntdir])
 293         self.assertEqual(args.mode, None)
 294         getrlimit.return_value = (1024, 1048576)
 295         self.mnt = arvados_fuse.command.Mount(args)
 296         setrlimit.assert_called_with(resource.RLIMIT_NOFILE, (30517, 1048576))
 297
 298     @noexit
 299     @mock.patch('resource.setrlimit')
 300     @mock.patch('resource.getrlimit')
 301     def test_file_cache_hard_limit(self, getrlimit, setrlimit):
 302         args = arvados_fuse.command.ArgumentParser().parse_args([
 303             '--foreground', '--file-cache=256000000000', self.mntdir])
 304         self.assertEqual(args.mode, None)
 305         getrlimit.return_value = (1024, 2048)
 306         self.mnt = arvados_fuse.command.Mount(args)
 307         setrlimit.assert_called_with(resource.RLIMIT_NOFILE, (2048, 2048))
 308
 309 class MountErrorTest(unittest.TestCase):
 310     def setUp(self):
 311         self.mntdir = tempfile.mkdtemp()
 312         run_test_server.run()
 313         run_test_server.authorize_with("active")
 314         self.logger = logging.getLogger("null")
 315         self.logger.setLevel(logging.CRITICAL+1)
 316
 317     def tearDown(self):
 318         if os.path.exists(self.mntdir):
 319             # If the directory was not unmounted, this will raise an exception.
 320             os.rmdir(self.mntdir)
 321         run_test_server.reset()
 322
 323     def test_no_token(self):
 324         del arvados.config._settings["ARVADOS_API_TOKEN"]
 325         arvados.config._settings = {}
 326         with self.assertRaises(SystemExit) as ex:
 327             args = arvados_fuse.command.ArgumentParser().parse_args([self.mntdir])
 328             arvados_fuse.command.Mount(args, logger=self.logger).run()
 329         self.assertEqual(1, ex.exception.code)
 330
 331     def test_no_host(self):
 332         del arvados.config._settings["ARVADOS_API_HOST"]
 333         with self.assertRaises(SystemExit) as ex:
 334             args = arvados_fuse.command.ArgumentParser().parse_args([self.mntdir])
 335             arvados_fuse.command.Mount(args, logger=self.logger).run()
 336         self.assertEqual(1, ex.exception.code)
 337
 338     def test_bogus_host(self):
 339         arvados.config._settings["ARVADOS_API_HOST"] = "100::"
 340         with self.assertRaises(SystemExit) as ex, mock.patch('time.sleep'):
 341             args = arvados_fuse.command.ArgumentParser().parse_args([self.mntdir])
 342             arvados_fuse.command.Mount(args, logger=self.logger).run()
 343         self.assertEqual(1, ex.exception.code)
 344
 345     def test_bogus_token(self):
 346         arvados.config._settings["ARVADOS_API_TOKEN"] = "zzzzzzzzzzzzz"
 347         with self.assertRaises(SystemExit) as ex:
 348             args = arvados_fuse.command.ArgumentParser().parse_args([self.mntdir])
 349             arvados_fuse.command.Mount(args, logger=self.logger).run()
 350         self.assertEqual(1, ex.exception.code)
 351
 352     def test_bogus_mount_dir(self):
 353         # All FUSE errors in llfuse.init() are raised as RuntimeError
 354         # An easy error to trigger is to supply a nonexistent mount point,
 355         # so test that one.
 356         #
 357         # Other possible errors that also raise RuntimeError (but are much
 358         # harder to test automatically because they depend on operating
 359         # system configuration):
 360         #
 361         # The user doesn't have permission to use FUSE
 362         # The user specified --allow-other but user_allow_other is not set
 363         # in /etc/fuse.conf
 364         os.rmdir(self.mntdir)
 365         with self.assertRaises(SystemExit) as ex:
 366             args = arvados_fuse.command.ArgumentParser().parse_args([self.mntdir])
 367             arvados_fuse.command.Mount(args, logger=self.logger).run()
 368         self.assertEqual(1, ex.exception.code)
 369
 370     def test_unreadable_collection(self):
 371         with self.assertRaises(SystemExit) as ex:
 372             args = arvados_fuse.command.ArgumentParser().parse_args([
 373                 "--collection", "zzzzz-4zz18-zzzzzzzzzzzzzzz", self.mntdir])
 374             arvados_fuse.command.Mount(args, logger=self.logger).run()
 375         self.assertEqual(1, ex.exception.code)
 376
 377     def test_unreadable_project(self):
 378         with self.assertRaises(SystemExit) as ex:
 379             args = arvados_fuse.command.ArgumentParser().parse_args([
 380                 "--project", "zzzzz-j7d0g-zzzzzzzzzzzzzzz", self.mntdir])
 381             arvados_fuse.command.Mount(args, logger=self.logger).run()
 382         self.assertEqual(1, ex.exception.code)