services/fuse/tests/test_command_args.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: AGPL-3.0
   4
   5 from __future__ import absolute_import
   6 from __future__ import print_function
   7 from six import assertRegex
   8 import arvados
   9 import arvados_fuse
  10 import arvados_fuse.command
  11 import contextlib
  12 import functools
  13 import io
  14 import json
  15 import llfuse
  16 import logging
  17 import os
  18 from . import run_test_server
  19 import sys
  20 import tempfile
  21 import unittest
  22 import resource
  23
  24 from unittest import mock
  25
  26 def noexit(func):
  27     """If argparse or arvados_fuse tries to exit, fail the test instead"""
  28     class SystemExitCaught(Exception):
  29         pass
  30     @functools.wraps(func)
  31     def wrapper(*args, **kwargs):
  32         try:
  33             return func(*args, **kwargs)
  34         except SystemExit:
  35             raise SystemExitCaught
  36     return wrapper
  37
  38 @contextlib.contextmanager
  39 def nostderr():
  40     with open(os.devnull, 'w') as dn:
  41         orig, sys.stderr = sys.stderr, dn
  42         try:
  43             yield
  44         finally:
  45             sys.stderr = orig
  46
  47
  48 class MountArgsTest(unittest.TestCase):
  49     def setUp(self):
  50         self.mntdir = tempfile.mkdtemp()
  51         run_test_server.authorize_with('active')
  52
  53     def tearDown(self):
  54         os.rmdir(self.mntdir)
  55
  56     def lookup(self, mnt, *path):
  57         ent = mnt.operations.inodes[llfuse.ROOT_INODE]
  58         for p in path:
  59             ent = ent[p]
  60         return ent
  61
  62     @contextlib.contextmanager
  63     def stderrMatches(self, stderr):
  64         orig, sys.stderr = sys.stderr, stderr
  65         try:
  66             yield
  67         finally:
  68             sys.stderr = orig
  69
  70     def check_ent_type(self, cls, *path):
  71         ent = self.lookup(self.mnt, *path)
  72         self.assertEqual(ent.__class__, cls)
  73         return ent
  74
  75     @noexit
  76     def test_default_all(self):
  77         args = arvados_fuse.command.ArgumentParser().parse_args([
  78             '--foreground', self.mntdir])
  79         self.assertEqual(args.mode, None)
  80         self.mnt = arvados_fuse.command.Mount(args)
  81         e = self.check_ent_type(arvados_fuse.ProjectDirectory, 'home')
  82         self.assertEqual(e.project_object['uuid'],
  83                          run_test_server.fixture('users')['active']['uuid'])
  84         e = self.check_ent_type(arvados_fuse.MagicDirectory, 'by_id')
  85
  86         e = self.check_ent_type(arvados_fuse.StringFile, 'README')
  87         readme = e.readfrom(0, -1).decode()
  88         assertRegex(self, readme, r'active-user@arvados\.local')
  89         assertRegex(self, readme, r'\n$')
  90
  91         e = self.check_ent_type(arvados_fuse.StringFile, 'by_id', 'README')
  92         txt = e.readfrom(0, -1).decode()
  93         assertRegex(self, txt, r'portable data hash')
  94         assertRegex(self, txt, r'\n$')
  95
  96     @noexit
  97     def test_by_id(self):
  98         args = arvados_fuse.command.ArgumentParser().parse_args([
  99             '--by-id',
 100             '--foreground', self.mntdir])
 101         self.assertEqual(args.mode, 'by_id')
 102         self.mnt = arvados_fuse.command.Mount(args)
 103         e = self.check_ent_type(arvados_fuse.MagicDirectory)
 104         self.assertEqual(e.pdh_only, False)
 105         self.assertEqual(True, self.mnt.listen_for_events)
 106
 107     @noexit
 108     def test_by_pdh(self):
 109         args = arvados_fuse.command.ArgumentParser().parse_args([
 110             '--by-pdh',
 111             '--foreground', self.mntdir])
 112         self.assertEqual(args.mode, 'by_pdh')
 113         self.mnt = arvados_fuse.command.Mount(args)
 114         e = self.check_ent_type(arvados_fuse.MagicDirectory)
 115         self.assertEqual(e.pdh_only, True)
 116         self.assertEqual(False, self.mnt.listen_for_events)
 117
 118     @noexit
 119     def test_by_tag(self):
 120         args = arvados_fuse.command.ArgumentParser().parse_args([
 121             '--by-tag',
 122             '--foreground', self.mntdir])
 123         self.assertEqual(args.mode, 'by_tag')
 124         self.mnt = arvados_fuse.command.Mount(args)
 125         e = self.check_ent_type(arvados_fuse.TagsDirectory)
 126         self.assertEqual(True, self.mnt.listen_for_events)
 127
 128     @noexit
 129     def test_collection(self, id_type='uuid'):
 130         c = run_test_server.fixture('collections')['public_text_file']
 131         cid = c[id_type]
 132         args = arvados_fuse.command.ArgumentParser().parse_args([
 133             '--collection', cid,
 134             '--foreground', self.mntdir])
 135         self.mnt = arvados_fuse.command.Mount(args)
 136         e = self.check_ent_type(arvados_fuse.CollectionDirectory)
 137         self.assertEqual(e.collection_locator, cid)
 138         self.assertEqual(id_type == 'uuid', self.mnt.listen_for_events)
 139
 140     def test_collection_pdh(self):
 141         self.test_collection('portable_data_hash')
 142
 143     @noexit
 144     def test_home(self):
 145         args = arvados_fuse.command.ArgumentParser().parse_args([
 146             '--home',
 147             '--foreground', self.mntdir])
 148         self.assertEqual(args.mode, 'home')
 149         self.mnt = arvados_fuse.command.Mount(args)
 150         e = self.check_ent_type(arvados_fuse.ProjectDirectory)
 151         self.assertEqual(e.project_object['uuid'],
 152                          run_test_server.fixture('users')['active']['uuid'])
 153         self.assertEqual(True, self.mnt.listen_for_events)
 154
 155     def test_mutually_exclusive_args(self):
 156         cid = run_test_server.fixture('collections')['public_text_file']['uuid']
 157         gid = run_test_server.fixture('groups')['aproject']['uuid']
 158         for badargs in [
 159                 ['--mount-tmp', 'foo', '--collection', cid],
 160                 ['--mount-tmp', 'foo', '--project', gid],
 161                 ['--collection', cid, '--project', gid],
 162                 ['--by-id', '--project', gid],
 163                 ['--mount-tmp', 'foo', '--by-id'],
 164         ]:
 165             with nostderr():
 166                 with self.assertRaises(SystemExit):
 167                     args = arvados_fuse.command.ArgumentParser().parse_args(
 168                         badargs + ['--foreground', self.mntdir])
 169                     arvados_fuse.command.Mount(args)
 170     @noexit
 171     def test_project(self):
 172         uuid = run_test_server.fixture('groups')['aproject']['uuid']
 173         args = arvados_fuse.command.ArgumentParser().parse_args([
 174             '--project', uuid,
 175             '--foreground', self.mntdir])
 176         self.mnt = arvados_fuse.command.Mount(args)
 177         e = self.check_ent_type(arvados_fuse.ProjectDirectory)
 178         self.assertEqual(e.project_object['uuid'], uuid)
 179
 180     @noexit
 181     def test_shared(self):
 182         args = arvados_fuse.command.ArgumentParser().parse_args([
 183             '--shared',
 184             '--foreground', self.mntdir])
 185         self.assertEqual(args.mode, 'shared')
 186         self.mnt = arvados_fuse.command.Mount(args)
 187         e = self.check_ent_type(arvados_fuse.SharedDirectory)
 188         self.assertEqual(e.current_user['uuid'],
 189                          run_test_server.fixture('users')['active']['uuid'])
 190         self.assertEqual(True, self.mnt.listen_for_events)
 191
 192     def test_version_argument(self):
 193         # The argparse version action prints to stderr in Python 2 and stdout
 194         # in Python 3.4 and up. Write both to the same stream so the test can pass
 195         # in both cases.
 196         origerr = sys.stderr
 197         origout = sys.stdout
 198         sys.stderr = io.StringIO()
 199         sys.stdout = sys.stderr
 200
 201         with self.assertRaises(SystemExit):
 202             args = arvados_fuse.command.ArgumentParser().parse_args(['--version'])
 203         assertRegex(self, sys.stdout.getvalue(), "[0-9]+\.[0-9]+\.[0-9]+")
 204         sys.stderr.close()
 205         sys.stderr = origerr
 206         sys.stdout = origout
 207
 208     @noexit
 209     @mock.patch('arvados.events.subscribe')
 210     def test_disable_event_listening(self, mock_subscribe):
 211         args = arvados_fuse.command.ArgumentParser().parse_args([
 212             '--disable-event-listening',
 213             '--by-id',
 214             '--foreground', self.mntdir])
 215         self.mnt = arvados_fuse.command.Mount(args)
 216         self.assertEqual(True, self.mnt.listen_for_events)
 217         self.assertEqual(True, self.mnt.args.disable_event_listening)
 218         with self.mnt:
 219             pass
 220         self.assertEqual(0, mock_subscribe.call_count)
 221
 222     @noexit
 223     @mock.patch('arvados.events.subscribe')
 224     def test_custom(self, mock_subscribe):
 225         args = arvados_fuse.command.ArgumentParser().parse_args([
 226             '--mount-tmp', 'foo',
 227             '--mount-tmp', 'bar',
 228             '--mount-home', 'my_home',
 229             '--foreground', self.mntdir])
 230         self.assertEqual(args.mode, None)
 231         self.mnt = arvados_fuse.command.Mount(args)
 232         self.check_ent_type(arvados_fuse.Directory)
 233         self.check_ent_type(arvados_fuse.TmpCollectionDirectory, 'foo')
 234         self.check_ent_type(arvados_fuse.TmpCollectionDirectory, 'bar')
 235         e = self.check_ent_type(arvados_fuse.ProjectDirectory, 'my_home')
 236         self.assertEqual(e.project_object['uuid'],
 237                          run_test_server.fixture('users')['active']['uuid'])
 238         self.assertEqual(True, self.mnt.listen_for_events)
 239         with self.mnt:
 240             pass
 241         self.assertEqual(1, mock_subscribe.call_count)
 242
 243     @noexit
 244     @mock.patch('arvados.events.subscribe')
 245     def test_custom_no_listen(self, mock_subscribe):
 246         args = arvados_fuse.command.ArgumentParser().parse_args([
 247             '--mount-by-pdh', 'pdh',
 248             '--mount-tmp', 'foo',
 249             '--mount-tmp', 'bar',
 250             '--foreground', self.mntdir])
 251         self.mnt = arvados_fuse.command.Mount(args)
 252         self.assertEqual(False, self.mnt.listen_for_events)
 253         with self.mnt:
 254             pass
 255         self.assertEqual(0, mock_subscribe.call_count)
 256
 257     def test_custom_unsupported_layouts(self):
 258         for name in ['.', '..', '', 'foo/bar', '/foo']:
 259             with nostderr():
 260                 with self.assertRaises(SystemExit):
 261                     args = arvados_fuse.command.ArgumentParser().parse_args([
 262                         '--mount-tmp', name,
 263                         '--foreground', self.mntdir])
 264                     arvados_fuse.command.Mount(args)
 265
 266     @noexit
 267     @mock.patch('resource.setrlimit')
 268     @mock.patch('resource.getrlimit')
 269     def test_default_file_cache(self, getrlimit, setrlimit):
 270         args = arvados_fuse.command.ArgumentParser().parse_args([
 271             '--foreground', self.mntdir])
 272         self.assertEqual(args.mode, None)
 273         getrlimit.return_value = (1024, 1048576)
 274         self.mnt = arvados_fuse.command.Mount(args)
 275         setrlimit.assert_called_with(resource.RLIMIT_NOFILE, (10240, 1048576))
 276
 277     @noexit
 278     @mock.patch('resource.setrlimit')
 279     @mock.patch('resource.getrlimit')
 280     def test_small_file_cache(self, getrlimit, setrlimit):
 281         args = arvados_fuse.command.ArgumentParser().parse_args([
 282             '--foreground', '--file-cache=256000000', self.mntdir])
 283         self.assertEqual(args.mode, None)
 284         getrlimit.return_value = (1024, 1048576)
 285         self.mnt = arvados_fuse.command.Mount(args)
 286         setrlimit.assert_not_called()
 287
 288     @noexit
 289     @mock.patch('resource.setrlimit')
 290     @mock.patch('resource.getrlimit')
 291     def test_large_file_cache(self, getrlimit, setrlimit):
 292         args = arvados_fuse.command.ArgumentParser().parse_args([
 293             '--foreground', '--file-cache=256000000000', self.mntdir])
 294         self.assertEqual(args.mode, None)
 295         getrlimit.return_value = (1024, 1048576)
 296         self.mnt = arvados_fuse.command.Mount(args)
 297         setrlimit.assert_called_with(resource.RLIMIT_NOFILE, (30517, 1048576))
 298
 299     @noexit
 300     @mock.patch('resource.setrlimit')
 301     @mock.patch('resource.getrlimit')
 302     def test_file_cache_hard_limit(self, getrlimit, setrlimit):
 303         args = arvados_fuse.command.ArgumentParser().parse_args([
 304             '--foreground', '--file-cache=256000000000', self.mntdir])
 305         self.assertEqual(args.mode, None)
 306         getrlimit.return_value = (1024, 2048)
 307         self.mnt = arvados_fuse.command.Mount(args)
 308         setrlimit.assert_called_with(resource.RLIMIT_NOFILE, (2048, 2048))
 309
 310 class MountErrorTest(unittest.TestCase):
 311     def setUp(self):
 312         self.mntdir = tempfile.mkdtemp()
 313         run_test_server.run()
 314         run_test_server.authorize_with("active")
 315         self.logger = logging.getLogger("null")
 316         self.logger.setLevel(logging.CRITICAL+1)
 317
 318     def tearDown(self):
 319         if os.path.exists(self.mntdir):
 320             # If the directory was not unmounted, this will raise an exception.
 321             os.rmdir(self.mntdir)
 322         run_test_server.reset()
 323
 324     def test_no_token(self):
 325         del arvados.config._settings["ARVADOS_API_TOKEN"]
 326         arvados.config._settings = {}
 327         with self.assertRaises(SystemExit) as ex:
 328             args = arvados_fuse.command.ArgumentParser().parse_args([self.mntdir])
 329             arvados_fuse.command.Mount(args, logger=self.logger).run()
 330         self.assertEqual(1, ex.exception.code)
 331
 332     def test_no_host(self):
 333         del arvados.config._settings["ARVADOS_API_HOST"]
 334         with self.assertRaises(SystemExit) as ex:
 335             args = arvados_fuse.command.ArgumentParser().parse_args([self.mntdir])
 336             arvados_fuse.command.Mount(args, logger=self.logger).run()
 337         self.assertEqual(1, ex.exception.code)
 338
 339     def test_bogus_host(self):
 340         arvados.config._settings["ARVADOS_API_HOST"] = "100::"
 341         with self.assertRaises(SystemExit) as ex, mock.patch('time.sleep'):
 342             args = arvados_fuse.command.ArgumentParser().parse_args([self.mntdir])
 343             arvados_fuse.command.Mount(args, logger=self.logger).run()
 344         self.assertEqual(1, ex.exception.code)
 345
 346     def test_bogus_token(self):
 347         arvados.config._settings["ARVADOS_API_TOKEN"] = "zzzzzzzzzzzzz"
 348         with self.assertRaises(SystemExit) as ex:
 349             args = arvados_fuse.command.ArgumentParser().parse_args([self.mntdir])
 350             arvados_fuse.command.Mount(args, logger=self.logger).run()
 351         self.assertEqual(1, ex.exception.code)
 352
 353     def test_bogus_mount_dir(self):
 354         # All FUSE errors in llfuse.init() are raised as RuntimeError
 355         # An easy error to trigger is to supply a nonexistent mount point,
 356         # so test that one.
 357         #
 358         # Other possible errors that also raise RuntimeError (but are much
 359         # harder to test automatically because they depend on operating
 360         # system configuration):
 361         #
 362         # The user doesn't have permission to use FUSE
 363         # The user specified --allow-other but user_allow_other is not set
 364         # in /etc/fuse.conf
 365         os.rmdir(self.mntdir)
 366         with self.assertRaises(SystemExit) as ex:
 367             args = arvados_fuse.command.ArgumentParser().parse_args([self.mntdir])
 368             arvados_fuse.command.Mount(args, logger=self.logger).run()
 369         self.assertEqual(1, ex.exception.code)
 370
 371     def test_unreadable_collection(self):
 372         with self.assertRaises(SystemExit) as ex:
 373             args = arvados_fuse.command.ArgumentParser().parse_args([
 374                 "--collection", "zzzzz-4zz18-zzzzzzzzzzzzzzz", self.mntdir])
 375             arvados_fuse.command.Mount(args, logger=self.logger).run()
 376         self.assertEqual(1, ex.exception.code)
 377
 378     def test_unreadable_project(self):
 379         with self.assertRaises(SystemExit) as ex:
 380             args = arvados_fuse.command.ArgumentParser().parse_args([
 381                 "--project", "zzzzz-j7d0g-zzzzzzzzzzzzzzz", self.mntdir])
 382             arvados_fuse.command.Mount(args, logger=self.logger).run()
 383         self.assertEqual(1, ex.exception.code)