services/fuse/tests/test_mount.py

   1 import unittest
   2 import arvados
   3 import arvados_fuse as fuse
   4 import threading
   5 import time
   6 import os
   7 import llfuse
   8 import tempfile
   9 import shutil
  10 import subprocess
  11 import glob
  12 import run_test_server
  13 import json
  14
  15 class MountTestBase(unittest.TestCase):
  16     def setUp(self):
  17         self.keeptmp = tempfile.mkdtemp()
  18         os.environ['KEEP_LOCAL_STORE'] = self.keeptmp
  19         self.mounttmp = tempfile.mkdtemp()
  20
  21     def tearDown(self):
  22         # llfuse.close is buggy, so use fusermount instead.
  23         #llfuse.close(unmount=True)
  24         count = 0
  25         success = 1
  26         while (count < 9 and success != 0):
  27           success = subprocess.call(["fusermount", "-u", self.mounttmp])
  28           time.sleep(0.5)
  29           count += 1
  30
  31         os.rmdir(self.mounttmp)
  32         shutil.rmtree(self.keeptmp)
  33
  34
  35 class FuseMountTest(MountTestBase):
  36     def setUp(self):
  37         super(FuseMountTest, self).setUp()
  38
  39         cw = arvados.CollectionWriter()
  40
  41         cw.start_new_file('thing1.txt')
  42         cw.write("data 1")
  43         cw.start_new_file('thing2.txt')
  44         cw.write("data 2")
  45         cw.start_new_stream('dir1')
  46
  47         cw.start_new_file('thing3.txt')
  48         cw.write("data 3")
  49         cw.start_new_file('thing4.txt')
  50         cw.write("data 4")
  51
  52         cw.start_new_stream('dir2')
  53         cw.start_new_file('thing5.txt')
  54         cw.write("data 5")
  55         cw.start_new_file('thing6.txt')
  56         cw.write("data 6")
  57
  58         cw.start_new_stream('dir2/dir3')
  59         cw.start_new_file('thing7.txt')
  60         cw.write("data 7")
  61
  62         cw.start_new_file('thing8.txt')
  63         cw.write("data 8")
  64
  65         self.testcollection = cw.finish()
  66
  67     def runTest(self):
  68         # Create the request handler
  69         operations = fuse.Operations(os.getuid(), os.getgid())
  70         e = operations.inodes.add_entry(fuse.CollectionDirectory(llfuse.ROOT_INODE, operations.inodes, self.testcollection))
  71
  72         llfuse.init(operations, self.mounttmp, [])
  73         t = threading.Thread(None, lambda: llfuse.main())
  74         t.start()
  75
  76         # wait until the driver is finished initializing
  77         operations.initlock.wait()
  78
  79         # now check some stuff
  80         d1 = os.listdir(self.mounttmp)
  81         d1.sort()
  82         self.assertEqual(['dir1', 'dir2', 'thing1.txt', 'thing2.txt'], d1)
  83
  84         d2 = os.listdir(os.path.join(self.mounttmp, 'dir1'))
  85         d2.sort()
  86         self.assertEqual(['thing3.txt', 'thing4.txt'], d2)
  87
  88         d3 = os.listdir(os.path.join(self.mounttmp, 'dir2'))
  89         d3.sort()
  90         self.assertEqual(['dir3', 'thing5.txt', 'thing6.txt'], d3)
  91
  92         d4 = os.listdir(os.path.join(self.mounttmp, 'dir2/dir3'))
  93         d4.sort()
  94         self.assertEqual(['thing7.txt', 'thing8.txt'], d4)
  95
  96         files = {'thing1.txt': 'data 1',
  97                  'thing2.txt': 'data 2',
  98                  'dir1/thing3.txt': 'data 3',
  99                  'dir1/thing4.txt': 'data 4',
 100                  'dir2/thing5.txt': 'data 5',
 101                  'dir2/thing6.txt': 'data 6',
 102                  'dir2/dir3/thing7.txt': 'data 7',
 103                  'dir2/dir3/thing8.txt': 'data 8'}
 104
 105         for k, v in files.items():
 106             with open(os.path.join(self.mounttmp, k)) as f:
 107                 self.assertEqual(v, f.read())
 108
 109
 110 class FuseMagicTest(MountTestBase):
 111     def setUp(self):
 112         super(FuseMagicTest, self).setUp()
 113
 114         cw = arvados.CollectionWriter()
 115
 116         cw.start_new_file('thing1.txt')
 117         cw.write("data 1")
 118
 119         self.testcollection = cw.finish()
 120
 121     def runTest(self):
 122         # Create the request handler
 123         operations = fuse.Operations(os.getuid(), os.getgid())
 124         e = operations.inodes.add_entry(fuse.MagicDirectory(llfuse.ROOT_INODE, operations.inodes))
 125
 126         self.mounttmp = tempfile.mkdtemp()
 127
 128         llfuse.init(operations, self.mounttmp, [])
 129         t = threading.Thread(None, lambda: llfuse.main())
 130         t.start()
 131
 132         # wait until the driver is finished initializing
 133         operations.initlock.wait()
 134
 135         # now check some stuff
 136         d1 = os.listdir(self.mounttmp)
 137         d1.sort()
 138         self.assertEqual([], d1)
 139
 140         d2 = os.listdir(os.path.join(self.mounttmp, self.testcollection))
 141         d2.sort()
 142         self.assertEqual(['thing1.txt'], d2)
 143
 144         d3 = os.listdir(self.mounttmp)
 145         d3.sort()
 146         self.assertEqual([self.testcollection], d3)
 147
 148         files = {}
 149         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
 150
 151         for k, v in files.items():
 152             with open(os.path.join(self.mounttmp, k)) as f:
 153                 self.assertEqual(v, f.read())
 154
 155
 156 class FuseTagsTest(MountTestBase):
 157     def setUp(self):
 158         super(FuseTagsTest, self).setUp()
 159
 160         cw = arvados.CollectionWriter()
 161
 162         cw.start_new_file('foo')
 163         cw.write("foo")
 164
 165         self.testcollection = cw.finish()
 166
 167         run_test_server.run()
 168
 169     def runTest(self):
 170         run_test_server.authorize_with("admin")
 171         api = arvados.api('v1', cache=False)
 172
 173         operations = fuse.Operations(os.getuid(), os.getgid())
 174         e = operations.inodes.add_entry(fuse.TagsDirectory(llfuse.ROOT_INODE, operations.inodes, api))
 175
 176         llfuse.init(operations, self.mounttmp, [])
 177         t = threading.Thread(None, lambda: llfuse.main())
 178         t.start()
 179
 180         # wait until the driver is finished initializing
 181         operations.initlock.wait()
 182
 183         d1 = os.listdir(self.mounttmp)
 184         d1.sort()
 185         self.assertEqual(['foo_tag'], d1)
 186
 187         d2 = os.listdir(os.path.join(self.mounttmp, 'foo_tag'))
 188         d2.sort()
 189         self.assertEqual(['1f4b0bc7583c2a7f9102c395f4ffc5e3+45'], d2)
 190
 191         d3 = os.listdir(os.path.join(self.mounttmp, 'foo_tag', '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'))
 192         d3.sort()
 193         self.assertEqual(['foo'], d3)
 194
 195         files = {}
 196         files[os.path.join(self.mounttmp, 'foo_tag', '1f4b0bc7583c2a7f9102c395f4ffc5e3+45', 'foo')] = 'foo'
 197
 198         for k, v in files.items():
 199             with open(os.path.join(self.mounttmp, k)) as f:
 200                 self.assertEqual(v, f.read())
 201
 202
 203     def tearDown(self):
 204         run_test_server.stop()
 205
 206         super(FuseTagsTest, self).tearDown()
 207
 208 class FuseTagsUpdateTestBase(MountTestBase):
 209
 210     def runRealTest(self):
 211         run_test_server.authorize_with("admin")
 212         api = arvados.api('v1', cache=False)
 213
 214         operations = fuse.Operations(os.getuid(), os.getgid())
 215         e = operations.inodes.add_entry(fuse.TagsDirectory(llfuse.ROOT_INODE, operations.inodes, api, poll_time=1))
 216
 217         llfuse.init(operations, self.mounttmp, [])
 218         t = threading.Thread(None, lambda: llfuse.main())
 219         t.start()
 220
 221         # wait until the driver is finished initializing
 222         operations.initlock.wait()
 223
 224         d1 = os.listdir(self.mounttmp)
 225         d1.sort()
 226         self.assertEqual(['foo_tag'], d1)
 227
 228         api.links().create(body={'link': {
 229             'head_uuid': 'fa7aeb5140e2848d39b416daeef4ffc5+45',
 230             'link_class': 'tag',
 231             'name': 'bar_tag'
 232         }}).execute()
 233
 234         time.sleep(1)
 235
 236         d2 = os.listdir(self.mounttmp)
 237         d2.sort()
 238         self.assertEqual(['bar_tag', 'foo_tag'], d2)
 239
 240         d3 = os.listdir(os.path.join(self.mounttmp, 'bar_tag'))
 241         d3.sort()
 242         self.assertEqual(['fa7aeb5140e2848d39b416daeef4ffc5+45'], d3)
 243
 244         l = api.links().create(body={'link': {
 245             'head_uuid': 'ea10d51bcf88862dbcc36eb292017dfd+45',
 246             'link_class': 'tag',
 247             'name': 'bar_tag'
 248         }}).execute()
 249
 250         time.sleep(1)
 251
 252         d4 = os.listdir(os.path.join(self.mounttmp, 'bar_tag'))
 253         d4.sort()
 254         self.assertEqual(['ea10d51bcf88862dbcc36eb292017dfd+45', 'fa7aeb5140e2848d39b416daeef4ffc5+45'], d4)
 255
 256         api.links().delete(uuid=l['uuid']).execute()
 257
 258         time.sleep(1)
 259
 260         d5 = os.listdir(os.path.join(self.mounttmp, 'bar_tag'))
 261         d5.sort()
 262         self.assertEqual(['fa7aeb5140e2848d39b416daeef4ffc5+45'], d5)
 263
 264
 265 class FuseTagsUpdateTestWebsockets(FuseTagsUpdateTestBase):
 266     def setUp(self):
 267         super(FuseTagsUpdateTestWebsockets, self).setUp()
 268         run_test_server.run(True)
 269
 270     def runTest(self):
 271         self.runRealTest()
 272
 273     def tearDown(self):
 274         run_test_server.stop()
 275         super(FuseTagsUpdateTestWebsockets, self).tearDown()
 276
 277
 278 class FuseTagsUpdateTestPoll(FuseTagsUpdateTestBase):
 279     def setUp(self):
 280         super(FuseTagsUpdateTestPoll, self).setUp()
 281         run_test_server.run(False)
 282
 283     def runTest(self):
 284         self.runRealTest()
 285
 286     def tearDown(self):
 287         run_test_server.stop()
 288         super(FuseTagsUpdateTestPoll, self).tearDown()
 289
 290
 291 class FuseGroupsTest(MountTestBase):
 292     def setUp(self):
 293         super(FuseGroupsTest, self).setUp()
 294         run_test_server.run()
 295
 296     def runTest(self):
 297         run_test_server.authorize_with("admin")
 298         api = arvados.api('v1', cache=False)
 299
 300         operations = fuse.Operations(os.getuid(), os.getgid())
 301         e = operations.inodes.add_entry(fuse.GroupsDirectory(llfuse.ROOT_INODE, operations.inodes, api))
 302
 303         llfuse.init(operations, self.mounttmp, [])
 304         t = threading.Thread(None, lambda: llfuse.main())
 305         t.start()
 306
 307         # wait until the driver is finished initializing
 308         operations.initlock.wait()
 309
 310         d1 = os.listdir(self.mounttmp)
 311         d1.sort()
 312         self.assertIn('zzzzz-j7d0g-v955i6s2oi1cbso', d1)
 313
 314         d2 = os.listdir(os.path.join(self.mounttmp, 'zzzzz-j7d0g-v955i6s2oi1cbso'))
 315         d2.sort()
 316         self.assertEqual(['1f4b0bc7583c2a7f9102c395f4ffc5e3+45 added sometime',
 317                           "I'm a job in a folder",
 318                           "I'm a template in a folder",
 319                           "zzzzz-j58dm-5gid26432uujf79",
 320                           "zzzzz-j58dm-7r18rnd5nzhg5yk",
 321                           "zzzzz-j58dm-ypsjlol9dofwijz",
 322                           "zzzzz-j7d0g-axqo7eu9pwvna1x"
 323                       ], d2)
 324
 325         d3 = os.listdir(os.path.join(self.mounttmp, 'zzzzz-j7d0g-v955i6s2oi1cbso', 'zzzzz-j7d0g-axqo7eu9pwvna1x'))
 326         d3.sort()
 327         self.assertEqual(["I'm in a subfolder, too",
 328                           "zzzzz-j58dm-c40lddwcqqr1ffs"
 329                       ], d3)
 330
 331         with open(os.path.join(self.mounttmp, 'zzzzz-j7d0g-v955i6s2oi1cbso', "I'm a template in a folder")) as f:
 332             j = json.load(f)
 333             self.assertEqual("Two Part Pipeline Template", j['name'])
 334
 335     def tearDown(self):
 336         run_test_server.stop()
 337         super(FuseGroupsTest, self).tearDown()