services/fuse/tests/test_mount.py

   1 import unittest
   2 import arvados
   3 import arvados_fuse as fuse
   4 import threading
   5 import time
   6 import os
   7 import llfuse
   8 import tempfile
   9 import shutil
  10 import subprocess
  11 import glob
  12 import run_test_server
  13 import json
  14
  15 class MountTestBase(unittest.TestCase):
  16     def setUp(self):
  17         self.keeptmp = tempfile.mkdtemp()
  18         os.environ['KEEP_LOCAL_STORE'] = self.keeptmp
  19         self.mounttmp = tempfile.mkdtemp()
  20
  21     def tearDown(self):
  22         # llfuse.close is buggy, so use fusermount instead.
  23         #llfuse.close(unmount=True)
  24         count = 0
  25         success = 1
  26         while (count < 9 and success != 0):
  27           success = subprocess.call(["fusermount", "-u", self.mounttmp])
  28           time.sleep(0.5)
  29           count += 1
  30
  31         os.rmdir(self.mounttmp)
  32         shutil.rmtree(self.keeptmp)
  33
  34
  35 class FuseMountTest(MountTestBase):
  36     def setUp(self):
  37         super(FuseMountTest, self).setUp()
  38
  39         cw = arvados.CollectionWriter()
  40
  41         cw.start_new_file('thing1.txt')
  42         cw.write("data 1")
  43         cw.start_new_file('thing2.txt')
  44         cw.write("data 2")
  45         cw.start_new_stream('dir1')
  46
  47         cw.start_new_file('thing3.txt')
  48         cw.write("data 3")
  49         cw.start_new_file('thing4.txt')
  50         cw.write("data 4")
  51
  52         cw.start_new_stream('dir2')
  53         cw.start_new_file('thing5.txt')
  54         cw.write("data 5")
  55         cw.start_new_file('thing6.txt')
  56         cw.write("data 6")
  57
  58         cw.start_new_stream('dir2/dir3')
  59         cw.start_new_file('thing7.txt')
  60         cw.write("data 7")
  61
  62         cw.start_new_file('thing8.txt')
  63         cw.write("data 8")
  64
  65         self.testcollection = cw.finish()
  66
  67     def runTest(self):
  68         # Create the request handler
  69         operations = fuse.Operations(os.getuid(), os.getgid())
  70         e = operations.inodes.add_entry(fuse.CollectionDirectory(llfuse.ROOT_INODE, operations.inodes, self.testcollection))
  71
  72         llfuse.init(operations, self.mounttmp, [])
  73         t = threading.Thread(None, lambda: llfuse.main())
  74         t.start()
  75
  76         # wait until the driver is finished initializing
  77         operations.initlock.wait()
  78
  79         # now check some stuff
  80         d1 = os.listdir(self.mounttmp)
  81         d1.sort()
  82         self.assertEqual(['dir1', 'dir2', 'thing1.txt', 'thing2.txt'], d1)
  83
  84         d2 = os.listdir(os.path.join(self.mounttmp, 'dir1'))
  85         d2.sort()
  86         self.assertEqual(['thing3.txt', 'thing4.txt'], d2)
  87
  88         d3 = os.listdir(os.path.join(self.mounttmp, 'dir2'))
  89         d3.sort()
  90         self.assertEqual(['dir3', 'thing5.txt', 'thing6.txt'], d3)
  91
  92         d4 = os.listdir(os.path.join(self.mounttmp, 'dir2/dir3'))
  93         d4.sort()
  94         self.assertEqual(['thing7.txt', 'thing8.txt'], d4)
  95
  96         files = {'thing1.txt': 'data 1',
  97                  'thing2.txt': 'data 2',
  98                  'dir1/thing3.txt': 'data 3',
  99                  'dir1/thing4.txt': 'data 4',
 100                  'dir2/thing5.txt': 'data 5',
 101                  'dir2/thing6.txt': 'data 6',
 102                  'dir2/dir3/thing7.txt': 'data 7',
 103                  'dir2/dir3/thing8.txt': 'data 8'}
 104
 105         for k, v in files.items():
 106             with open(os.path.join(self.mounttmp, k)) as f:
 107                 self.assertEqual(v, f.read())
 108
 109
 110 class FuseMagicTest(MountTestBase):
 111     def setUp(self):
 112         super(FuseMagicTest, self).setUp()
 113
 114         cw = arvados.CollectionWriter()
 115
 116         cw.start_new_file('thing1.txt')
 117         cw.write("data 1")
 118
 119         self.testcollection = cw.finish()
 120
 121     def runTest(self):
 122         # Create the request handler
 123         operations = fuse.Operations(os.getuid(), os.getgid())
 124         e = operations.inodes.add_entry(fuse.MagicDirectory(llfuse.ROOT_INODE, operations.inodes))
 125
 126         self.mounttmp = tempfile.mkdtemp()
 127
 128         llfuse.init(operations, self.mounttmp, [])
 129         t = threading.Thread(None, lambda: llfuse.main())
 130         t.start()
 131
 132         # wait until the driver is finished initializing
 133         operations.initlock.wait()
 134
 135         # now check some stuff
 136         d1 = os.listdir(self.mounttmp)
 137         d1.sort()
 138         self.assertEqual([], d1)
 139
 140         d2 = os.listdir(os.path.join(self.mounttmp, self.testcollection))
 141         d2.sort()
 142         self.assertEqual(['thing1.txt'], d2)
 143
 144         d3 = os.listdir(self.mounttmp)
 145         d3.sort()
 146         self.assertEqual([self.testcollection], d3)
 147
 148         files = {}
 149         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
 150
 151         for k, v in files.items():
 152             with open(os.path.join(self.mounttmp, k)) as f:
 153                 self.assertEqual(v, f.read())
 154
 155 #
 156 # Restore these tests when working on issue #3644
 157 #
 158 # class FuseTagsTest(MountTestBase):
 159 #     def setUp(self):
 160 #         super(FuseTagsTest, self).setUp()
 161
 162 #         cw = arvados.CollectionWriter()
 163
 164 #         cw.start_new_file('foo')
 165 #         cw.write("foo")
 166
 167 #         self.testcollection = cw.finish()
 168
 169 #         run_test_server.run()
 170
 171 #     def runTest(self):
 172 #         run_test_server.authorize_with("admin")
 173 #         api = arvados.api('v1', cache=False)
 174
 175 #         operations = fuse.Operations(os.getuid(), os.getgid())
 176 #         e = operations.inodes.add_entry(fuse.TagsDirectory(llfuse.ROOT_INODE, operations.inodes, api))
 177
 178 #         llfuse.init(operations, self.mounttmp, [])
 179 #         t = threading.Thread(None, lambda: llfuse.main())
 180 #         t.start()
 181
 182 #         # wait until the driver is finished initializing
 183 #         operations.initlock.wait()
 184
 185 #         d1 = os.listdir(self.mounttmp)
 186 #         d1.sort()
 187 #         self.assertEqual(['foo_tag'], d1)
 188
 189 #         d2 = os.listdir(os.path.join(self.mounttmp, 'foo_tag'))
 190 #         d2.sort()
 191 #         self.assertEqual(['1f4b0bc7583c2a7f9102c395f4ffc5e3+45'], d2)
 192
 193 #         d3 = os.listdir(os.path.join(self.mounttmp, 'foo_tag', '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'))
 194 #         d3.sort()
 195 #         self.assertEqual(['foo'], d3)
 196
 197 #         files = {}
 198 #         files[os.path.join(self.mounttmp, 'foo_tag', '1f4b0bc7583c2a7f9102c395f4ffc5e3+45', 'foo')] = 'foo'
 199
 200 #         for k, v in files.items():
 201 #             with open(os.path.join(self.mounttmp, k)) as f:
 202 #                 self.assertEqual(v, f.read())
 203
 204
 205 #     def tearDown(self):
 206 #         run_test_server.stop()
 207
 208 #         super(FuseTagsTest, self).tearDown()
 209
 210 # class FuseTagsUpdateTestBase(MountTestBase):
 211
 212 #     def runRealTest(self):
 213 #         run_test_server.authorize_with("admin")
 214 #         api = arvados.api('v1', cache=False)
 215
 216 #         operations = fuse.Operations(os.getuid(), os.getgid())
 217 #         e = operations.inodes.add_entry(fuse.TagsDirectory(llfuse.ROOT_INODE, operations.inodes, api, poll_time=1))
 218
 219 #         llfuse.init(operations, self.mounttmp, [])
 220 #         t = threading.Thread(None, lambda: llfuse.main())
 221 #         t.start()
 222
 223 #         # wait until the driver is finished initializing
 224 #         operations.initlock.wait()
 225
 226 #         d1 = os.listdir(self.mounttmp)
 227 #         d1.sort()
 228 #         self.assertEqual(['foo_tag'], d1)
 229
 230 #         api.links().create(body={'link': {
 231 #             'head_uuid': 'fa7aeb5140e2848d39b416daeef4ffc5+45',
 232 #             'link_class': 'tag',
 233 #             'name': 'bar_tag'
 234 #         }}).execute()
 235
 236 #         time.sleep(1)
 237
 238 #         d2 = os.listdir(self.mounttmp)
 239 #         d2.sort()
 240 #         self.assertEqual(['bar_tag', 'foo_tag'], d2)
 241
 242 #         d3 = os.listdir(os.path.join(self.mounttmp, 'bar_tag'))
 243 #         d3.sort()
 244 #         self.assertEqual(['fa7aeb5140e2848d39b416daeef4ffc5+45'], d3)
 245
 246 #         l = api.links().create(body={'link': {
 247 #             'head_uuid': 'ea10d51bcf88862dbcc36eb292017dfd+45',
 248 #             'link_class': 'tag',
 249 #             'name': 'bar_tag'
 250 #         }}).execute()
 251
 252 #         time.sleep(1)
 253
 254 #         d4 = os.listdir(os.path.join(self.mounttmp, 'bar_tag'))
 255 #         d4.sort()
 256 #         self.assertEqual(['ea10d51bcf88862dbcc36eb292017dfd+45', 'fa7aeb5140e2848d39b416daeef4ffc5+45'], d4)
 257
 258 #         api.links().delete(uuid=l['uuid']).execute()
 259
 260 #         time.sleep(1)
 261
 262 #         d5 = os.listdir(os.path.join(self.mounttmp, 'bar_tag'))
 263 #         d5.sort()
 264 #         self.assertEqual(['fa7aeb5140e2848d39b416daeef4ffc5+45'], d5)
 265
 266
 267 # class FuseTagsUpdateTestWebsockets(FuseTagsUpdateTestBase):
 268 #     def setUp(self):
 269 #         super(FuseTagsUpdateTestWebsockets, self).setUp()
 270 #         run_test_server.run(True)
 271
 272 #     def runTest(self):
 273 #         self.runRealTest()
 274
 275 #     def tearDown(self):
 276 #         run_test_server.stop()
 277 #         super(FuseTagsUpdateTestWebsockets, self).tearDown()
 278
 279
 280 # class FuseTagsUpdateTestPoll(FuseTagsUpdateTestBase):
 281 #     def setUp(self):
 282 #         super(FuseTagsUpdateTestPoll, self).setUp()
 283 #         run_test_server.run(False)
 284
 285 #     def runTest(self):
 286 #         self.runRealTest()
 287
 288 #     def tearDown(self):
 289 #         run_test_server.stop()
 290 #         super(FuseTagsUpdateTestPoll, self).tearDown()
 291
 292
 293 # class FuseGroupsTest(MountTestBase):
 294 #     def setUp(self):
 295 #         super(FuseGroupsTest, self).setUp()
 296 #         run_test_server.run()
 297
 298 #     def runTest(self):
 299 #         run_test_server.authorize_with("admin")
 300 #         api = arvados.api('v1', cache=False)
 301
 302 #         operations = fuse.Operations(os.getuid(), os.getgid())
 303 #         e = operations.inodes.add_entry(fuse.GroupsDirectory(llfuse.ROOT_INODE, operations.inodes, api))
 304
 305 #         llfuse.init(operations, self.mounttmp, [])
 306 #         t = threading.Thread(None, lambda: llfuse.main())
 307 #         t.start()
 308
 309 #         # wait until the driver is finished initializing
 310 #         operations.initlock.wait()
 311
 312 #         d1 = os.listdir(self.mounttmp)
 313 #         d1.sort()
 314 #         self.assertIn('zzzzz-j7d0g-v955i6s2oi1cbso', d1)
 315
 316 #         d2 = os.listdir(os.path.join(self.mounttmp, 'zzzzz-j7d0g-v955i6s2oi1cbso'))
 317 #         d2.sort()
 318 #         self.assertEqual(['1f4b0bc7583c2a7f9102c395f4ffc5e3+45 added sometime',
 319 #                           "I'm a job in a project",
 320 #                           "I'm a template in a project",
 321 #                           "zzzzz-j58dm-5gid26432uujf79",
 322 #                           "zzzzz-j58dm-7r18rnd5nzhg5yk",
 323 #                           "zzzzz-j58dm-ypsjlol9dofwijz",
 324 #                           "zzzzz-j7d0g-axqo7eu9pwvna1x"
 325 #                       ], d2)
 326
 327 #         d3 = os.listdir(os.path.join(self.mounttmp, 'zzzzz-j7d0g-v955i6s2oi1cbso', 'zzzzz-j7d0g-axqo7eu9pwvna1x'))
 328 #         d3.sort()
 329 #         self.assertEqual(["I'm in a subproject, too",
 330 #                           "ea10d51bcf88862dbcc36eb292017dfd+45 added sometime",
 331 #                           "zzzzz-j58dm-c40lddwcqqr1ffs"
 332 #                       ], d3)
 333
 334 #         with open(os.path.join(self.mounttmp, 'zzzzz-j7d0g-v955i6s2oi1cbso', "I'm a template in a project")) as f:
 335 #             j = json.load(f)
 336 #             self.assertEqual("Two Part Pipeline Template", j['name'])
 337
 338 #     def tearDown(self):
 339 #         run_test_server.stop()
 340 #         super(FuseGroupsTest, self).tearDown()