2 import arvados_fuse as fuse
8 from .. import run_test_server
9 from ..mount_test_base import MountTestBase
10 from ..slow_test import slow_test
12 logger = logging.getLogger('arvados.arv-mount')
14 from performance_profiler import profiled
16 def fuse_createCollectionWithMultipleBlocks(mounttmp, streams=1, files_per_stream=1, data='x'):
17 class Test(unittest.TestCase):
19 self.createCollectionWithMultipleBlocks()
22 def createCollectionWithMultipleBlocks(self):
23 for i in range(0, streams):
24 os.mkdir(os.path.join(mounttmp, "./stream" + str(i)))
27 for j in range(0, files_per_stream):
28 with open(os.path.join(mounttmp, "./stream" + str(i), "file" + str(j) +".txt"), "w") as f:
33 def fuse_readContentsFromCollectionWithMultipleBlocks(mounttmp, streams=1, files_per_stream=1, data='x'):
34 class Test(unittest.TestCase):
36 self.readContentsFromCollectionWithMultipleBlocks()
39 def readContentsFromCollectionWithMultipleBlocks(self):
40 for i in range(0, streams):
41 d1 = llfuse.listdir(os.path.join(mounttmp, 'stream'+str(i)))
42 for j in range(0, files_per_stream):
43 with open(os.path.join(mounttmp, 'stream'+str(i), 'file'+str(i)+'.txt')) as f:
44 self.assertEqual(data, f.read())
48 def fuse_moveFileFromCollectionWithMultipleBlocks(mounttmp, stream, filename):
49 class Test(unittest.TestCase):
51 self.moveFileFromCollectionWithMultipleBlocks()
54 def moveFileFromCollectionWithMultipleBlocks(self):
55 d1 = llfuse.listdir(os.path.join(mounttmp, stream))
56 self.assertIn(filename, d1)
58 os.rename(os.path.join(mounttmp, stream, filename), os.path.join(mounttmp, 'moved_from_'+stream+'_'+filename))
60 d1 = llfuse.listdir(os.path.join(mounttmp))
61 self.assertIn('moved_from_'+stream+'_'+filename, d1)
63 d1 = llfuse.listdir(os.path.join(mounttmp, stream))
64 self.assertNotIn(filename, d1)
68 def fuse_deleteFileFromCollectionWithMultipleBlocks(mounttmp, stream, filename):
69 class Test(unittest.TestCase):
71 self.deleteFileFromCollectionWithMultipleBlocks()
74 def deleteFileFromCollectionWithMultipleBlocks(self):
75 os.remove(os.path.join(mounttmp, stream, filename))
79 # Create a collection with 2 streams, 3 files_per_stream, 2 blocks_per_file, 2**26 bytes_per_block
80 class CreateCollectionWithMultipleBlocksAndMoveAndDeleteFile(MountTestBase):
82 super(CreateCollectionWithMultipleBlocksAndMoveAndDeleteFile, self).setUp()
85 def test_CreateCollectionWithManyBlocksAndMoveAndDeleteFile(self):
86 collection = arvados.collection.Collection(api_client=self.api)
89 m = self.make_mount(fuse.CollectionDirectory)
91 m.new_collection(collection.api_response(), collection)
92 self.assertTrue(m.writable())
97 bytes_per_block = 2**26
99 data = 'x' * blocks_per_file * bytes_per_block
101 self.pool.apply(fuse_createCollectionWithMultipleBlocks, (self.mounttmp, streams, files_per_stream, data,))
103 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
105 for i in range(0, streams):
106 self.assertIn('./stream' + str(i), collection2["manifest_text"])
108 for i in range(0, files_per_stream):
109 self.assertIn('file' + str(i) + '.txt', collection2["manifest_text"])
112 self.pool.apply(fuse_readContentsFromCollectionWithMultipleBlocks, (self.mounttmp, streams, files_per_stream, data,))
114 # Move file0.txt out of the streams into .
115 for i in range(0, streams):
116 self.pool.apply(fuse_moveFileFromCollectionWithMultipleBlocks, (self.mounttmp, 'stream'+str(i), 'file0.txt',))
118 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
120 manifest_streams = collection2['manifest_text'].split('\n')
121 self.assertEqual(4, len(manifest_streams))
123 for i in range(0, streams):
124 self.assertIn('file0.txt', manifest_streams[0])
126 for i in range(0, streams):
127 self.assertNotIn('file0.txt', manifest_streams[i+1])
129 for i in range(0, streams):
130 for j in range(1, files_per_stream):
131 self.assertIn('file' + str(j) + '.txt', manifest_streams[i+1])
133 # Delete 'file1.txt' from all the streams
134 for i in range(0, streams):
135 self.pool.apply(fuse_deleteFileFromCollectionWithMultipleBlocks, (self.mounttmp, 'stream'+str(i), 'file1.txt'))
137 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
139 manifest_streams = collection2['manifest_text'].split('\n')
140 self.assertEqual(4, len(manifest_streams))
142 for i in range(0, streams):
143 self.assertIn('file0.txt', manifest_streams[0])
145 self.assertNotIn('file1.txt', collection2['manifest_text'])
147 for i in range(0, streams):
148 for j in range(2, files_per_stream):
149 self.assertIn('file' + str(j) + '.txt', manifest_streams[i+1])
152 def fuse_createCollectionWithManyFiles(mounttmp, streams=1, files_per_stream=1, data='x'):
153 class Test(unittest.TestCase):
155 self.createCollectionWithManyFiles()
158 def createCollectionWithManyFiles(self):
159 for i in range(0, streams):
160 os.mkdir(os.path.join(mounttmp, "./stream" + str(i)))
163 for j in range(0, files_per_stream):
164 with open(os.path.join(mounttmp, "./stream" + str(i), "file" + str(j) +".txt"), "w") as f:
169 def fuse_readContentsFromCollectionWithManyFiles(mounttmp, streams=1, files_per_stream=1, data='x'):
170 class Test(unittest.TestCase):
172 self.readContentsFromCollectionWithManyFiles()
175 def readContentsFromCollectionWithManyFiles(self):
176 for i in range(0, streams):
177 d1 = llfuse.listdir(os.path.join(mounttmp, 'stream'+str(i)))
178 for j in range(0, files_per_stream):
179 with open(os.path.join(mounttmp, 'stream'+str(i), 'file'+str(i)+'.txt')) as f:
180 self.assertEqual(data, f.read())
184 def fuse_moveFileFromCollectionWithManyFiles(mounttmp, stream, filename):
185 class Test(unittest.TestCase):
187 self.moveFileFromCollectionWithManyFiles()
190 def moveFileFromCollectionWithManyFiles(self):
191 d1 = llfuse.listdir(os.path.join(mounttmp, stream))
192 self.assertIn(filename, d1)
194 os.rename(os.path.join(mounttmp, stream, filename), os.path.join(mounttmp, 'moved_from_'+stream+'_'+filename))
196 d1 = llfuse.listdir(os.path.join(mounttmp))
197 self.assertIn('moved_from_'+stream+'_'+filename, d1)
199 d1 = llfuse.listdir(os.path.join(mounttmp, stream))
200 self.assertNotIn(filename, d1)
204 def fuse_deleteFileFromCollectionWithManyFiles(mounttmp, stream, filename):
205 class Test(unittest.TestCase):
207 self.deleteFileFromCollectionWithManyFiles()
210 def deleteFileFromCollectionWithManyFiles(self):
211 os.remove(os.path.join(mounttmp, stream, filename))
215 # Create a collection with two streams, each with 200 files
216 class CreateCollectionWithManyFilesAndMoveAndDeleteFile(MountTestBase):
218 super(CreateCollectionWithManyFilesAndMoveAndDeleteFile, self).setUp()
221 def test_CreateCollectionWithManyFilesAndMoveAndDeleteFile(self):
222 collection = arvados.collection.Collection(api_client=self.api)
223 collection.save_new()
225 m = self.make_mount(fuse.CollectionDirectory)
227 m.new_collection(collection.api_response(), collection)
228 self.assertTrue(m.writable())
231 files_per_stream = 200
234 self.pool.apply(fuse_createCollectionWithManyFiles, (self.mounttmp, streams, files_per_stream, data,))
236 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
238 for i in range(0, streams):
239 self.assertIn('./stream' + str(i), collection2["manifest_text"])
241 for i in range(0, files_per_stream):
242 self.assertIn('file' + str(i) + '.txt', collection2["manifest_text"])
245 self.pool.apply(fuse_readContentsFromCollectionWithManyFiles, (self.mounttmp, streams, files_per_stream, data,))
247 # Move file0.txt out of the streams into .
248 for i in range(0, streams):
249 self.pool.apply(fuse_moveFileFromCollectionWithManyFiles, (self.mounttmp, 'stream'+str(i), 'file0.txt',))
251 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
253 manifest_streams = collection2['manifest_text'].split('\n')
254 self.assertEqual(4, len(manifest_streams))
256 for i in range(0, streams):
257 self.assertIn('file0.txt', manifest_streams[0])
259 for i in range(0, streams):
260 self.assertNotIn('file0.txt', manifest_streams[i+1])
262 for i in range(0, streams):
263 for j in range(1, files_per_stream):
264 self.assertIn('file' + str(j) + '.txt', manifest_streams[i+1])
266 # Delete 'file1.txt' from all the streams
267 for i in range(0, streams):
268 self.pool.apply(fuse_deleteFileFromCollectionWithManyFiles, (self.mounttmp, 'stream'+str(i), 'file1.txt'))
270 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
272 manifest_streams = collection2['manifest_text'].split('\n')
273 self.assertEqual(4, len(manifest_streams))
275 for i in range(0, streams):
276 self.assertIn('file0.txt', manifest_streams[0])
278 self.assertNotIn('file1.txt', collection2['manifest_text'])
280 for i in range(0, streams):
281 for j in range(2, files_per_stream):
282 self.assertIn('file' + str(j) + '.txt', manifest_streams[i+1])
285 def magicDirTest_MoveFileFromCollection(mounttmp, collection1, collection2, stream, filename):
286 class Test(unittest.TestCase):
288 self.magicDirTest_moveFileFromCollection()
291 def magicDirTest_moveFileFromCollection(self):
292 os.rename(os.path.join(mounttmp, collection1, filename), os.path.join(mounttmp, collection2, filename))
296 def magicDirTest_RemoveFileFromCollection(mounttmp, collection1, stream, filename):
297 class Test(unittest.TestCase):
299 self.magicDirTest_removeFileFromCollection()
302 def magicDirTest_removeFileFromCollection(self):
303 os.remove(os.path.join(mounttmp, collection1, filename))
307 class UsingMagicDir_CreateCollectionWithManyFilesAndMoveAndDeleteFile(MountTestBase):
309 super(UsingMagicDir_CreateCollectionWithManyFilesAndMoveAndDeleteFile, self).setUp()
312 def magicDirTest_createCollectionWithManyFiles(self, streams=0, files_per_stream=0, data='x'):
314 collection = arvados.collection.Collection(api_client=self.api)
315 for j in range(0, files_per_stream):
316 with collection.open("file"+str(j)+".txt", "w") as f:
318 collection.save_new()
322 def magicDirTest_readCollectionContents(self, collection, streams=1, files_per_stream=1, data='x'):
323 mount_ls = os.listdir(os.path.join(self.mounttmp, collection))
326 for j in range(0, files_per_stream):
327 files[os.path.join(self.mounttmp, collection, 'file'+str(j)+'.txt')] = data
329 for k, v in files.items():
330 with open(os.path.join(self.mounttmp, collection, k)) as f:
331 self.assertEqual(v, f.read())
334 def test_UsingMagicDirCreateCollectionWithManyFilesAndMoveAndDeleteFile(self):
336 files_per_stream = 200
339 collection1 = self.magicDirTest_createCollectionWithManyFiles()
340 # Create collection with multiple files
341 collection2 = self.magicDirTest_createCollectionWithManyFiles(streams, files_per_stream, data)
344 self.make_mount(fuse.MagicDirectory)
346 self.magicDirTest_readCollectionContents(collection2.manifest_locator(), streams, files_per_stream, data)
348 # Move file0.txt out of the collection2 into collection1
349 self.pool.apply(magicDirTest_MoveFileFromCollection, (self.mounttmp, collection2.manifest_locator(),
350 collection1.manifest_locator(), 'stream0', 'file0.txt',))
351 updated_collection = self.api.collections().get(uuid=collection2.manifest_locator()).execute()
352 self.assertFalse('file0.txt' in updated_collection['manifest_text'])
353 self.assertTrue('file1.txt' in updated_collection['manifest_text'])
355 # Delete file1.txt from collection2
356 self.pool.apply(magicDirTest_RemoveFileFromCollection, (self.mounttmp, collection2.manifest_locator(), 'stream0', 'file1.txt',))
357 updated_collection = self.api.collections().get(uuid=collection2.manifest_locator()).execute()
358 self.assertFalse('file1.txt' in updated_collection['manifest_text'])
359 self.assertTrue('file2.txt' in updated_collection['manifest_text'])
362 def magicDirTest_MoveAllFilesFromCollection(mounttmp, from_collection, to_collection, stream, files_per_stream):
363 class Test(unittest.TestCase):
365 self.magicDirTest_moveAllFilesFromCollection()
368 def magicDirTest_moveAllFilesFromCollection(self):
369 for j in range(0, files_per_stream):
370 os.rename(os.path.join(mounttmp, from_collection, 'file'+str(j)+'.txt'), os.path.join(mounttmp, to_collection, 'file'+str(j)+'.txt'))
374 class UsingMagicDir_CreateCollectionWithManyFilesAndMoveAllFilesIntoAnother(MountTestBase):
376 super(UsingMagicDir_CreateCollectionWithManyFilesAndMoveAllFilesIntoAnother, self).setUp()
379 def magicDirTestMoveAllFiles_createCollectionWithManyFiles(self, streams=0, files_per_stream=0,
380 blocks_per_file=0, bytes_per_block=0, data='x'):
382 collection = arvados.collection.Collection(api_client=self.api)
383 for j in range(0, files_per_stream):
384 with collection.open("file"+str(j)+".txt", "w") as f:
386 collection.save_new()
390 def test_UsingMagicDirCreateCollectionWithManyFilesAndMoveAllFilesIntoAnother(self):
392 files_per_stream = 200
395 collection1 = self.magicDirTestMoveAllFiles_createCollectionWithManyFiles()
396 # Create collection with multiple files
397 collection2 = self.magicDirTestMoveAllFiles_createCollectionWithManyFiles(streams, files_per_stream, data)
400 self.make_mount(fuse.MagicDirectory)
402 # Move all files from collection2 into collection1
403 self.pool.apply(magicDirTest_MoveAllFilesFromCollection, (self.mounttmp, collection2.manifest_locator(),
404 collection1.manifest_locator(), 'stream0', files_per_stream,))
406 updated_collection = self.api.collections().get(uuid=collection2.manifest_locator()).execute()
407 file_names = ["file%i.txt" % i for i in range(0, files_per_stream)]
408 for name in file_names:
409 self.assertFalse(name in updated_collection['manifest_text'])
411 updated_collection = self.api.collections().get(uuid=collection1.manifest_locator()).execute()
412 for name in file_names:
413 self.assertTrue(name in updated_collection['manifest_text'])
416 # Move one file at a time from one collection into another
417 class UsingMagicDir_CreateCollectionWithManyFilesAndMoveEachFileIntoAnother(MountTestBase):
419 super(UsingMagicDir_CreateCollectionWithManyFilesAndMoveEachFileIntoAnother, self).setUp()
422 def magicDirTestMoveFiles_createCollectionWithManyFiles(self, streams=0, files_per_stream=0, data='x'):
424 collection = arvados.collection.Collection(api_client=self.api)
425 for j in range(0, files_per_stream):
426 with collection.open("file"+str(j)+".txt", "w") as f:
428 collection.save_new()
431 def magicDirTestMoveFiles_oneEachIntoAnother(self, from_collection, to_collection, files_per_stream):
432 for j in range(0, files_per_stream):
433 self.pool.apply(magicDirTest_MoveFileFromCollection, (self.mounttmp, from_collection.manifest_locator(),
434 to_collection.manifest_locator(), 'stream0', 'file'+str(j)+'.txt',))
437 def test_UsingMagicDirCreateCollectionWithManyFilesAndMoveEachFileIntoAnother(self):
439 files_per_stream = 200
442 collection1 = self.magicDirTestMoveFiles_createCollectionWithManyFiles()
443 # Create collection with multiple files
444 collection2 = self.magicDirTestMoveFiles_createCollectionWithManyFiles(streams, files_per_stream, data)
447 self.make_mount(fuse.MagicDirectory)
449 # Move all files from collection2 into collection1
450 self.magicDirTestMoveFiles_oneEachIntoAnother(collection2, collection1, files_per_stream)
452 updated_collection = self.api.collections().get(uuid=collection2.manifest_locator()).execute()
453 file_names = ["file%i.txt" % i for i in range(0, files_per_stream)]
454 for name in file_names:
455 self.assertFalse(name in updated_collection['manifest_text'])
457 updated_collection = self.api.collections().get(uuid=collection1.manifest_locator()).execute()
458 for name in file_names:
459 self.assertTrue(name in updated_collection['manifest_text'])
461 class FuseListLargeProjectContents(MountTestBase):
463 def getProjectWithManyCollections(self):
464 project_contents = llfuse.listdir(self.mounttmp)
465 self.assertEqual(201, len(project_contents))
466 self.assertIn('Collection_1', project_contents)
467 return project_contents
470 def listContentsInProjectWithManyCollections(self, project_contents):
471 project_contents = llfuse.listdir(self.mounttmp)
472 self.assertEqual(201, len(project_contents))
473 self.assertIn('Collection_1', project_contents)
475 for collection_name in project_contents:
476 collection_contents = llfuse.listdir(os.path.join(self.mounttmp, collection_name))
477 self.assertIn('baz', collection_contents)
480 def test_listLargeProjectContents(self):
481 self.make_mount(fuse.ProjectDirectory,
482 project_object=run_test_server.fixture('groups')['project_with_201_collections'])
483 project_contents = self.getProjectWithManyCollections()
484 self.listContentsInProjectWithManyCollections(project_contents)