1 # Copyright (C) The Arvados Authors. All rights reserved.
3 # SPDX-License-Identifier: AGPL-3.0
6 import arvados_fuse as fuse
12 from .. import run_test_server
13 from ..mount_test_base import MountTestBase
14 from ..slow_test import slow_test
16 logger = logging.getLogger('arvados.arv-mount')
18 from performance_profiler import profiled
20 def fuse_createCollectionWithMultipleBlocks(mounttmp, streams=1, files_per_stream=1, data='x'):
21 class Test(unittest.TestCase):
23 self.createCollectionWithMultipleBlocks()
26 def createCollectionWithMultipleBlocks(self):
27 for i in range(0, streams):
28 os.mkdir(os.path.join(mounttmp, "./stream" + str(i)))
31 for j in range(0, files_per_stream):
32 with open(os.path.join(mounttmp, "./stream" + str(i), "file" + str(j) +".txt"), "w") as f:
37 def fuse_readContentsFromCollectionWithMultipleBlocks(mounttmp, streams=1, files_per_stream=1, data='x'):
38 class Test(unittest.TestCase):
40 self.readContentsFromCollectionWithMultipleBlocks()
43 def readContentsFromCollectionWithMultipleBlocks(self):
44 for i in range(0, streams):
45 d1 = llfuse.listdir(os.path.join(mounttmp, 'stream'+str(i)))
46 for j in range(0, files_per_stream):
47 with open(os.path.join(mounttmp, 'stream'+str(i), 'file'+str(i)+'.txt')) as f:
48 self.assertEqual(data, f.read())
52 def fuse_moveFileFromCollectionWithMultipleBlocks(mounttmp, stream, filename):
53 class Test(unittest.TestCase):
55 self.moveFileFromCollectionWithMultipleBlocks()
58 def moveFileFromCollectionWithMultipleBlocks(self):
59 d1 = llfuse.listdir(os.path.join(mounttmp, stream))
60 self.assertIn(filename, d1)
62 os.rename(os.path.join(mounttmp, stream, filename), os.path.join(mounttmp, 'moved_from_'+stream+'_'+filename))
64 d1 = llfuse.listdir(os.path.join(mounttmp))
65 self.assertIn('moved_from_'+stream+'_'+filename, d1)
67 d1 = llfuse.listdir(os.path.join(mounttmp, stream))
68 self.assertNotIn(filename, d1)
72 def fuse_deleteFileFromCollectionWithMultipleBlocks(mounttmp, stream, filename):
73 class Test(unittest.TestCase):
75 self.deleteFileFromCollectionWithMultipleBlocks()
78 def deleteFileFromCollectionWithMultipleBlocks(self):
79 os.remove(os.path.join(mounttmp, stream, filename))
83 # Create a collection with 2 streams, 3 files_per_stream, 2 blocks_per_file, 2**26 bytes_per_block
84 class CreateCollectionWithMultipleBlocksAndMoveAndDeleteFile(MountTestBase):
86 super(CreateCollectionWithMultipleBlocksAndMoveAndDeleteFile, self).setUp()
89 def test_CreateCollectionWithManyBlocksAndMoveAndDeleteFile(self):
90 collection = arvados.collection.Collection(api_client=self.api)
93 m = self.make_mount(fuse.CollectionDirectory)
95 m.new_collection(collection.api_response(), collection)
96 self.assertTrue(m.writable())
101 bytes_per_block = 2**26
103 data = 'x' * blocks_per_file * bytes_per_block
105 self.pool.apply(fuse_createCollectionWithMultipleBlocks, (self.mounttmp, streams, files_per_stream, data,))
107 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
109 for i in range(0, streams):
110 self.assertIn('./stream' + str(i), collection2["manifest_text"])
112 for i in range(0, files_per_stream):
113 self.assertIn('file' + str(i) + '.txt', collection2["manifest_text"])
116 self.pool.apply(fuse_readContentsFromCollectionWithMultipleBlocks, (self.mounttmp, streams, files_per_stream, data,))
118 # Move file0.txt out of the streams into .
119 for i in range(0, streams):
120 self.pool.apply(fuse_moveFileFromCollectionWithMultipleBlocks, (self.mounttmp, 'stream'+str(i), 'file0.txt',))
122 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
124 manifest_streams = collection2['manifest_text'].split('\n')
125 self.assertEqual(4, len(manifest_streams))
127 for i in range(0, streams):
128 self.assertIn('file0.txt', manifest_streams[0])
130 for i in range(0, streams):
131 self.assertNotIn('file0.txt', manifest_streams[i+1])
133 for i in range(0, streams):
134 for j in range(1, files_per_stream):
135 self.assertIn('file' + str(j) + '.txt', manifest_streams[i+1])
137 # Delete 'file1.txt' from all the streams
138 for i in range(0, streams):
139 self.pool.apply(fuse_deleteFileFromCollectionWithMultipleBlocks, (self.mounttmp, 'stream'+str(i), 'file1.txt'))
141 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
143 manifest_streams = collection2['manifest_text'].split('\n')
144 self.assertEqual(4, len(manifest_streams))
146 for i in range(0, streams):
147 self.assertIn('file0.txt', manifest_streams[0])
149 self.assertNotIn('file1.txt', collection2['manifest_text'])
151 for i in range(0, streams):
152 for j in range(2, files_per_stream):
153 self.assertIn('file' + str(j) + '.txt', manifest_streams[i+1])
156 def fuse_createCollectionWithManyFiles(mounttmp, streams=1, files_per_stream=1, data='x'):
157 class Test(unittest.TestCase):
159 self.createCollectionWithManyFiles()
162 def createCollectionWithManyFiles(self):
163 for i in range(0, streams):
164 os.mkdir(os.path.join(mounttmp, "./stream" + str(i)))
167 for j in range(0, files_per_stream):
168 with open(os.path.join(mounttmp, "./stream" + str(i), "file" + str(j) +".txt"), "w") as f:
173 def fuse_readContentsFromCollectionWithManyFiles(mounttmp, streams=1, files_per_stream=1, data='x'):
174 class Test(unittest.TestCase):
176 self.readContentsFromCollectionWithManyFiles()
179 def readContentsFromCollectionWithManyFiles(self):
180 for i in range(0, streams):
181 d1 = llfuse.listdir(os.path.join(mounttmp, 'stream'+str(i)))
182 for j in range(0, files_per_stream):
183 with open(os.path.join(mounttmp, 'stream'+str(i), 'file'+str(i)+'.txt')) as f:
184 self.assertEqual(data, f.read())
188 def fuse_moveFileFromCollectionWithManyFiles(mounttmp, stream, filename):
189 class Test(unittest.TestCase):
191 self.moveFileFromCollectionWithManyFiles()
194 def moveFileFromCollectionWithManyFiles(self):
195 d1 = llfuse.listdir(os.path.join(mounttmp, stream))
196 self.assertIn(filename, d1)
198 os.rename(os.path.join(mounttmp, stream, filename), os.path.join(mounttmp, 'moved_from_'+stream+'_'+filename))
200 d1 = llfuse.listdir(os.path.join(mounttmp))
201 self.assertIn('moved_from_'+stream+'_'+filename, d1)
203 d1 = llfuse.listdir(os.path.join(mounttmp, stream))
204 self.assertNotIn(filename, d1)
208 def fuse_deleteFileFromCollectionWithManyFiles(mounttmp, stream, filename):
209 class Test(unittest.TestCase):
211 self.deleteFileFromCollectionWithManyFiles()
214 def deleteFileFromCollectionWithManyFiles(self):
215 os.remove(os.path.join(mounttmp, stream, filename))
219 # Create a collection with two streams, each with 200 files
220 class CreateCollectionWithManyFilesAndMoveAndDeleteFile(MountTestBase):
222 super(CreateCollectionWithManyFilesAndMoveAndDeleteFile, self).setUp()
225 def test_CreateCollectionWithManyFilesAndMoveAndDeleteFile(self):
226 collection = arvados.collection.Collection(api_client=self.api)
227 collection.save_new()
229 m = self.make_mount(fuse.CollectionDirectory)
231 m.new_collection(collection.api_response(), collection)
232 self.assertTrue(m.writable())
235 files_per_stream = 200
238 self.pool.apply(fuse_createCollectionWithManyFiles, (self.mounttmp, streams, files_per_stream, data,))
240 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
242 for i in range(0, streams):
243 self.assertIn('./stream' + str(i), collection2["manifest_text"])
245 for i in range(0, files_per_stream):
246 self.assertIn('file' + str(i) + '.txt', collection2["manifest_text"])
249 self.pool.apply(fuse_readContentsFromCollectionWithManyFiles, (self.mounttmp, streams, files_per_stream, data,))
251 # Move file0.txt out of the streams into .
252 for i in range(0, streams):
253 self.pool.apply(fuse_moveFileFromCollectionWithManyFiles, (self.mounttmp, 'stream'+str(i), 'file0.txt',))
255 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
257 manifest_streams = collection2['manifest_text'].split('\n')
258 self.assertEqual(4, len(manifest_streams))
260 for i in range(0, streams):
261 self.assertIn('file0.txt', manifest_streams[0])
263 for i in range(0, streams):
264 self.assertNotIn('file0.txt', manifest_streams[i+1])
266 for i in range(0, streams):
267 for j in range(1, files_per_stream):
268 self.assertIn('file' + str(j) + '.txt', manifest_streams[i+1])
270 # Delete 'file1.txt' from all the streams
271 for i in range(0, streams):
272 self.pool.apply(fuse_deleteFileFromCollectionWithManyFiles, (self.mounttmp, 'stream'+str(i), 'file1.txt'))
274 collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
276 manifest_streams = collection2['manifest_text'].split('\n')
277 self.assertEqual(4, len(manifest_streams))
279 for i in range(0, streams):
280 self.assertIn('file0.txt', manifest_streams[0])
282 self.assertNotIn('file1.txt', collection2['manifest_text'])
284 for i in range(0, streams):
285 for j in range(2, files_per_stream):
286 self.assertIn('file' + str(j) + '.txt', manifest_streams[i+1])
289 def magicDirTest_MoveFileFromCollection(mounttmp, collection1, collection2, stream, filename):
290 class Test(unittest.TestCase):
292 self.magicDirTest_moveFileFromCollection()
295 def magicDirTest_moveFileFromCollection(self):
296 os.rename(os.path.join(mounttmp, collection1, filename), os.path.join(mounttmp, collection2, filename))
300 def magicDirTest_RemoveFileFromCollection(mounttmp, collection1, stream, filename):
301 class Test(unittest.TestCase):
303 self.magicDirTest_removeFileFromCollection()
306 def magicDirTest_removeFileFromCollection(self):
307 os.remove(os.path.join(mounttmp, collection1, filename))
311 class UsingMagicDir_CreateCollectionWithManyFilesAndMoveAndDeleteFile(MountTestBase):
313 super(UsingMagicDir_CreateCollectionWithManyFilesAndMoveAndDeleteFile, self).setUp()
316 def magicDirTest_createCollectionWithManyFiles(self, streams=0, files_per_stream=0, data='x'):
318 collection = arvados.collection.Collection(api_client=self.api)
319 for j in range(0, files_per_stream):
320 with collection.open("file"+str(j)+".txt", "w") as f:
322 collection.save_new()
326 def magicDirTest_readCollectionContents(self, collection, streams=1, files_per_stream=1, data='x'):
327 mount_ls = os.listdir(os.path.join(self.mounttmp, collection))
330 for j in range(0, files_per_stream):
331 files[os.path.join(self.mounttmp, collection, 'file'+str(j)+'.txt')] = data
333 for k, v in files.items():
334 with open(os.path.join(self.mounttmp, collection, k)) as f:
335 self.assertEqual(v, f.read())
338 def test_UsingMagicDirCreateCollectionWithManyFilesAndMoveAndDeleteFile(self):
340 files_per_stream = 200
343 collection1 = self.magicDirTest_createCollectionWithManyFiles()
344 # Create collection with multiple files
345 collection2 = self.magicDirTest_createCollectionWithManyFiles(streams, files_per_stream, data)
348 self.make_mount(fuse.MagicDirectory)
350 self.magicDirTest_readCollectionContents(collection2.manifest_locator(), streams, files_per_stream, data)
352 # Move file0.txt out of the collection2 into collection1
353 self.pool.apply(magicDirTest_MoveFileFromCollection, (self.mounttmp, collection2.manifest_locator(),
354 collection1.manifest_locator(), 'stream0', 'file0.txt',))
355 updated_collection = self.api.collections().get(uuid=collection2.manifest_locator()).execute()
356 self.assertFalse('file0.txt' in updated_collection['manifest_text'])
357 self.assertTrue('file1.txt' in updated_collection['manifest_text'])
359 # Delete file1.txt from collection2
360 self.pool.apply(magicDirTest_RemoveFileFromCollection, (self.mounttmp, collection2.manifest_locator(), 'stream0', 'file1.txt',))
361 updated_collection = self.api.collections().get(uuid=collection2.manifest_locator()).execute()
362 self.assertFalse('file1.txt' in updated_collection['manifest_text'])
363 self.assertTrue('file2.txt' in updated_collection['manifest_text'])
366 def magicDirTest_MoveAllFilesFromCollection(mounttmp, from_collection, to_collection, stream, files_per_stream):
367 class Test(unittest.TestCase):
369 self.magicDirTest_moveAllFilesFromCollection()
372 def magicDirTest_moveAllFilesFromCollection(self):
373 for j in range(0, files_per_stream):
374 os.rename(os.path.join(mounttmp, from_collection, 'file'+str(j)+'.txt'), os.path.join(mounttmp, to_collection, 'file'+str(j)+'.txt'))
378 class UsingMagicDir_CreateCollectionWithManyFilesAndMoveAllFilesIntoAnother(MountTestBase):
380 super(UsingMagicDir_CreateCollectionWithManyFilesAndMoveAllFilesIntoAnother, self).setUp()
383 def magicDirTestMoveAllFiles_createCollectionWithManyFiles(self, streams=0, files_per_stream=0,
384 blocks_per_file=0, bytes_per_block=0, data='x'):
386 collection = arvados.collection.Collection(api_client=self.api)
387 for j in range(0, files_per_stream):
388 with collection.open("file"+str(j)+".txt", "w") as f:
390 collection.save_new()
394 def test_UsingMagicDirCreateCollectionWithManyFilesAndMoveAllFilesIntoAnother(self):
396 files_per_stream = 200
399 collection1 = self.magicDirTestMoveAllFiles_createCollectionWithManyFiles()
400 # Create collection with multiple files
401 collection2 = self.magicDirTestMoveAllFiles_createCollectionWithManyFiles(streams, files_per_stream, data)
404 self.make_mount(fuse.MagicDirectory)
406 # Move all files from collection2 into collection1
407 self.pool.apply(magicDirTest_MoveAllFilesFromCollection, (self.mounttmp, collection2.manifest_locator(),
408 collection1.manifest_locator(), 'stream0', files_per_stream,))
410 updated_collection = self.api.collections().get(uuid=collection2.manifest_locator()).execute()
411 file_names = ["file%i.txt" % i for i in range(0, files_per_stream)]
412 for name in file_names:
413 self.assertFalse(name in updated_collection['manifest_text'])
415 updated_collection = self.api.collections().get(uuid=collection1.manifest_locator()).execute()
416 for name in file_names:
417 self.assertTrue(name in updated_collection['manifest_text'])
420 # Move one file at a time from one collection into another
421 class UsingMagicDir_CreateCollectionWithManyFilesAndMoveEachFileIntoAnother(MountTestBase):
423 super(UsingMagicDir_CreateCollectionWithManyFilesAndMoveEachFileIntoAnother, self).setUp()
426 def magicDirTestMoveFiles_createCollectionWithManyFiles(self, streams=0, files_per_stream=0, data='x'):
428 collection = arvados.collection.Collection(api_client=self.api)
429 for j in range(0, files_per_stream):
430 with collection.open("file"+str(j)+".txt", "w") as f:
432 collection.save_new()
435 def magicDirTestMoveFiles_oneEachIntoAnother(self, from_collection, to_collection, files_per_stream):
436 for j in range(0, files_per_stream):
437 self.pool.apply(magicDirTest_MoveFileFromCollection, (self.mounttmp, from_collection.manifest_locator(),
438 to_collection.manifest_locator(), 'stream0', 'file'+str(j)+'.txt',))
441 def test_UsingMagicDirCreateCollectionWithManyFilesAndMoveEachFileIntoAnother(self):
443 files_per_stream = 200
446 collection1 = self.magicDirTestMoveFiles_createCollectionWithManyFiles()
447 # Create collection with multiple files
448 collection2 = self.magicDirTestMoveFiles_createCollectionWithManyFiles(streams, files_per_stream, data)
451 self.make_mount(fuse.MagicDirectory)
453 # Move all files from collection2 into collection1
454 self.magicDirTestMoveFiles_oneEachIntoAnother(collection2, collection1, files_per_stream)
456 updated_collection = self.api.collections().get(uuid=collection2.manifest_locator()).execute()
457 file_names = ["file%i.txt" % i for i in range(0, files_per_stream)]
458 for name in file_names:
459 self.assertFalse(name in updated_collection['manifest_text'])
461 updated_collection = self.api.collections().get(uuid=collection1.manifest_locator()).execute()
462 for name in file_names:
463 self.assertTrue(name in updated_collection['manifest_text'])
465 class FuseListLargeProjectContents(MountTestBase):
467 def getProjectWithManyCollections(self):
468 project_contents = llfuse.listdir(self.mounttmp)
469 self.assertEqual(201, len(project_contents))
470 self.assertIn('Collection_1', project_contents)
471 return project_contents
474 def listContentsInProjectWithManyCollections(self, project_contents):
475 project_contents = llfuse.listdir(self.mounttmp)
476 self.assertEqual(201, len(project_contents))
477 self.assertIn('Collection_1', project_contents)
479 for collection_name in project_contents:
480 collection_contents = llfuse.listdir(os.path.join(self.mounttmp, collection_name))
481 self.assertIn('baz', collection_contents)
484 def test_listLargeProjectContents(self):
485 self.make_mount(fuse.ProjectDirectory,
486 project_object=run_test_server.fixture('groups')['project_with_201_collections'])
487 project_contents = self.getProjectWithManyCollections()
488 self.listContentsInProjectWithManyCollections(project_contents)