X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/f56d3a6876f246f78d5bc231a0ac5b6e4c6bdb9c..a7b16675337995c65b909a7519178efed98a3089:/sdk/python/tests/test_arvfile.py diff --git a/sdk/python/tests/test_arvfile.py b/sdk/python/tests/test_arvfile.py index 5d71ec456d..06c70f004d 100644 --- a/sdk/python/tests/test_arvfile.py +++ b/sdk/python/tests/test_arvfile.py @@ -10,6 +10,7 @@ import hashlib import arvados from arvados import ArvadosFile, ArvadosFileReader, Range, import_manifest, export_manifest, KeepLocator +from arvados.arvfile import SYNC_READONLY, SYNC_EXPLICIT import arvados_testutil as tutil from test_stream import StreamFileReaderTestCase, StreamRetryTestMixin @@ -56,7 +57,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase): "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n"}, {"uuid":"zzzzz-4zz18-mockcollection0"}) with import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', - api_client=api, keep=keep) as c: + api_client=api, keep=keep, sync=SYNC_EXPLICIT) as c: writer = c.open("count.txt", "r+") self.assertEqual(writer.size(), 10) writer.seek(5) @@ -77,7 +78,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase): "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n"}, {"uuid":"zzzzz-4zz18-mockcollection0"}) with import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', - api_client=api, keep=keep) as c: + api_client=api, keep=keep, sync=SYNC_EXPLICIT) as c: writer = c.open("count.txt", "r+") writer.seek(5, os.SEEK_SET) self.assertEqual("56789", writer.read(8)) @@ -98,7 +99,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase): def test_write0(self): keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"}) with import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', - keep=keep) as c: + keep=keep, sync=SYNC_EXPLICIT) as c: writer = c.open("count.txt", "r+") self.assertEqual("0123456789", writer.readfrom(0, 13)) writer.seek(0, os.SEEK_SET) @@ -110,7 +111,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase): def test_write1(self): keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"}) with import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', - keep=keep) as c: + keep=keep, sync=SYNC_EXPLICIT) as c: writer = c.open("count.txt", "r+") self.assertEqual("0123456789", writer.readfrom(0, 13)) writer.seek(3, os.SEEK_SET) @@ -122,7 +123,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase): def test_write2(self): keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"}) with import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', - keep=keep) as c: + keep=keep, sync=SYNC_EXPLICIT) as c: writer = c.open("count.txt", "r+") self.assertEqual("0123456789", writer.readfrom(0, 13)) writer.seek(7, os.SEEK_SET) @@ -134,7 +135,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase): def test_write3(self): keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"}) with import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt 0:10:count.txt\n', - keep=keep) as c: + keep=keep, sync=SYNC_EXPLICIT) as c: writer = c.open("count.txt", "r+") self.assertEqual("012345678901234", writer.readfrom(0, 15)) writer.seek(7, os.SEEK_SET) @@ -146,7 +147,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase): def test_write4(self): keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"}) with import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:4:count.txt 0:4:count.txt 0:4:count.txt', - keep=keep) as c: + keep=keep, sync=SYNC_EXPLICIT) as c: writer = c.open("count.txt", "r+") self.assertEqual("012301230123", writer.readfrom(0, 15)) writer.seek(2, os.SEEK_SET) @@ -161,7 +162,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase): "manifest_text": ". a5de24f4417cfba9d5825eadc2f4ca49+67108000 598cc1a4ccaef8ab6e4724d87e675d78+32892000 0:100000000:count.txt\n"}, {"uuid":"zzzzz-4zz18-mockcollection0"}) with import_manifest('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt', - api_client=api, keep=keep) as c: + api_client=api, keep=keep, sync=SYNC_EXPLICIT) as c: writer = c.open("count.txt", "r+") text = ''.join(["0123456789" for a in xrange(0, 100)]) for b in xrange(0, 100000): @@ -177,7 +178,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase): def test_write_rewrite0(self): keep = ArvadosFileWriterTestCase.MockKeep({}) with import_manifest('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt', - keep=keep) as c: + keep=keep, sync=SYNC_EXPLICIT) as c: writer = c.open("count.txt", "r+") for b in xrange(0, 10): writer.seek(0, os.SEEK_SET) @@ -190,7 +191,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase): def test_write_rewrite1(self): keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"}) with import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt', - keep=keep) as c: + keep=keep, sync=SYNC_EXPLICIT) as c: writer = c.open("count.txt", "r+") for b in xrange(0, 10): writer.seek(10, os.SEEK_SET) @@ -203,7 +204,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase): def test_write_rewrite2(self): keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"}) with import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt', - keep=keep) as c: + keep=keep, sync=SYNC_EXPLICIT) as c: writer = c.open("count.txt", "r+") for b in xrange(0, 10): writer.seek(5, os.SEEK_SET) @@ -219,7 +220,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase): "manifest_text": ". 37400a68af9abdd76ca5bf13e819e42a+32892003 a5de24f4417cfba9d5825eadc2f4ca49+67108000 32892000:3:count.txt 32892006:67107997:count.txt 0:32892000:count.txt\n"}, {"uuid":"zzzzz-4zz18-mockcollection0"}) with import_manifest('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt', - api_client=api, keep=keep) as c: + api_client=api, keep=keep, sync=SYNC_EXPLICIT) as c: writer = c.open("count.txt", "r+") text = ''.join(["0123456789" for a in xrange(0, 100)]) for b in xrange(0, 100000): @@ -239,7 +240,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase): api = ArvadosFileWriterTestCase.MockApi({"name":"test_create", "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"}, {"uuid":"zzzzz-4zz18-mockcollection0"}) - with arvados.Collection(api_client=api, keep_client=keep) as c: + with arvados.Collection(api_client=api, keep_client=keep, sync=SYNC_EXPLICIT) as c: writer = c.open("count.txt", "w+") self.assertEqual(writer.size(), 0) writer.write("01234567") @@ -259,7 +260,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase): api = ArvadosFileWriterTestCase.MockApi({"name":"test_create", "manifest_text":"./foo/bar 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"}, {"uuid":"zzzzz-4zz18-mockcollection0"}) - with arvados.Collection(api_client=api, keep_client=keep) as c: + with arvados.Collection(api_client=api, keep_client=keep, sync=SYNC_EXPLICIT) as c: writer = c.open("foo/bar/count.txt", "w+") writer.write("01234567") c.save_as("test_create") @@ -270,7 +271,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase): "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"}, {"uuid":"zzzzz-4zz18-mockcollection0"}) with import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', - api_client=api, keep=keep) as c: + api_client=api, keep=keep, sync=SYNC_EXPLICIT) as c: writer = c.open("count.txt", "w+") self.assertEqual(writer.size(), 0) writer.write("01234567") @@ -297,7 +298,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase): api = ArvadosFileWriterTestCase.MockApi({"name":"test_create_multiple", "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:8:count1.txt 8:8:count2.txt\n"}, {"uuid":"zzzzz-4zz18-mockcollection0"}) - with arvados.Collection(api_client=api, keep_client=keep) as c: + with arvados.Collection(api_client=api, keep_client=keep, sync=SYNC_EXPLICIT) as c: w1 = c.open("count1.txt", "w") w2 = c.open("count2.txt", "w") w1.write("01234567") @@ -313,31 +314,6 @@ class ArvadosFileWriterTestCase(unittest.TestCase): self.assertEqual(False, c.modified()) self.assertEqual("01234567", keep.get("2e9ec317e197819358fbc43afca7d837+8")) - def test_remove(self): - with import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n') as c: - self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n", export_manifest(c)) - self.assertTrue("count1.txt" in c) - c.remove("count1.txt") - self.assertFalse("count1.txt" in c) - self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n", export_manifest(c)) - - def test_remove_in_subdir(self): - with import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n') as c: - c.remove("foo/count2.txt") - self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", export_manifest(c)) - - def test_remove_subdir(self): - with import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n') as c: - c.remove("foo") - self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", export_manifest(c)) - - def test_prefetch(self): - keep = ArvadosFileWriterTestCase.MockKeep({"2e9ec317e197819358fbc43afca7d837+8": "01234567", "e8dc4081b13434b45189a720b77b6818+8": "abcdefgh"}) - with import_manifest(". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:16:count.txt\n", keep=keep) as c: - r = c.open("count.txt", "r") - self.assertEqual("0123", r.read(4)) - self.assertEqual(["2e9ec317e197819358fbc43afca7d837+8", "2e9ec317e197819358fbc43afca7d837+8", "e8dc4081b13434b45189a720b77b6818+8"], keep.requests) - class ArvadosFileReaderTestCase(StreamFileReaderTestCase): class MockParent(object): @@ -361,6 +337,9 @@ class ArvadosFileReaderTestCase(StreamFileReaderTestCase): def _my_block_manager(self): return ArvadosFileReaderTestCase.MockParent.MockBlockMgr(self.blocks, self.nocache) + def sync_mode(self): + return SYNC_READONLY + def make_count_reader(self, nocache=False): stream = [] n = 0 @@ -388,6 +367,14 @@ class ArvadosFileReaderTestCase(StreamFileReaderTestCase): sfile.read(5) self.assertEqual(3, sfile.tell()) + def test_prefetch(self): + keep = ArvadosFileWriterTestCase.MockKeep({"2e9ec317e197819358fbc43afca7d837+8": "01234567", "e8dc4081b13434b45189a720b77b6818+8": "abcdefgh"}) + with import_manifest(". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:16:count.txt\n", keep=keep) as c: + r = c.open("count.txt", "r") + self.assertEqual("0123", r.read(4)) + self.assertTrue("2e9ec317e197819358fbc43afca7d837+8" in keep.requests) + self.assertTrue("e8dc4081b13434b45189a720b77b6818+8" in keep.requests) + class ArvadosFileReadTestCase(unittest.TestCase, StreamRetryTestMixin): def reader_for(self, coll_name, **kwargs): @@ -402,7 +389,9 @@ class ArvadosFileReadTestCase(unittest.TestCase, StreamRetryTestMixin): n += k.size except ValueError: pass - af = ArvadosFile(arvados.Collection(keep_client=self.keep_client()), + col = arvados.Collection(keep_client=self.keep_client()) + col._my_block_manager().prefetch_enabled = False + af = ArvadosFile(col, stream=stream, segments=segments) return ArvadosFileReader(af, "test", **kwargs)