import hashlib
import arvados
-from arvados import StreamReader, StreamFileReader, Range, import_manifest, export_manifest
+from arvados import ArvadosFile, ArvadosFileReader, Range, import_manifest, export_manifest, KeepLocator
+from arvados.arvfile import SYNC_READONLY, SYNC_EXPLICIT
import arvados_testutil as tutil
-
+from test_stream import StreamFileReaderTestCase, StreamRetryTestMixin
class ArvadosFileWriterTestCase(unittest.TestCase):
class MockKeep(object):
def __init__(self, blocks):
self.blocks = blocks
self.requests = []
- def get(self, locator, num_retries=0):
+ def get(self, locator, num_retries=0, cache_only=False):
self.requests.append(locator)
return self.blocks.get(locator)
def put(self, data):
"manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n"},
{"uuid":"zzzzz-4zz18-mockcollection0"})
with import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
- api_client=api, keep=keep) as c:
+ api_client=api, keep=keep, sync=SYNC_EXPLICIT) as c:
writer = c.open("count.txt", "r+")
self.assertEqual(writer.size(), 10)
writer.seek(5)
"manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n"},
{"uuid":"zzzzz-4zz18-mockcollection0"})
with import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
- api_client=api, keep=keep) as c:
+ api_client=api, keep=keep, sync=SYNC_EXPLICIT) as c:
writer = c.open("count.txt", "r+")
writer.seek(5, os.SEEK_SET)
self.assertEqual("56789", writer.read(8))
def test_write0(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
with import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
- keep=keep) as c:
+ keep=keep, sync=SYNC_EXPLICIT) as c:
writer = c.open("count.txt", "r+")
self.assertEqual("0123456789", writer.readfrom(0, 13))
writer.seek(0, os.SEEK_SET)
def test_write1(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
with import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
- keep=keep) as c:
+ keep=keep, sync=SYNC_EXPLICIT) as c:
writer = c.open("count.txt", "r+")
self.assertEqual("0123456789", writer.readfrom(0, 13))
writer.seek(3, os.SEEK_SET)
def test_write2(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
with import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
- keep=keep) as c:
+ keep=keep, sync=SYNC_EXPLICIT) as c:
writer = c.open("count.txt", "r+")
self.assertEqual("0123456789", writer.readfrom(0, 13))
writer.seek(7, os.SEEK_SET)
def test_write3(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
with import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt 0:10:count.txt\n',
- keep=keep) as c:
+ keep=keep, sync=SYNC_EXPLICIT) as c:
writer = c.open("count.txt", "r+")
self.assertEqual("012345678901234", writer.readfrom(0, 15))
writer.seek(7, os.SEEK_SET)
def test_write4(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
with import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:4:count.txt 0:4:count.txt 0:4:count.txt',
- keep=keep) as c:
+ keep=keep, sync=SYNC_EXPLICIT) as c:
writer = c.open("count.txt", "r+")
self.assertEqual("012301230123", writer.readfrom(0, 15))
writer.seek(2, os.SEEK_SET)
"manifest_text": ". a5de24f4417cfba9d5825eadc2f4ca49+67108000 598cc1a4ccaef8ab6e4724d87e675d78+32892000 0:100000000:count.txt\n"},
{"uuid":"zzzzz-4zz18-mockcollection0"})
with import_manifest('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
- api_client=api, keep=keep) as c:
+ api_client=api, keep=keep, sync=SYNC_EXPLICIT) as c:
writer = c.open("count.txt", "r+")
text = ''.join(["0123456789" for a in xrange(0, 100)])
for b in xrange(0, 100000):
def test_write_rewrite0(self):
keep = ArvadosFileWriterTestCase.MockKeep({})
with import_manifest('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
- keep=keep) as c:
+ keep=keep, sync=SYNC_EXPLICIT) as c:
writer = c.open("count.txt", "r+")
for b in xrange(0, 10):
writer.seek(0, os.SEEK_SET)
def test_write_rewrite1(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
with import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
- keep=keep) as c:
+ keep=keep, sync=SYNC_EXPLICIT) as c:
writer = c.open("count.txt", "r+")
for b in xrange(0, 10):
writer.seek(10, os.SEEK_SET)
def test_write_rewrite2(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
with import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
- keep=keep) as c:
+ keep=keep, sync=SYNC_EXPLICIT) as c:
writer = c.open("count.txt", "r+")
for b in xrange(0, 10):
writer.seek(5, os.SEEK_SET)
"manifest_text": ". 37400a68af9abdd76ca5bf13e819e42a+32892003 a5de24f4417cfba9d5825eadc2f4ca49+67108000 32892000:3:count.txt 32892006:67107997:count.txt 0:32892000:count.txt\n"},
{"uuid":"zzzzz-4zz18-mockcollection0"})
with import_manifest('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
- api_client=api, keep=keep) as c:
+ api_client=api, keep=keep, sync=SYNC_EXPLICIT) as c:
writer = c.open("count.txt", "r+")
text = ''.join(["0123456789" for a in xrange(0, 100)])
for b in xrange(0, 100000):
api = ArvadosFileWriterTestCase.MockApi({"name":"test_create",
"manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"},
{"uuid":"zzzzz-4zz18-mockcollection0"})
- with arvados.Collection(api_client=api, keep_client=keep) as c:
+ with arvados.Collection(api_client=api, keep_client=keep, sync=SYNC_EXPLICIT) as c:
writer = c.open("count.txt", "w+")
self.assertEqual(writer.size(), 0)
writer.write("01234567")
api = ArvadosFileWriterTestCase.MockApi({"name":"test_create",
"manifest_text":"./foo/bar 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"},
{"uuid":"zzzzz-4zz18-mockcollection0"})
- with arvados.Collection(api_client=api, keep_client=keep) as c:
+ with arvados.Collection(api_client=api, keep_client=keep, sync=SYNC_EXPLICIT) as c:
writer = c.open("foo/bar/count.txt", "w+")
writer.write("01234567")
c.save_as("test_create")
"manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"},
{"uuid":"zzzzz-4zz18-mockcollection0"})
with import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
- api_client=api, keep=keep) as c:
+ api_client=api, keep=keep, sync=SYNC_EXPLICIT) as c:
writer = c.open("count.txt", "w+")
self.assertEqual(writer.size(), 0)
writer.write("01234567")
api = ArvadosFileWriterTestCase.MockApi({"name":"test_create_multiple",
"manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:8:count1.txt 8:8:count2.txt\n"},
{"uuid":"zzzzz-4zz18-mockcollection0"})
- with arvados.Collection(api_client=api, keep_client=keep) as c:
+ with arvados.Collection(api_client=api, keep_client=keep, sync=SYNC_EXPLICIT) as c:
w1 = c.open("count1.txt", "w")
w2 = c.open("count2.txt", "w")
w1.write("01234567")
self.assertEqual(False, c.modified())
self.assertEqual("01234567", keep.get("2e9ec317e197819358fbc43afca7d837+8"))
- def test_remove(self):
- with import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n') as c:
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n", export_manifest(c))
- self.assertTrue("count1.txt" in c)
- c.remove("count1.txt")
- self.assertFalse("count1.txt" in c)
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n", export_manifest(c))
- def test_remove_in_subdir(self):
- with import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n') as c:
- c.remove("foo/count2.txt")
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", export_manifest(c))
+class ArvadosFileReaderTestCase(StreamFileReaderTestCase):
+ class MockParent(object):
+ class MockBlockMgr(object):
+ def __init__(self, blocks, nocache):
+ self.blocks = blocks
+ self.nocache = nocache
+
+ def block_prefetch(self, loc):
+ pass
- def test_remove_subdir(self):
- with import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n') as c:
- c.remove("foo")
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", export_manifest(c))
+ def get_block(self, loc, num_retries=0, cache_only=False):
+ if self.nocache and cache_only:
+ return None
+ return self.blocks[loc]
+
+ def __init__(self, blocks, nocache):
+ self.blocks = blocks
+ self.nocache = nocache
+
+ def _my_block_manager(self):
+ return ArvadosFileReaderTestCase.MockParent.MockBlockMgr(self.blocks, self.nocache)
+
+ def sync_mode(self):
+ return SYNC_READONLY
+
+ def make_count_reader(self, nocache=False):
+ stream = []
+ n = 0
+ blocks = {}
+ for d in ['01234', '34567', '67890']:
+ loc = '{}+{}'.format(hashlib.md5(d).hexdigest(), len(d))
+ blocks[loc] = d
+ stream.append(Range(loc, n, len(d)))
+ n += len(d)
+ af = ArvadosFile(ArvadosFileReaderTestCase.MockParent(blocks, nocache), stream=stream, segments=[Range(1, 0, 3), Range(6, 3, 3), Range(11, 6, 3)])
+ return ArvadosFileReader(af, "count.txt")
+
+ def test_read_returns_first_block(self):
+ # read() calls will be aligned on block boundaries - see #3663.
+ sfile = self.make_count_reader(nocache=True)
+ self.assertEqual('123', sfile.read(10))
+
+ def test_successive_reads(self):
+ sfile = self.make_count_reader(nocache=True)
+ for expect in ['123', '456', '789', '']:
+ self.assertEqual(expect, sfile.read(10))
+
+ def test_tell_after_block_read(self):
+ sfile = self.make_count_reader(nocache=True)
+ sfile.read(5)
+ self.assertEqual(3, sfile.tell())
def test_prefetch(self):
keep = ArvadosFileWriterTestCase.MockKeep({"2e9ec317e197819358fbc43afca7d837+8": "01234567", "e8dc4081b13434b45189a720b77b6818+8": "abcdefgh"})
with import_manifest(". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:16:count.txt\n", keep=keep) as c:
r = c.open("count.txt", "r")
self.assertEqual("0123", r.read(4))
- self.assertEqual(["2e9ec317e197819358fbc43afca7d837+8", "2e9ec317e197819358fbc43afca7d837+8", "e8dc4081b13434b45189a720b77b6818+8"], keep.requests)
+ self.assertTrue("2e9ec317e197819358fbc43afca7d837+8" in keep.requests)
+ self.assertTrue("e8dc4081b13434b45189a720b77b6818+8" in keep.requests)
+
+
+class ArvadosFileReadTestCase(unittest.TestCase, StreamRetryTestMixin):
+ def reader_for(self, coll_name, **kwargs):
+ stream = []
+ segments = []
+ n = 0
+ for d in self.manifest_for(coll_name).split():
+ try:
+ k = KeepLocator(d)
+ segments.append(Range(n, n, k.size))
+ stream.append(Range(d, n, k.size))
+ n += k.size
+ except ValueError:
+ pass
+ col = arvados.Collection(keep_client=self.keep_client())
+ col._my_block_manager().prefetch_enabled = False
+ af = ArvadosFile(col,
+ stream=stream,
+ segments=segments)
+ return ArvadosFileReader(af, "test", **kwargs)
+
+ def read_for_test(self, reader, byte_count, **kwargs):
+ return reader.read(byte_count, **kwargs)
+
+
+class ArvadosFileReadFromTestCase(ArvadosFileReadTestCase):
+ def read_for_test(self, reader, byte_count, **kwargs):
+ return reader.readfrom(0, byte_count, **kwargs)
+
+
+class ArvadosFileReadAllTestCase(ArvadosFileReadTestCase):
+ def read_for_test(self, reader, byte_count, **kwargs):
+ return ''.join(reader.readall(**kwargs))
+
+
+class ArvadosFileReadAllDecompressedTestCase(ArvadosFileReadTestCase):
+ def read_for_test(self, reader, byte_count, **kwargs):
+ return ''.join(reader.readall_decompressed(**kwargs))
+
+
+class ArvadosFileReadlinesTestCase(ArvadosFileReadTestCase):
+ def read_for_test(self, reader, byte_count, **kwargs):
+ return ''.join(reader.readlines(**kwargs))
projects / arvados.git / blobdiff