12 from arvados._ranges import Range
13 from arvados.keep import KeepLocator
14 from arvados.collection import Collection, CollectionReader
15 from arvados.arvfile import ArvadosFile, ArvadosFileReader
17 import arvados_testutil as tutil
18 from test_stream import StreamFileReaderTestCase, StreamRetryTestMixin
20 class ArvadosFileWriterTestCase(unittest.TestCase):
21 class MockKeep(object):
22 def __init__(self, blocks):
25 def get(self, locator, num_retries=0):
26 self.requests.append(locator)
27 return self.blocks.get(locator)
28 def get_from_cache(self, locator):
29 self.requests.append(locator)
30 return self.blocks.get(locator)
31 def put(self, data, num_retries=None):
32 pdh = "%s+%i" % (hashlib.md5(data).hexdigest(), len(data))
33 self.blocks[pdh] = str(data)
36 class MockApi(object):
37 def __init__(self, b, r):
40 class MockCollections(object):
41 def __init__(self, b, r):
44 class Execute(object):
45 def __init__(self, r):
47 def execute(self, num_retries=None):
49 def create(self, ensure_unique_name=False, body=None):
51 raise Exception("Body %s does not match expectation %s" % (body, self.body))
52 return ArvadosFileWriterTestCase.MockApi.MockCollections.Execute(self.response)
53 def update(self, uuid=None, body=None):
54 return ArvadosFileWriterTestCase.MockApi.MockCollections.Execute(self.response)
55 def collections(self):
56 return ArvadosFileWriterTestCase.MockApi.MockCollections(self.body, self.response)
59 def test_truncate(self):
60 keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
61 api = ArvadosFileWriterTestCase.MockApi({"name":"test_truncate",
62 "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n"},
63 {"uuid":"zzzzz-4zz18-mockcollection0",
64 "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n"})
65 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
66 api_client=api, keep_client=keep) as c:
67 writer = c.open("count.txt", "r+")
68 self.assertEqual(writer.size(), 10)
69 self.assertEqual("0123456789", writer.read(12))
73 # Make sure reading off the end doesn't break
74 self.assertEqual("", writer.read(12))
76 self.assertEqual(writer.size(), 8)
77 writer.seek(0, os.SEEK_SET)
78 self.assertEqual("01234567", writer.read(12))
80 self.assertEqual(None, c.manifest_locator())
81 self.assertEqual(True, c.modified())
82 c.save_new("test_truncate")
83 self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
84 self.assertEqual(False, c.modified())
86 def test_write_to_end(self):
87 keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
88 api = ArvadosFileWriterTestCase.MockApi({"name":"test_append",
89 "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n"},
90 {"uuid":"zzzzz-4zz18-mockcollection0",
91 "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n"})
92 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
93 api_client=api, keep_client=keep) as c:
94 writer = c.open("count.txt", "r+")
95 self.assertEqual(writer.size(), 10)
97 writer.seek(5, os.SEEK_SET)
98 self.assertEqual("56789", writer.read(8))
100 writer.seek(10, os.SEEK_SET)
102 self.assertEqual(writer.size(), 13)
104 writer.seek(5, os.SEEK_SET)
105 self.assertEqual("56789foo", writer.read(8))
107 self.assertEqual(None, c.manifest_locator())
108 self.assertEqual(True, c.modified())
109 self.assertEqual(None, keep.get("acbd18db4cc2f85cedef654fccc4a4d8+3"))
111 c.save_new("test_append")
112 self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
113 self.assertEqual(False, c.modified())
114 self.assertEqual("foo", keep.get("acbd18db4cc2f85cedef654fccc4a4d8+3"))
117 def test_append(self):
118 keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
119 c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', keep_client=keep)
120 writer = c.open("count.txt", "a+")
121 self.assertEqual(writer.read(20), "0123456789")
122 writer.seek(0, os.SEEK_SET)
124 writer.write("hello")
125 self.assertEqual(writer.read(20), "0123456789hello")
126 writer.seek(0, os.SEEK_SET)
128 writer.write("world")
129 self.assertEqual(writer.read(20), "0123456789helloworld")
131 self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 fc5e038d38a57032085441e7fe7010b0+10 0:20:count.txt\n", c.manifest_text())
133 def test_write_at_beginning(self):
134 keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
135 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
136 keep_client=keep) as c:
137 writer = c.open("count.txt", "r+")
138 self.assertEqual("0123456789", writer.readfrom(0, 13))
139 writer.seek(0, os.SEEK_SET)
141 self.assertEqual(writer.size(), 10)
142 self.assertEqual("foo3456789", writer.readfrom(0, 13))
143 self.assertEqual(". acbd18db4cc2f85cedef654fccc4a4d8+3 781e5e245d69b566979b86e28d23f2c7+10 0:3:count.txt 6:7:count.txt\n", c.manifest_text())
145 def test_write_empty(self):
146 keep = ArvadosFileWriterTestCase.MockKeep({})
147 with Collection(keep_client=keep) as c:
148 writer = c.open("count.txt", "w")
149 self.assertEqual(writer.size(), 0)
150 self.assertEqual(". d41d8cd98f00b204e9800998ecf8427e+0 0:0:count.txt\n", c.manifest_text())
152 def test_get_manifest_text_commits(self):
153 keep = ArvadosFileWriterTestCase.MockKeep({})
154 with Collection(keep_client=keep) as c:
155 writer = c.open("count.txt", "w")
156 writer.write("0123456789")
157 self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.manifest_text(strip=True))
158 self.assertNotIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks)
159 self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.manifest_text(strip=False))
160 self.assertIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks)
162 def test_write_in_middle(self):
163 keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
164 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
165 keep_client=keep) as c:
166 writer = c.open("count.txt", "r+")
167 self.assertEqual("0123456789", writer.readfrom(0, 13))
168 writer.seek(3, os.SEEK_SET)
170 self.assertEqual(writer.size(), 10)
171 self.assertEqual("012foo6789", writer.readfrom(0, 13))
172 self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:count.txt 10:3:count.txt 6:4:count.txt\n", c.manifest_text())
174 def test_write_at_end(self):
175 keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
176 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
177 keep_client=keep) as c:
178 writer = c.open("count.txt", "r+")
179 self.assertEqual("0123456789", writer.readfrom(0, 13))
180 writer.seek(7, os.SEEK_SET)
182 self.assertEqual(writer.size(), 10)
183 self.assertEqual("0123456foo", writer.readfrom(0, 13))
184 self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:7:count.txt 10:3:count.txt\n", c.manifest_text())
186 def test_write_across_segment_boundary(self):
187 keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
188 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt 0:10:count.txt\n',
189 keep_client=keep) as c:
190 writer = c.open("count.txt", "r+")
191 self.assertEqual("012345678901234", writer.readfrom(0, 15))
192 writer.seek(7, os.SEEK_SET)
193 writer.write("foobar")
194 self.assertEqual(writer.size(), 20)
195 self.assertEqual("0123456foobar34", writer.readfrom(0, 15))
196 self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 3858f62230ac3c915f300c664312c63f+6 0:7:count.txt 10:6:count.txt 3:7:count.txt\n", c.manifest_text())
198 def test_write_across_several_segments(self):
199 keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
200 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:4:count.txt 0:4:count.txt 0:4:count.txt',
201 keep_client=keep) as c:
202 writer = c.open("count.txt", "r+")
203 self.assertEqual("012301230123", writer.readfrom(0, 15))
204 writer.seek(2, os.SEEK_SET)
205 writer.write("abcdefg")
206 self.assertEqual(writer.size(), 12)
207 self.assertEqual("01abcdefg123", writer.readfrom(0, 15))
208 self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 7ac66c0f148de9519b8bd264312c4d64+7 0:2:count.txt 10:7:count.txt 1:3:count.txt\n", c.manifest_text())
210 def test_write_large(self):
211 keep = ArvadosFileWriterTestCase.MockKeep({})
212 api = ArvadosFileWriterTestCase.MockApi({"name":"test_write_large",
213 "manifest_text": ". a5de24f4417cfba9d5825eadc2f4ca49+67108000 598cc1a4ccaef8ab6e4724d87e675d78+32892000 0:100000000:count.txt\n"},
214 {"uuid":"zzzzz-4zz18-mockcollection0",
215 "manifest_text": ". a5de24f4417cfba9d5825eadc2f4ca49+67108000 598cc1a4ccaef8ab6e4724d87e675d78+32892000 0:100000000:count.txt\n"})
216 with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
217 api_client=api, keep_client=keep) as c:
218 writer = c.open("count.txt", "r+")
219 text = ''.join(["0123456789" for a in xrange(0, 100)])
220 for b in xrange(0, 100000):
222 self.assertEqual(writer.size(), 100000000)
224 self.assertEqual(None, c.manifest_locator())
225 self.assertEqual(True, c.modified())
226 c.save_new("test_write_large")
227 self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
228 self.assertEqual(False, c.modified())
230 def test_rewrite_on_empty_file(self):
231 keep = ArvadosFileWriterTestCase.MockKeep({})
232 with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
233 keep_client=keep) as c:
234 writer = c.open("count.txt", "r+")
235 for b in xrange(0, 10):
236 writer.seek(0, os.SEEK_SET)
237 writer.write("0123456789")
239 self.assertEqual(writer.size(), 10)
240 self.assertEqual("0123456789", writer.readfrom(0, 20))
241 self.assertEqual(". 7a08b07e84641703e5f2c836aa59a170+100 90:10:count.txt\n", c.manifest_text(strip=True))
243 self.assertEqual(writer.size(), 10)
244 self.assertEqual("0123456789", writer.readfrom(0, 20))
245 self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n", c.manifest_text(strip=True))
247 def test_rewrite_append_existing_file(self):
248 keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
249 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
250 keep_client=keep) as c:
251 writer = c.open("count.txt", "r+")
252 for b in xrange(0, 10):
253 writer.seek(10, os.SEEK_SET)
254 writer.write("abcdefghij")
256 self.assertEqual(writer.size(), 20)
257 self.assertEqual("0123456789abcdefghij", writer.readfrom(0, 20))
258 self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 ae5f43bab79cf0be33f025fa97ae7398+100 0:10:count.txt 100:10:count.txt\n", c.manifest_text(strip=True))
260 writer.arvadosfile.flush()
261 self.assertEqual(writer.size(), 20)
262 self.assertEqual("0123456789abcdefghij", writer.readfrom(0, 20))
263 self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:20:count.txt\n", c.manifest_text(strip=True))
265 def test_rewrite_over_existing_file(self):
266 keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
267 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
268 keep_client=keep) as c:
269 writer = c.open("count.txt", "r+")
270 for b in xrange(0, 10):
271 writer.seek(5, os.SEEK_SET)
272 writer.write("abcdefghij")
274 self.assertEqual(writer.size(), 15)
275 self.assertEqual("01234abcdefghij", writer.readfrom(0, 20))
276 self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 ae5f43bab79cf0be33f025fa97ae7398+100 0:5:count.txt 100:10:count.txt\n", c.manifest_text(strip=True))
278 writer.arvadosfile.flush()
280 self.assertEqual(writer.size(), 15)
281 self.assertEqual("01234abcdefghij", writer.readfrom(0, 20))
282 self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:5:count.txt 10:10:count.txt\n", c.manifest_text(strip=True))
284 def test_write_large_rewrite(self):
285 keep = ArvadosFileWriterTestCase.MockKeep({})
286 api = ArvadosFileWriterTestCase.MockApi({"name":"test_write_large",
287 "manifest_text": ". 37400a68af9abdd76ca5bf13e819e42a+32892003 a5de24f4417cfba9d5825eadc2f4ca49+67108000 32892000:3:count.txt 32892006:67107997:count.txt 0:32892000:count.txt\n"},
288 {"uuid":"zzzzz-4zz18-mockcollection0",
289 "manifest_text": ". 37400a68af9abdd76ca5bf13e819e42a+32892003 a5de24f4417cfba9d5825eadc2f4ca49+67108000 32892000:3:count.txt 32892006:67107997:count.txt 0:32892000:count.txt\n"})
290 with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
291 api_client=api, keep_client=keep) as c:
292 writer = c.open("count.txt", "r+")
293 text = ''.join(["0123456789" for a in xrange(0, 100)])
294 for b in xrange(0, 100000):
296 writer.seek(0, os.SEEK_SET)
298 self.assertEqual(writer.size(), 100000000)
300 self.assertEqual(None, c.manifest_locator())
301 self.assertEqual(True, c.modified())
302 c.save_new("test_write_large")
303 self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
304 self.assertEqual(False, c.modified())
306 def test_create(self):
307 keep = ArvadosFileWriterTestCase.MockKeep({})
308 api = ArvadosFileWriterTestCase.MockApi({"name":"test_create",
309 "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"},
310 {"uuid":"zzzzz-4zz18-mockcollection0",
311 "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"})
312 with Collection(api_client=api, keep_client=keep) as c:
313 writer = c.open("count.txt", "w+")
314 self.assertEqual(writer.size(), 0)
315 writer.write("01234567")
316 self.assertEqual(writer.size(), 8)
318 self.assertEqual(None, c.manifest_locator())
319 self.assertEqual(True, c.modified())
320 self.assertEqual(None, keep.get("2e9ec317e197819358fbc43afca7d837+8"))
321 c.save_new("test_create")
322 self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
323 self.assertEqual(False, c.modified())
324 self.assertEqual("01234567", keep.get("2e9ec317e197819358fbc43afca7d837+8"))
327 def test_create_subdir(self):
328 keep = ArvadosFileWriterTestCase.MockKeep({})
329 api = ArvadosFileWriterTestCase.MockApi({"name":"test_create",
330 "manifest_text":"./foo/bar 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"},
331 {"uuid":"zzzzz-4zz18-mockcollection0",
332 "manifest_text":"./foo/bar 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"})
333 with Collection(api_client=api, keep_client=keep) as c:
334 self.assertIsNone(c.api_response())
335 writer = c.open("foo/bar/count.txt", "w+")
336 writer.write("01234567")
337 c.save_new("test_create")
338 self.assertEqual(c.api_response(), api.response)
340 def test_overwrite(self):
341 keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
342 api = ArvadosFileWriterTestCase.MockApi({"name":"test_overwrite",
343 "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"},
344 {"uuid":"zzzzz-4zz18-mockcollection0",
345 "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"})
346 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
347 api_client=api, keep_client=keep) as c:
348 writer = c.open("count.txt", "w+")
349 self.assertEqual(writer.size(), 0)
350 writer.write("01234567")
351 self.assertEqual(writer.size(), 8)
353 self.assertEqual(None, c.manifest_locator())
354 self.assertEqual(True, c.modified())
355 c.save_new("test_overwrite")
356 self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
357 self.assertEqual(False, c.modified())
359 def test_file_not_found(self):
360 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n') as c:
361 with self.assertRaises(IOError):
362 writer = c.open("nocount.txt", "r")
364 def test_cannot_open_directory(self):
365 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n') as c:
366 with self.assertRaises(IOError):
367 writer = c.open(".", "r")
369 def test_create_multiple(self):
370 keep = ArvadosFileWriterTestCase.MockKeep({})
371 api = ArvadosFileWriterTestCase.MockApi({"name":"test_create_multiple",
372 "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:8:count1.txt 8:8:count2.txt\n"},
373 {"uuid":"zzzzz-4zz18-mockcollection0",
374 "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:8:count1.txt 8:8:count2.txt\n"})
375 with Collection(api_client=api, keep_client=keep) as c:
376 w1 = c.open("count1.txt", "w")
377 w2 = c.open("count2.txt", "w")
380 self.assertEqual(w1.size(), 8)
381 self.assertEqual(w2.size(), 8)
383 self.assertEqual(None, c.manifest_locator())
384 self.assertEqual(True, c.modified())
385 self.assertEqual(None, keep.get("2e9ec317e197819358fbc43afca7d837+8"))
386 c.save_new("test_create_multiple")
387 self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
388 self.assertEqual(False, c.modified())
389 self.assertEqual("01234567", keep.get("2e9ec317e197819358fbc43afca7d837+8"))
392 class ArvadosFileReaderTestCase(StreamFileReaderTestCase):
393 class MockParent(object):
394 class MockBlockMgr(object):
395 def __init__(self, blocks, nocache):
397 self.nocache = nocache
399 def block_prefetch(self, loc):
402 def get_block_contents(self, loc, num_retries=0, cache_only=False):
403 if self.nocache and cache_only:
405 return self.blocks[loc]
407 def __init__(self, blocks, nocache):
409 self.nocache = nocache
410 self.lock = arvados.arvfile.NoopLock()
412 def root_collection(self):
415 def _my_block_manager(self):
416 return ArvadosFileReaderTestCase.MockParent.MockBlockMgr(self.blocks, self.nocache)
419 def make_count_reader(self, nocache=False):
423 for d in ['01234', '34567', '67890']:
424 loc = '{}+{}'.format(hashlib.md5(d).hexdigest(), len(d))
426 stream.append(Range(loc, n, len(d)))
428 af = ArvadosFile(ArvadosFileReaderTestCase.MockParent(blocks, nocache), stream=stream, segments=[Range(1, 0, 3), Range(6, 3, 3), Range(11, 6, 3)])
429 return ArvadosFileReader(af, "count.txt")
431 def test_read_returns_first_block(self):
432 # read() calls will be aligned on block boundaries - see #3663.
433 sfile = self.make_count_reader(nocache=True)
434 self.assertEqual('123', sfile.read(10))
436 def test_successive_reads(self):
437 sfile = self.make_count_reader(nocache=True)
438 for expect in ['123', '456', '789', '']:
439 self.assertEqual(expect, sfile.read(10))
441 def test_tell_after_block_read(self):
442 sfile = self.make_count_reader(nocache=True)
444 self.assertEqual(3, sfile.tell())
446 def test_prefetch(self):
447 keep = ArvadosFileWriterTestCase.MockKeep({"2e9ec317e197819358fbc43afca7d837+8": "01234567", "e8dc4081b13434b45189a720b77b6818+8": "abcdefgh"})
448 with Collection(". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:16:count.txt\n", keep_client=keep) as c:
449 r = c.open("count.txt", "r")
450 self.assertEqual("0123", r.read(4))
451 self.assertIn("2e9ec317e197819358fbc43afca7d837+8", keep.requests)
452 self.assertIn("e8dc4081b13434b45189a720b77b6818+8", keep.requests)
454 def test__eq__from_manifest(self):
455 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
456 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c2:
457 self.assertTrue(c1["count1.txt"] == c2["count1.txt"])
458 self.assertFalse(c1["count1.txt"] != c2["count1.txt"])
460 def test__eq__from_writes(self):
461 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
462 with Collection() as c2:
463 with c2.open("count1.txt", "w") as f:
464 f.write("0123456789")
466 self.assertTrue(c1["count1.txt"] == c2["count1.txt"])
467 self.assertFalse(c1["count1.txt"] != c2["count1.txt"])
469 def test__ne__(self):
470 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
471 with Collection() as c2:
472 with c2.open("count1.txt", "w") as f:
473 f.write("1234567890")
475 self.assertTrue(c1["count1.txt"] != c2["count1.txt"])
476 self.assertFalse(c1["count1.txt"] == c2["count1.txt"])
479 class ArvadosFileReadTestCase(unittest.TestCase, StreamRetryTestMixin):
480 def reader_for(self, coll_name, **kwargs):
484 for d in self.manifest_for(coll_name).split():
487 segments.append(Range(n, n, k.size))
488 stream.append(Range(d, n, k.size))
493 blockmanager = arvados.arvfile._BlockManager(self.keep_client())
494 blockmanager.prefetch_enabled = False
495 col = Collection(keep_client=self.keep_client(), block_manager=blockmanager)
496 af = ArvadosFile(col,
499 return ArvadosFileReader(af, "test", **kwargs)
501 def read_for_test(self, reader, byte_count, **kwargs):
502 return reader.read(byte_count, **kwargs)
505 class ArvadosFileReadFromTestCase(ArvadosFileReadTestCase):
506 def read_for_test(self, reader, byte_count, **kwargs):
507 return reader.readfrom(0, byte_count, **kwargs)
510 class ArvadosFileReadAllTestCase(ArvadosFileReadTestCase):
511 def read_for_test(self, reader, byte_count, **kwargs):
512 return ''.join(reader.readall(**kwargs))
515 class ArvadosFileReadAllDecompressedTestCase(ArvadosFileReadTestCase):
516 def read_for_test(self, reader, byte_count, **kwargs):
517 return ''.join(reader.readall_decompressed(**kwargs))
520 class ArvadosFileReadlinesTestCase(ArvadosFileReadTestCase):
521 def read_for_test(self, reader, byte_count, **kwargs):
522 return ''.join(reader.readlines(**kwargs))
524 class BlockManagerTest(unittest.TestCase):
525 def test_bufferblock_append(self):
526 keep = ArvadosFileWriterTestCase.MockKeep({})
527 blockmanager = arvados.arvfile._BlockManager(keep)
528 bufferblock = blockmanager.alloc_bufferblock()
529 bufferblock.append("foo")
531 self.assertEqual(bufferblock.size(), 3)
532 self.assertEqual(bufferblock.buffer_view[0:3], "foo")
533 self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3")
535 bufferblock.append("bar")
537 self.assertEqual(bufferblock.size(), 6)
538 self.assertEqual(bufferblock.buffer_view[0:6], "foobar")
539 self.assertEqual(bufferblock.locator(), "3858f62230ac3c915f300c664312c63f+6")
541 bufferblock.set_state(arvados.arvfile._BufferBlock.PENDING)
542 with self.assertRaises(arvados.errors.AssertionError):
543 bufferblock.append("bar")
545 def test_bufferblock_dup(self):
546 keep = ArvadosFileWriterTestCase.MockKeep({})
547 blockmanager = arvados.arvfile._BlockManager(keep)
548 bufferblock = blockmanager.alloc_bufferblock()
549 bufferblock.append("foo")
551 self.assertEqual(bufferblock.size(), 3)
552 self.assertEqual(bufferblock.buffer_view[0:3], "foo")
553 self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3")
554 bufferblock.set_state(arvados.arvfile._BufferBlock.PENDING)
556 bufferblock2 = blockmanager.dup_block(bufferblock, None)
557 self.assertNotEqual(bufferblock.blockid, bufferblock2.blockid)
559 bufferblock2.append("bar")
561 self.assertEqual(bufferblock2.size(), 6)
562 self.assertEqual(bufferblock2.buffer_view[0:6], "foobar")
563 self.assertEqual(bufferblock2.locator(), "3858f62230ac3c915f300c664312c63f+6")
565 self.assertEqual(bufferblock.size(), 3)
566 self.assertEqual(bufferblock.buffer_view[0:3], "foo")
567 self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3")
569 def test_bufferblock_get(self):
570 keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
571 blockmanager = arvados.arvfile._BlockManager(keep)
572 bufferblock = blockmanager.alloc_bufferblock()
573 bufferblock.append("foo")
575 self.assertEqual(blockmanager.get_block_contents("781e5e245d69b566979b86e28d23f2c7+10", 1), "0123456789")
576 self.assertEqual(blockmanager.get_block_contents(bufferblock.blockid, 1), "foo")
578 def test_bufferblock_commit(self):
579 mockkeep = mock.MagicMock()
580 blockmanager = arvados.arvfile._BlockManager(mockkeep)
581 bufferblock = blockmanager.alloc_bufferblock()
582 bufferblock.owner = ArvadosFile(Collection(block_manager=blockmanager),
583 [Range(bufferblock.blockid, 0, 3)], [Range(0, 0, 3)])
584 bufferblock.owner._current_bblock = bufferblock
585 bufferblock.append("foo")
586 blockmanager.commit_all()
587 self.assertTrue(mockkeep.put.called)
588 self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.COMMITTED)
589 self.assertIsNone(bufferblock.buffer_view)
592 def test_bufferblock_commit_with_error(self):
593 mockkeep = mock.MagicMock()
594 mockkeep.put.side_effect = arvados.errors.KeepWriteError("fail")
595 blockmanager = arvados.arvfile._BlockManager(mockkeep)
596 bufferblock = blockmanager.alloc_bufferblock()
597 bufferblock.owner = ArvadosFile(Collection(block_manager=blockmanager),
598 [Range(bufferblock.blockid, 0, 3)], [Range(0, 0, 3)])
599 bufferblock.owner._current_bblock = bufferblock
600 bufferblock.append("foo")
601 with self.assertRaises(arvados.errors.KeepWriteError) as err:
602 blockmanager.commit_all()
603 self.assertEqual(str(err.exception), "Error writing some blocks: block acbd18db4cc2f85cedef654fccc4a4d8+3 raised KeepWriteError (fail)")
604 self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.PENDING)