13 from arvados._ranges import Range
14 from arvados.keep import KeepLocator
15 from arvados.collection import Collection, CollectionReader
16 from arvados.arvfile import ArvadosFile, ArvadosFileReader
18 import arvados_testutil as tutil
19 from test_stream import StreamFileReaderTestCase, StreamRetryTestMixin
21 class ArvadosFileWriterTestCase(unittest.TestCase):
22 class MockKeep(object):
23 def __init__(self, blocks):
26 def get(self, locator, num_retries=0):
27 self.requests.append(locator)
28 return self.blocks.get(locator)
29 def get_from_cache(self, locator):
30 self.requests.append(locator)
31 return self.blocks.get(locator)
32 def put(self, data, num_retries=None, copies=None):
33 pdh = tutil.str_keep_locator(data)
34 self.blocks[pdh] = str(data)
37 class MockApi(object):
38 def __init__(self, b, r):
41 self._schema = ArvadosFileWriterTestCase.MockApi.MockSchema()
43 class MockSchema(object):
45 self.schemas = {'Collection': {'properties': {'replication_desired': {'type':'integer'}}}}
46 class MockCollections(object):
47 def __init__(self, b, r):
50 class Execute(object):
51 def __init__(self, r):
53 def execute(self, num_retries=None):
55 def create(self, ensure_unique_name=False, body=None):
57 raise Exception("Body %s does not match expectation %s" % (body, self.body))
58 return ArvadosFileWriterTestCase.MockApi.MockCollections.Execute(self.response)
59 def update(self, uuid=None, body=None):
60 return ArvadosFileWriterTestCase.MockApi.MockCollections.Execute(self.response)
61 def collections(self):
62 return ArvadosFileWriterTestCase.MockApi.MockCollections(self.body, self.response)
65 def test_truncate(self):
66 keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
67 api = ArvadosFileWriterTestCase.MockApi({"name":"test_truncate",
68 "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
69 "replication_desired":None},
70 {"uuid":"zzzzz-4zz18-mockcollection0",
71 "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
72 "portable_data_hash":"7fcd0eaac3aad4c31a6a0e756475da92+52"})
73 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
74 api_client=api, keep_client=keep) as c:
75 writer = c.open("count.txt", "r+")
76 self.assertEqual(writer.size(), 10)
77 self.assertEqual("0123456789", writer.read(12))
81 # Make sure reading off the end doesn't break
82 self.assertEqual("", writer.read(12))
84 self.assertEqual(writer.size(), 8)
85 writer.seek(0, os.SEEK_SET)
86 self.assertEqual("01234567", writer.read(12))
88 self.assertIsNone(c.manifest_locator())
89 self.assertTrue(c.modified())
90 c.save_new("test_truncate")
91 self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
92 self.assertFalse(c.modified())
94 def test_write_to_end(self):
95 keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
96 api = ArvadosFileWriterTestCase.MockApi({"name":"test_append",
97 "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n",
98 "replication_desired":None},
99 {"uuid":"zzzzz-4zz18-mockcollection0",
100 "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n",
101 "portable_data_hash":"c5c3af76565c8efb6a806546bcf073f3+88"})
102 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
103 api_client=api, keep_client=keep) as c:
104 writer = c.open("count.txt", "r+")
105 self.assertEqual(writer.size(), 10)
107 writer.seek(5, os.SEEK_SET)
108 self.assertEqual("56789", writer.read(8))
110 writer.seek(10, os.SEEK_SET)
112 self.assertEqual(writer.size(), 13)
114 writer.seek(5, os.SEEK_SET)
115 self.assertEqual("56789foo", writer.read(8))
117 self.assertIsNone(c.manifest_locator())
118 self.assertTrue(c.modified())
119 self.assertIsNone(keep.get("acbd18db4cc2f85cedef654fccc4a4d8+3"))
121 c.save_new("test_append")
122 self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
123 self.assertFalse(c.modified())
124 self.assertEqual("foo", keep.get("acbd18db4cc2f85cedef654fccc4a4d8+3"))
127 def test_append(self):
128 keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
129 c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', keep_client=keep)
130 writer = c.open("count.txt", "a+")
131 self.assertEqual(writer.read(20), "0123456789")
132 writer.seek(0, os.SEEK_SET)
134 writer.write("hello")
135 self.assertEqual(writer.read(20), "0123456789hello")
136 writer.seek(0, os.SEEK_SET)
138 writer.write("world")
139 self.assertEqual(writer.read(20), "0123456789helloworld")
141 self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 fc5e038d38a57032085441e7fe7010b0+10 0:20:count.txt\n", c.portable_manifest_text())
143 def test_write_at_beginning(self):
144 keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
145 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
146 keep_client=keep) as c:
147 writer = c.open("count.txt", "r+")
148 self.assertEqual("0123456789", writer.readfrom(0, 13))
149 writer.seek(0, os.SEEK_SET)
151 self.assertEqual(writer.size(), 10)
152 self.assertEqual("foo3456789", writer.readfrom(0, 13))
153 self.assertEqual(". acbd18db4cc2f85cedef654fccc4a4d8+3 781e5e245d69b566979b86e28d23f2c7+10 0:3:count.txt 6:7:count.txt\n", c.portable_manifest_text())
155 def test_write_empty(self):
156 keep = ArvadosFileWriterTestCase.MockKeep({})
157 with Collection(keep_client=keep) as c:
158 writer = c.open("count.txt", "w")
159 self.assertEqual(writer.size(), 0)
160 self.assertEqual(". d41d8cd98f00b204e9800998ecf8427e+0 0:0:count.txt\n", c.portable_manifest_text())
162 def test_save_manifest_text(self):
163 keep = ArvadosFileWriterTestCase.MockKeep({})
164 with Collection(keep_client=keep) as c:
165 writer = c.open("count.txt", "w")
166 writer.write("0123456789")
167 self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.portable_manifest_text())
168 self.assertNotIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks)
170 self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.save_new(create_collection_record=False))
171 self.assertIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks)
173 def test_get_manifest_text_commits(self):
174 keep = ArvadosFileWriterTestCase.MockKeep({})
175 with Collection(keep_client=keep) as c:
176 writer = c.open("count.txt", "w")
177 writer.write("0123456789")
178 self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.portable_manifest_text())
179 self.assertNotIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks)
180 self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.manifest_text())
181 self.assertIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks)
184 def test_write_in_middle(self):
185 keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
186 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
187 keep_client=keep) as c:
188 writer = c.open("count.txt", "r+")
189 self.assertEqual("0123456789", writer.readfrom(0, 13))
190 writer.seek(3, os.SEEK_SET)
192 self.assertEqual(writer.size(), 10)
193 self.assertEqual("012foo6789", writer.readfrom(0, 13))
194 self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:count.txt 10:3:count.txt 6:4:count.txt\n", c.portable_manifest_text())
196 def test_write_at_end(self):
197 keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
198 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
199 keep_client=keep) as c:
200 writer = c.open("count.txt", "r+")
201 self.assertEqual("0123456789", writer.readfrom(0, 13))
202 writer.seek(7, os.SEEK_SET)
204 self.assertEqual(writer.size(), 10)
205 self.assertEqual("0123456foo", writer.readfrom(0, 13))
206 self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:7:count.txt 10:3:count.txt\n", c.portable_manifest_text())
208 def test_write_across_segment_boundary(self):
209 keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
210 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt 0:10:count.txt\n',
211 keep_client=keep) as c:
212 writer = c.open("count.txt", "r+")
213 self.assertEqual("012345678901234", writer.readfrom(0, 15))
214 writer.seek(7, os.SEEK_SET)
215 writer.write("foobar")
216 self.assertEqual(writer.size(), 20)
217 self.assertEqual("0123456foobar34", writer.readfrom(0, 15))
218 self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 3858f62230ac3c915f300c664312c63f+6 0:7:count.txt 10:6:count.txt 3:7:count.txt\n", c.portable_manifest_text())
220 def test_write_across_several_segments(self):
221 keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
222 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:4:count.txt 0:4:count.txt 0:4:count.txt',
223 keep_client=keep) as c:
224 writer = c.open("count.txt", "r+")
225 self.assertEqual("012301230123", writer.readfrom(0, 15))
226 writer.seek(2, os.SEEK_SET)
227 writer.write("abcdefg")
228 self.assertEqual(writer.size(), 12)
229 self.assertEqual("01abcdefg123", writer.readfrom(0, 15))
230 self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 7ac66c0f148de9519b8bd264312c4d64+7 0:2:count.txt 10:7:count.txt 1:3:count.txt\n", c.portable_manifest_text())
232 def test_write_large(self):
233 keep = ArvadosFileWriterTestCase.MockKeep({})
234 api = ArvadosFileWriterTestCase.MockApi({"name":"test_write_large",
235 "manifest_text": ". a5de24f4417cfba9d5825eadc2f4ca49+67108000 598cc1a4ccaef8ab6e4724d87e675d78+32892000 0:100000000:count.txt\n",
236 "replication_desired":None},
237 {"uuid":"zzzzz-4zz18-mockcollection0",
238 "manifest_text": ". a5de24f4417cfba9d5825eadc2f4ca49+67108000 598cc1a4ccaef8ab6e4724d87e675d78+32892000 0:100000000:count.txt\n",
239 "portable_data_hash":"9132ca8e3f671c76103a38f5bc24328c+108"})
240 with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
241 api_client=api, keep_client=keep) as c:
242 writer = c.open("count.txt", "r+")
243 text = "0123456789" * 100
244 for b in xrange(0, 100000):
246 self.assertEqual(writer.size(), 100000000)
248 self.assertIsNone(c.manifest_locator())
249 self.assertTrue(c.modified())
250 c.save_new("test_write_large")
251 self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
252 self.assertFalse(c.modified())
255 def test_large_write(self):
256 keep = ArvadosFileWriterTestCase.MockKeep({})
257 api = ArvadosFileWriterTestCase.MockApi({}, {})
258 with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
259 api_client=api, keep_client=keep) as c:
260 writer = c.open("count.txt", "r+")
261 self.assertEqual(writer.size(), 0)
265 text = "0123456789" * 9999999
267 self.assertEqual(writer.size(), 100000000)
269 self.assertEqual(c.manifest_text(), ". 781e5e245d69b566979b86e28d23f2c7+10 48dd23ea1645fd47d789804d71b5bb8e+67108864 77c57dc6ac5a10bb2205caaa73187994+32891126 0:100000000:count.txt\n")
271 def test_rewrite_on_empty_file(self):
272 keep = ArvadosFileWriterTestCase.MockKeep({})
273 with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
274 keep_client=keep) as c:
275 writer = c.open("count.txt", "r+")
276 for b in xrange(0, 10):
277 writer.seek(0, os.SEEK_SET)
278 writer.write("0123456789")
280 self.assertEqual(writer.size(), 10)
281 self.assertEqual("0123456789", writer.readfrom(0, 20))
282 self.assertEqual(". 7a08b07e84641703e5f2c836aa59a170+100 90:10:count.txt\n", c.portable_manifest_text())
284 self.assertEqual(writer.size(), 10)
285 self.assertEqual("0123456789", writer.readfrom(0, 20))
286 self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n", c.portable_manifest_text())
288 def test_rewrite_append_existing_file(self):
289 keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
290 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
291 keep_client=keep) as c:
292 writer = c.open("count.txt", "r+")
293 for b in xrange(0, 10):
294 writer.seek(10, os.SEEK_SET)
295 writer.write("abcdefghij")
297 self.assertEqual(writer.size(), 20)
298 self.assertEqual("0123456789abcdefghij", writer.readfrom(0, 20))
299 self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 ae5f43bab79cf0be33f025fa97ae7398+100 0:10:count.txt 100:10:count.txt\n", c.portable_manifest_text())
301 writer.arvadosfile.flush()
302 self.assertEqual(writer.size(), 20)
303 self.assertEqual("0123456789abcdefghij", writer.readfrom(0, 20))
304 self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:20:count.txt\n", c.portable_manifest_text())
306 def test_rewrite_over_existing_file(self):
307 keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
308 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
309 keep_client=keep) as c:
310 writer = c.open("count.txt", "r+")
311 for b in xrange(0, 10):
312 writer.seek(5, os.SEEK_SET)
313 writer.write("abcdefghij")
315 self.assertEqual(writer.size(), 15)
316 self.assertEqual("01234abcdefghij", writer.readfrom(0, 20))
317 self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 ae5f43bab79cf0be33f025fa97ae7398+100 0:5:count.txt 100:10:count.txt\n", c.portable_manifest_text())
319 writer.arvadosfile.flush()
321 self.assertEqual(writer.size(), 15)
322 self.assertEqual("01234abcdefghij", writer.readfrom(0, 20))
323 self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:5:count.txt 10:10:count.txt\n", c.portable_manifest_text())
325 def test_write_large_rewrite(self):
326 keep = ArvadosFileWriterTestCase.MockKeep({})
327 api = ArvadosFileWriterTestCase.MockApi({"name":"test_write_large",
328 "manifest_text": ". 37400a68af9abdd76ca5bf13e819e42a+32892003 a5de24f4417cfba9d5825eadc2f4ca49+67108000 32892000:3:count.txt 32892006:67107997:count.txt 0:32892000:count.txt\n",
329 "replication_desired":None},
330 {"uuid":"zzzzz-4zz18-mockcollection0",
331 "manifest_text": ". 37400a68af9abdd76ca5bf13e819e42a+32892003 a5de24f4417cfba9d5825eadc2f4ca49+67108000 32892000:3:count.txt 32892006:67107997:count.txt 0:32892000:count.txt\n",
332 "portable_data_hash":"217665c6b713e1b78dfba7ebd42344db+156"})
333 with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
334 api_client=api, keep_client=keep) as c:
335 writer = c.open("count.txt", "r+")
336 text = ''.join(["0123456789" for a in xrange(0, 100)])
337 for b in xrange(0, 100000):
339 writer.seek(0, os.SEEK_SET)
341 self.assertEqual(writer.size(), 100000000)
343 self.assertIsNone(c.manifest_locator())
344 self.assertTrue(c.modified())
345 c.save_new("test_write_large")
346 self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
347 self.assertFalse(c.modified())
349 def test_create(self):
350 keep = ArvadosFileWriterTestCase.MockKeep({})
351 api = ArvadosFileWriterTestCase.MockApi({"name":"test_create",
352 "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
353 "replication_desired":None},
354 {"uuid":"zzzzz-4zz18-mockcollection0",
355 "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
356 "portable_data_hash":"7a461a8c58601798f690f8b368ac4423+51"})
357 with Collection(api_client=api, keep_client=keep) as c:
358 writer = c.open("count.txt", "w+")
359 self.assertEqual(writer.size(), 0)
360 writer.write("01234567")
361 self.assertEqual(writer.size(), 8)
363 self.assertIsNone(c.manifest_locator())
364 self.assertTrue(c.modified())
365 self.assertIsNone(keep.get("2e9ec317e197819358fbc43afca7d837+8"))
366 c.save_new("test_create")
367 self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
368 self.assertFalse(c.modified())
369 self.assertEqual("01234567", keep.get("2e9ec317e197819358fbc43afca7d837+8"))
372 def test_create_subdir(self):
373 keep = ArvadosFileWriterTestCase.MockKeep({})
374 api = ArvadosFileWriterTestCase.MockApi({"name":"test_create",
375 "manifest_text":"./foo/bar 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
376 "replication_desired":None},
377 {"uuid":"zzzzz-4zz18-mockcollection0",
378 "manifest_text":"./foo/bar 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
379 "portable_data_hash":"1b02aaa62528d28a5be41651cbb9d7c7+59"})
380 with Collection(api_client=api, keep_client=keep) as c:
381 self.assertIsNone(c.api_response())
382 writer = c.open("foo/bar/count.txt", "w+")
383 writer.write("01234567")
384 self.assertFalse(c.committed())
385 c.save_new("test_create")
386 self.assertTrue(c.committed())
387 self.assertEqual(c.api_response(), api.response)
389 def test_overwrite(self):
390 keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
391 api = ArvadosFileWriterTestCase.MockApi({"name":"test_overwrite",
392 "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
393 "replication_desired":None},
394 {"uuid":"zzzzz-4zz18-mockcollection0",
395 "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
396 "portable_data_hash":"7a461a8c58601798f690f8b368ac4423+51"})
397 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
398 api_client=api, keep_client=keep) as c:
399 writer = c.open("count.txt", "w+")
400 self.assertEqual(writer.size(), 0)
401 writer.write("01234567")
402 self.assertEqual(writer.size(), 8)
404 self.assertIsNone(c.manifest_locator())
405 self.assertTrue(c.modified())
406 c.save_new("test_overwrite")
407 self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
408 self.assertFalse(c.modified())
410 def test_file_not_found(self):
411 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n') as c:
412 with self.assertRaises(IOError):
413 writer = c.open("nocount.txt", "r")
415 def test_cannot_open_directory(self):
416 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n') as c:
417 with self.assertRaises(IOError):
418 writer = c.open(".", "r")
420 def test_create_multiple(self):
421 keep = ArvadosFileWriterTestCase.MockKeep({})
422 api = ArvadosFileWriterTestCase.MockApi({"name":"test_create_multiple",
423 "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:8:count1.txt 8:8:count2.txt\n",
424 "replication_desired":None},
425 {"uuid":"zzzzz-4zz18-mockcollection0",
426 "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:8:count1.txt 8:8:count2.txt\n",
427 "portable_data_hash":"71e7bb6c00d31fc2b4364199fd97be08+102"})
428 with Collection(api_client=api, keep_client=keep) as c:
429 w1 = c.open("count1.txt", "w")
430 w2 = c.open("count2.txt", "w")
433 self.assertEqual(w1.size(), 8)
434 self.assertEqual(w2.size(), 8)
436 self.assertIsNone(c.manifest_locator())
437 self.assertTrue(c.modified())
438 self.assertIsNone(keep.get("2e9ec317e197819358fbc43afca7d837+8"))
439 c.save_new("test_create_multiple")
440 self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
441 self.assertFalse(c.modified())
442 self.assertEqual("01234567", keep.get("2e9ec317e197819358fbc43afca7d837+8"))
445 class ArvadosFileReaderTestCase(StreamFileReaderTestCase):
446 class MockParent(object):
447 class MockBlockMgr(object):
448 def __init__(self, blocks, nocache):
450 self.nocache = nocache
452 def block_prefetch(self, loc):
455 def get_block_contents(self, loc, num_retries=0, cache_only=False):
456 if self.nocache and cache_only:
458 return self.blocks[loc]
460 def __init__(self, blocks, nocache):
462 self.nocache = nocache
463 self.lock = arvados.arvfile.NoopLock()
465 def root_collection(self):
468 def _my_block_manager(self):
469 return ArvadosFileReaderTestCase.MockParent.MockBlockMgr(self.blocks, self.nocache)
472 def make_count_reader(self, nocache=False):
476 for d in ['01234', '34567', '67890']:
477 loc = tutil.str_keep_locator(d)
479 stream.append(Range(loc, n, len(d)))
481 af = ArvadosFile(ArvadosFileReaderTestCase.MockParent(blocks, nocache), "count.txt", stream=stream, segments=[Range(1, 0, 3), Range(6, 3, 3), Range(11, 6, 3)])
482 return ArvadosFileReader(af)
484 def test_read_block_crossing_behavior(self):
485 # read() needs to return all the data requested if possible, even if it
486 # crosses uncached blocks: https://arvados.org/issues/5856
487 sfile = self.make_count_reader(nocache=True)
488 self.assertEqual('12345678', sfile.read(8))
490 def test_successive_reads(self):
491 # Override StreamFileReaderTestCase.test_successive_reads
492 sfile = self.make_count_reader(nocache=True)
493 self.assertEqual('1234', sfile.read(4))
494 self.assertEqual('5678', sfile.read(4))
495 self.assertEqual('9', sfile.read(4))
496 self.assertEqual('', sfile.read(4))
498 def test_tell_after_block_read(self):
499 # Override StreamFileReaderTestCase.test_tell_after_block_read
500 sfile = self.make_count_reader(nocache=True)
501 self.assertEqual('12345678', sfile.read(8))
502 self.assertEqual(8, sfile.tell())
504 def test_prefetch(self):
505 keep = ArvadosFileWriterTestCase.MockKeep({"2e9ec317e197819358fbc43afca7d837+8": "01234567", "e8dc4081b13434b45189a720b77b6818+8": "abcdefgh"})
506 with Collection(". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:16:count.txt\n", keep_client=keep) as c:
507 r = c.open("count.txt", "r")
508 self.assertEqual("0123", r.read(4))
509 self.assertIn("2e9ec317e197819358fbc43afca7d837+8", keep.requests)
510 self.assertIn("e8dc4081b13434b45189a720b77b6818+8", keep.requests)
512 def test__eq__from_manifest(self):
513 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
514 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c2:
515 self.assertTrue(c1["count1.txt"] == c2["count1.txt"])
516 self.assertFalse(c1["count1.txt"] != c2["count1.txt"])
518 def test__eq__from_writes(self):
519 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
520 with Collection() as c2:
521 f = c2.open("count1.txt", "w")
522 f.write("0123456789")
524 self.assertTrue(c1["count1.txt"] == c2["count1.txt"])
525 self.assertFalse(c1["count1.txt"] != c2["count1.txt"])
527 def test__ne__(self):
528 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
529 with Collection() as c2:
530 f = c2.open("count1.txt", "w")
531 f.write("1234567890")
533 self.assertTrue(c1["count1.txt"] != c2["count1.txt"])
534 self.assertFalse(c1["count1.txt"] == c2["count1.txt"])
537 class ArvadosFileReadTestCase(unittest.TestCase, StreamRetryTestMixin):
538 def reader_for(self, coll_name, **kwargs):
542 for d in self.manifest_for(coll_name).split():
545 segments.append(Range(n, n, k.size))
546 stream.append(Range(d, n, k.size))
551 blockmanager = arvados.arvfile._BlockManager(self.keep_client())
552 blockmanager.prefetch_enabled = False
553 col = Collection(keep_client=self.keep_client(), block_manager=blockmanager)
554 af = ArvadosFile(col, "test",
557 return ArvadosFileReader(af, **kwargs)
559 def read_for_test(self, reader, byte_count, **kwargs):
560 return reader.read(byte_count, **kwargs)
563 class ArvadosFileReadFromTestCase(ArvadosFileReadTestCase):
564 def read_for_test(self, reader, byte_count, **kwargs):
565 return reader.readfrom(0, byte_count, **kwargs)
568 class ArvadosFileReadAllTestCase(ArvadosFileReadTestCase):
569 def read_for_test(self, reader, byte_count, **kwargs):
570 return ''.join(reader.readall(**kwargs))
573 class ArvadosFileReadAllDecompressedTestCase(ArvadosFileReadTestCase):
574 def read_for_test(self, reader, byte_count, **kwargs):
575 return ''.join(reader.readall_decompressed(**kwargs))
578 class ArvadosFileReadlinesTestCase(ArvadosFileReadTestCase):
579 def read_for_test(self, reader, byte_count, **kwargs):
580 return ''.join(reader.readlines(**kwargs))
583 class ArvadosFileTestCase(unittest.TestCase):
584 def datetime_to_hex(self, dt):
585 return hex(int(time.mktime(dt.timetuple())))[2:]
587 def test_permission_expired(self):
588 base_manifest = ". 781e5e245d69b566979b86e28d23f2c7+10+A715fd31f8111894f717eb1003c1b0216799dd9ec@{} 0:10:count.txt\n"
589 now = datetime.datetime.now()
590 a_week_ago = now - datetime.timedelta(days=7)
591 a_month_ago = now - datetime.timedelta(days=30)
592 a_week_from_now = now + datetime.timedelta(days=7)
593 with Collection(base_manifest.format(self.datetime_to_hex(a_week_from_now))) as c:
594 self.assertFalse(c.find('count.txt').permission_expired())
595 with Collection(base_manifest.format(self.datetime_to_hex(a_week_ago))) as c:
596 f = c.find('count.txt')
597 self.assertTrue(f.permission_expired())
598 self.assertTrue(f.permission_expired(a_week_from_now))
599 self.assertFalse(f.permission_expired(a_month_ago))
602 class BlockManagerTest(unittest.TestCase):
603 def test_bufferblock_append(self):
604 keep = ArvadosFileWriterTestCase.MockKeep({})
605 with arvados.arvfile._BlockManager(keep) as blockmanager:
606 bufferblock = blockmanager.alloc_bufferblock()
607 bufferblock.append("foo")
609 self.assertEqual(bufferblock.size(), 3)
610 self.assertEqual(bufferblock.buffer_view[0:3], "foo")
611 self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3")
613 bufferblock.append("bar")
615 self.assertEqual(bufferblock.size(), 6)
616 self.assertEqual(bufferblock.buffer_view[0:6], "foobar")
617 self.assertEqual(bufferblock.locator(), "3858f62230ac3c915f300c664312c63f+6")
619 bufferblock.set_state(arvados.arvfile._BufferBlock.PENDING)
620 with self.assertRaises(arvados.errors.AssertionError):
621 bufferblock.append("bar")
623 def test_bufferblock_dup(self):
624 keep = ArvadosFileWriterTestCase.MockKeep({})
625 with arvados.arvfile._BlockManager(keep) as blockmanager:
626 bufferblock = blockmanager.alloc_bufferblock()
627 bufferblock.append("foo")
629 self.assertEqual(bufferblock.size(), 3)
630 self.assertEqual(bufferblock.buffer_view[0:3], "foo")
631 self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3")
632 bufferblock.set_state(arvados.arvfile._BufferBlock.PENDING)
634 bufferblock2 = blockmanager.dup_block(bufferblock, None)
635 self.assertNotEqual(bufferblock.blockid, bufferblock2.blockid)
637 bufferblock2.append("bar")
639 self.assertEqual(bufferblock2.size(), 6)
640 self.assertEqual(bufferblock2.buffer_view[0:6], "foobar")
641 self.assertEqual(bufferblock2.locator(), "3858f62230ac3c915f300c664312c63f+6")
643 self.assertEqual(bufferblock.size(), 3)
644 self.assertEqual(bufferblock.buffer_view[0:3], "foo")
645 self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3")
647 def test_bufferblock_get(self):
648 keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
649 with arvados.arvfile._BlockManager(keep) as blockmanager:
650 bufferblock = blockmanager.alloc_bufferblock()
651 bufferblock.append("foo")
653 self.assertEqual(blockmanager.get_block_contents("781e5e245d69b566979b86e28d23f2c7+10", 1), "0123456789")
654 self.assertEqual(blockmanager.get_block_contents(bufferblock.blockid, 1), "foo")
656 def test_bufferblock_commit(self):
657 mockkeep = mock.MagicMock()
658 with arvados.arvfile._BlockManager(mockkeep) as blockmanager:
659 bufferblock = blockmanager.alloc_bufferblock()
660 bufferblock.owner = mock.MagicMock()
661 def flush(sync=None):
662 blockmanager.commit_bufferblock(bufferblock, sync)
663 bufferblock.owner.flush.side_effect = flush
664 bufferblock.append("foo")
665 blockmanager.commit_all()
666 self.assertTrue(bufferblock.owner.flush.called)
667 self.assertTrue(mockkeep.put.called)
668 self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.COMMITTED)
669 self.assertIsNone(bufferblock.buffer_view)
671 def test_bufferblock_commit_pending(self):
673 mockkeep = mock.MagicMock()
674 mockkeep.put.side_effect = lambda x: time.sleep(1)
675 with arvados.arvfile._BlockManager(mockkeep) as blockmanager:
676 bufferblock = blockmanager.alloc_bufferblock()
677 bufferblock.append("foo")
679 blockmanager.commit_bufferblock(bufferblock, False)
680 self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.PENDING)
682 blockmanager.commit_bufferblock(bufferblock, True)
683 self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.COMMITTED)
686 def test_bufferblock_commit_with_error(self):
687 mockkeep = mock.MagicMock()
688 mockkeep.put.side_effect = arvados.errors.KeepWriteError("fail")
689 with arvados.arvfile._BlockManager(mockkeep) as blockmanager:
690 bufferblock = blockmanager.alloc_bufferblock()
691 bufferblock.owner = mock.MagicMock()
692 def flush(sync=None):
693 blockmanager.commit_bufferblock(bufferblock, sync)
694 bufferblock.owner.flush.side_effect = flush
695 bufferblock.append("foo")
696 with self.assertRaises(arvados.errors.KeepWriteError) as err:
697 blockmanager.commit_all()
698 self.assertTrue(bufferblock.owner.flush.called)
699 self.assertEqual(str(err.exception), "Error writing some blocks: block acbd18db4cc2f85cedef654fccc4a4d8+3 raised KeepWriteError (fail)")
700 self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.ERROR)