1 from __future__ import absolute_import
2 from builtins import hex
3 from builtins import str
4 from builtins import range
5 from builtins import object
16 from arvados._ranges import Range
17 from arvados.keep import KeepLocator
18 from arvados.collection import Collection, CollectionReader
19 from arvados.arvfile import ArvadosFile, ArvadosFileReader
21 from . import arvados_testutil as tutil
22 from .test_stream import StreamFileReaderTestCase, StreamRetryTestMixin
24 class ArvadosFileWriterTestCase(unittest.TestCase):
25 class MockKeep(object):
26 def __init__(self, blocks):
29 def get(self, locator, num_retries=0):
30 self.requests.append(locator)
31 return self.blocks.get(locator)
32 def get_from_cache(self, locator):
33 self.requests.append(locator)
34 return self.blocks.get(locator)
35 def put(self, data, num_retries=None, copies=None):
36 pdh = tutil.str_keep_locator(data)
37 self.blocks[pdh] = bytes(data)
40 class MockApi(object):
41 def __init__(self, b, r):
44 self._schema = ArvadosFileWriterTestCase.MockApi.MockSchema()
46 class MockSchema(object):
48 self.schemas = {'Collection': {'properties': {'replication_desired': {'type':'integer'}}}}
49 class MockCollections(object):
50 def __init__(self, b, r):
53 class Execute(object):
54 def __init__(self, r):
56 def execute(self, num_retries=None):
58 def create(self, ensure_unique_name=False, body=None):
60 raise Exception("Body %s does not match expectation %s" % (body, self.body))
61 return ArvadosFileWriterTestCase.MockApi.MockCollections.Execute(self.response)
62 def update(self, uuid=None, body=None):
63 return ArvadosFileWriterTestCase.MockApi.MockCollections.Execute(self.response)
64 def collections(self):
65 return ArvadosFileWriterTestCase.MockApi.MockCollections(self.body, self.response)
68 def test_truncate(self):
69 keep = ArvadosFileWriterTestCase.MockKeep({
70 "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
72 api = ArvadosFileWriterTestCase.MockApi({
73 "name": "test_truncate",
74 "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
75 "replication_desired": None,
77 "uuid": "zzzzz-4zz18-mockcollection0",
78 "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
79 "portable_data_hash":"7fcd0eaac3aad4c31a6a0e756475da92+52",
81 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
82 api_client=api, keep_client=keep) as c:
83 writer = c.open("count.txt", "rb+")
84 self.assertEqual(writer.size(), 10)
85 self.assertEqual(b"0123456789", writer.read(12))
89 # Make sure reading off the end doesn't break
90 self.assertEqual(b"", writer.read(12))
92 self.assertEqual(writer.size(), 8)
93 writer.seek(0, os.SEEK_SET)
94 self.assertEqual(b"01234567", writer.read(12))
96 self.assertIsNone(c.manifest_locator())
97 self.assertTrue(c.modified())
98 c.save_new("test_truncate")
99 self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
100 self.assertFalse(c.modified())
103 def test_truncate2(self):
104 keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
105 api = ArvadosFileWriterTestCase.MockApi({"name":"test_truncate2",
106 "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 7f614da9329cd3aebf59b91aadc30bf0+67108864 0:12:count.txt\n",
107 "replication_desired":None},
108 {"uuid":"zzzzz-4zz18-mockcollection0",
109 "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 7f614da9329cd3aebf59b91aadc30bf0+67108864 0:12:count.txt\n",
110 "portable_data_hash":"272da898abdf86ddc71994835e3155f8+95"})
111 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
112 api_client=api, keep_client=keep) as c:
113 writer = c.open("count.txt", "r+")
114 self.assertEqual(writer.size(), 10)
115 self.assertEqual("0123456789", writer.read(12))
120 self.assertEqual(writer.size(), 12)
121 writer.seek(0, os.SEEK_SET)
122 self.assertEqual(b"0123456789\x00\x00", writer.read(12))
124 self.assertIsNone(c.manifest_locator())
125 self.assertTrue(c.modified())
126 c.save_new("test_truncate2")
127 self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
128 self.assertFalse(c.modified())
130 def test_truncate3(self):
131 keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789",
132 "a925576942e94b2ef57a066101b48876+10": "abcdefghij"})
133 api = ArvadosFileWriterTestCase.MockApi({"name":"test_truncate",
134 "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
135 "replication_desired":None},
136 {"uuid":"zzzzz-4zz18-mockcollection0",
137 "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
138 "portable_data_hash":"7fcd0eaac3aad4c31a6a0e756475da92+52"})
139 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:20:count.txt\n',
140 api_client=api, keep_client=keep) as c:
141 writer = c.open("count.txt", "r+")
142 self.assertEqual(writer.size(), 20)
143 self.assertEqual("0123456789ab", writer.read(12))
144 self.assertEqual(12, writer.tell())
148 # Make sure reading off the end doesn't break
149 self.assertEqual(12, writer.tell())
150 self.assertEqual("", writer.read(12))
152 self.assertEqual(writer.size(), 8)
153 self.assertEqual(2, writer.seek(-10, os.SEEK_CUR))
154 self.assertEqual("234567", writer.read(12))
156 self.assertIsNone(c.manifest_locator())
157 self.assertTrue(c.modified())
158 c.save_new("test_truncate")
159 self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
160 self.assertFalse(c.modified())
164 def test_write_to_end(self):
165 keep = ArvadosFileWriterTestCase.MockKeep({
166 "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
168 api = ArvadosFileWriterTestCase.MockApi({
169 "name": "test_append",
170 "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n",
171 "replication_desired": None,
173 "uuid": "zzzzz-4zz18-mockcollection0",
174 "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n",
175 "portable_data_hash": "c5c3af76565c8efb6a806546bcf073f3+88",
177 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
178 api_client=api, keep_client=keep) as c:
179 writer = c.open("count.txt", "rb+")
180 self.assertEqual(writer.size(), 10)
182 self.assertEqual(5, writer.seek(5, os.SEEK_SET))
183 self.assertEqual(b"56789", writer.read(8))
185 writer.seek(10, os.SEEK_SET)
187 self.assertEqual(writer.size(), 13)
189 writer.seek(5, os.SEEK_SET)
190 self.assertEqual(b"56789foo", writer.read(8))
192 self.assertIsNone(c.manifest_locator())
193 self.assertTrue(c.modified())
194 self.assertIsNone(keep.get("acbd18db4cc2f85cedef654fccc4a4d8+3"))
196 c.save_new("test_append")
197 self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
198 self.assertFalse(c.modified())
199 self.assertEqual(b"foo", keep.get("acbd18db4cc2f85cedef654fccc4a4d8+3"))
202 def test_append(self):
203 keep = ArvadosFileWriterTestCase.MockKeep({
204 "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
206 c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', keep_client=keep)
207 writer = c.open("count.txt", "ab+")
208 self.assertEqual(writer.read(20), b"0123456789")
209 writer.seek(0, os.SEEK_SET)
211 writer.write("hello")
212 self.assertEqual(writer.read(20), b"0123456789hello")
213 writer.seek(0, os.SEEK_SET)
215 writer.write("world")
216 self.assertEqual(writer.read(20), b"0123456789helloworld")
218 self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 fc5e038d38a57032085441e7fe7010b0+10 0:20:count.txt\n", c.portable_manifest_text())
220 def test_write_at_beginning(self):
221 keep = ArvadosFileWriterTestCase.MockKeep({
222 "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
224 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
225 keep_client=keep) as c:
226 writer = c.open("count.txt", "rb+")
227 self.assertEqual(b"0123456789", writer.readfrom(0, 13))
228 writer.seek(0, os.SEEK_SET)
230 self.assertEqual(writer.size(), 10)
231 self.assertEqual(b"foo3456789", writer.readfrom(0, 13))
232 self.assertEqual(". acbd18db4cc2f85cedef654fccc4a4d8+3 781e5e245d69b566979b86e28d23f2c7+10 0:3:count.txt 6:7:count.txt\n", c.portable_manifest_text())
234 def test_write_empty(self):
235 keep = ArvadosFileWriterTestCase.MockKeep({})
236 with Collection(keep_client=keep) as c:
237 writer = c.open("count.txt", "wb")
238 self.assertEqual(writer.size(), 0)
239 self.assertEqual(". d41d8cd98f00b204e9800998ecf8427e+0 0:0:count.txt\n", c.portable_manifest_text())
241 def test_save_manifest_text(self):
242 keep = ArvadosFileWriterTestCase.MockKeep({})
243 with Collection(keep_client=keep) as c:
244 writer = c.open("count.txt", "wb")
245 writer.write(b"0123456789")
246 self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.portable_manifest_text())
247 self.assertNotIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks)
249 self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.save_new(create_collection_record=False))
250 self.assertIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks)
252 def test_get_manifest_text_commits(self):
253 keep = ArvadosFileWriterTestCase.MockKeep({})
254 with Collection(keep_client=keep) as c:
255 writer = c.open("count.txt", "wb")
256 writer.write("0123456789")
257 self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.portable_manifest_text())
258 self.assertNotIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks)
259 self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.manifest_text())
260 self.assertIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks)
263 def test_write_in_middle(self):
264 keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": b"0123456789"})
265 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
266 keep_client=keep) as c:
267 writer = c.open("count.txt", "rb+")
268 self.assertEqual(b"0123456789", writer.readfrom(0, 13))
269 writer.seek(3, os.SEEK_SET)
271 self.assertEqual(writer.size(), 10)
272 self.assertEqual(b"012foo6789", writer.readfrom(0, 13))
273 self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:count.txt 10:3:count.txt 6:4:count.txt\n", c.portable_manifest_text())
275 def test_write_at_end(self):
276 keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": b"0123456789"})
277 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
278 keep_client=keep) as c:
279 writer = c.open("count.txt", "rb+")
280 self.assertEqual(b"0123456789", writer.readfrom(0, 13))
281 writer.seek(7, os.SEEK_SET)
283 self.assertEqual(writer.size(), 10)
284 self.assertEqual(b"0123456foo", writer.readfrom(0, 13))
285 self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:7:count.txt 10:3:count.txt\n", c.portable_manifest_text())
287 def test_write_across_segment_boundary(self):
288 keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": b"0123456789"})
289 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt 0:10:count.txt\n',
290 keep_client=keep) as c:
291 writer = c.open("count.txt", "rb+")
292 self.assertEqual(b"012345678901234", writer.readfrom(0, 15))
293 writer.seek(7, os.SEEK_SET)
294 writer.write("foobar")
295 self.assertEqual(writer.size(), 20)
296 self.assertEqual(b"0123456foobar34", writer.readfrom(0, 15))
297 self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 3858f62230ac3c915f300c664312c63f+6 0:7:count.txt 10:6:count.txt 3:7:count.txt\n", c.portable_manifest_text())
299 def test_write_across_several_segments(self):
300 keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": b"0123456789"})
301 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:4:count.txt 0:4:count.txt 0:4:count.txt',
302 keep_client=keep) as c:
303 writer = c.open("count.txt", "rb+")
304 self.assertEqual(b"012301230123", writer.readfrom(0, 15))
305 writer.seek(2, os.SEEK_SET)
306 writer.write("abcdefg")
307 self.assertEqual(writer.size(), 12)
308 self.assertEqual(b"01abcdefg123", writer.readfrom(0, 15))
309 self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 7ac66c0f148de9519b8bd264312c4d64+7 0:2:count.txt 10:7:count.txt 1:3:count.txt\n", c.portable_manifest_text())
311 def test_write_large(self):
312 keep = ArvadosFileWriterTestCase.MockKeep({})
313 api = ArvadosFileWriterTestCase.MockApi({"name":"test_write_large",
314 "manifest_text": ". a5de24f4417cfba9d5825eadc2f4ca49+67108000 598cc1a4ccaef8ab6e4724d87e675d78+32892000 0:100000000:count.txt\n",
315 "replication_desired":None},
316 {"uuid":"zzzzz-4zz18-mockcollection0",
317 "manifest_text": ". a5de24f4417cfba9d5825eadc2f4ca49+67108000 598cc1a4ccaef8ab6e4724d87e675d78+32892000 0:100000000:count.txt\n",
318 "portable_data_hash":"9132ca8e3f671c76103a38f5bc24328c+108"})
319 with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
320 api_client=api, keep_client=keep) as c:
321 writer = c.open("count.txt", "rb+")
322 text = "0123456789" * 100
323 for b in range(0, 100000):
325 self.assertEqual(writer.size(), 100000000)
327 self.assertIsNone(c.manifest_locator())
328 self.assertTrue(c.modified())
329 c.save_new("test_write_large")
330 self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
331 self.assertFalse(c.modified())
334 def test_large_write(self):
335 keep = ArvadosFileWriterTestCase.MockKeep({})
336 api = ArvadosFileWriterTestCase.MockApi({}, {})
337 with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
338 api_client=api, keep_client=keep) as c:
339 writer = c.open("count.txt", "rb+")
340 self.assertEqual(writer.size(), 0)
344 text = "0123456789" * 9999999
346 self.assertEqual(writer.size(), 100000000)
348 self.assertEqual(c.manifest_text(), ". 781e5e245d69b566979b86e28d23f2c7+10 48dd23ea1645fd47d789804d71b5bb8e+67108864 77c57dc6ac5a10bb2205caaa73187994+32891126 0:100000000:count.txt\n")
350 def test_sparse_write(self):
351 keep = ArvadosFileWriterTestCase.MockKeep({})
352 api = ArvadosFileWriterTestCase.MockApi({}, {})
353 with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
354 api_client=api, keep_client=keep) as c:
355 writer = c.open("count.txt", "r+")
356 self.assertEqual(writer.size(), 0)
361 self.assertEqual(writer.size(), 12)
362 writer.seek(0, os.SEEK_SET)
363 self.assertEqual(writer.read(), b"\x00\x00"+text)
365 self.assertEqual(c.manifest_text(), ". 7f614da9329cd3aebf59b91aadc30bf0+67108864 781e5e245d69b566979b86e28d23f2c7+10 0:2:count.txt 67108864:10:count.txt\n")
368 def test_sparse_write2(self):
369 keep = ArvadosFileWriterTestCase.MockKeep({})
370 api = ArvadosFileWriterTestCase.MockApi({}, {})
371 with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
372 api_client=api, keep_client=keep) as c:
373 writer = c.open("count.txt", "r+")
374 self.assertEqual(writer.size(), 0)
377 writer.seek((arvados.config.KEEP_BLOCK_SIZE*2) + 2)
379 self.assertEqual(writer.size(), (arvados.config.KEEP_BLOCK_SIZE*2) + 12)
380 writer.seek(0, os.SEEK_SET)
382 self.assertEqual(c.manifest_text(), ". 7f614da9329cd3aebf59b91aadc30bf0+67108864 781e5e245d69b566979b86e28d23f2c7+10 0:67108864:count.txt 0:67108864:count.txt 0:2:count.txt 67108864:10:count.txt\n")
385 def test_sparse_write3(self):
386 keep = ArvadosFileWriterTestCase.MockKeep({})
387 api = ArvadosFileWriterTestCase.MockApi({}, {})
388 for r in [[0, 1, 2, 3, 4], [4, 3, 2, 1, 0], [3, 2, 0, 4, 1]]:
389 with Collection() as c:
390 writer = c.open("count.txt", "r+")
391 self.assertEqual(writer.size(), 0)
398 self.assertEqual(writer.read(), "00000000001111111111222222222233333333334444444444")
400 def test_sparse_write4(self):
401 keep = ArvadosFileWriterTestCase.MockKeep({})
402 api = ArvadosFileWriterTestCase.MockApi({}, {})
403 for r in [[0, 1, 2, 4], [4, 2, 1, 0], [2, 0, 4, 1]]:
404 with Collection() as c:
405 writer = c.open("count.txt", "r+")
406 self.assertEqual(writer.size(), 0)
413 self.assertEqual(writer.read(), b"000000000011111111112222222222\x00\x00\x00\x00\x00\x00\x00\x00\x00\x004444444444")
416 def test_rewrite_on_empty_file(self):
417 keep = ArvadosFileWriterTestCase.MockKeep({})
418 with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
419 keep_client=keep) as c:
420 writer = c.open("count.txt", "rb+")
421 for b in range(0, 10):
422 writer.seek(0, os.SEEK_SET)
423 writer.write("0123456789")
425 self.assertEqual(writer.size(), 10)
426 self.assertEqual(b"0123456789", writer.readfrom(0, 20))
427 self.assertEqual(". 7a08b07e84641703e5f2c836aa59a170+100 90:10:count.txt\n", c.portable_manifest_text())
429 self.assertEqual(writer.size(), 10)
430 self.assertEqual(b"0123456789", writer.readfrom(0, 20))
431 self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n", c.portable_manifest_text())
433 def test_rewrite_append_existing_file(self):
434 keep = ArvadosFileWriterTestCase.MockKeep({
435 "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
437 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
438 keep_client=keep) as c:
439 writer = c.open("count.txt", "rb+")
440 for b in range(0, 10):
441 writer.seek(10, os.SEEK_SET)
442 writer.write("abcdefghij")
444 self.assertEqual(writer.size(), 20)
445 self.assertEqual(b"0123456789abcdefghij", writer.readfrom(0, 20))
446 self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 ae5f43bab79cf0be33f025fa97ae7398+100 0:10:count.txt 100:10:count.txt\n", c.portable_manifest_text())
448 writer.arvadosfile.flush()
449 self.assertEqual(writer.size(), 20)
450 self.assertEqual(b"0123456789abcdefghij", writer.readfrom(0, 20))
451 self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:20:count.txt\n", c.portable_manifest_text())
453 def test_rewrite_over_existing_file(self):
454 keep = ArvadosFileWriterTestCase.MockKeep({
455 "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
457 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
458 keep_client=keep) as c:
459 writer = c.open("count.txt", "rb+")
460 for b in range(0, 10):
461 writer.seek(5, os.SEEK_SET)
462 writer.write("abcdefghij")
464 self.assertEqual(writer.size(), 15)
465 self.assertEqual(b"01234abcdefghij", writer.readfrom(0, 20))
466 self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 ae5f43bab79cf0be33f025fa97ae7398+100 0:5:count.txt 100:10:count.txt\n", c.portable_manifest_text())
468 writer.arvadosfile.flush()
470 self.assertEqual(writer.size(), 15)
471 self.assertEqual(b"01234abcdefghij", writer.readfrom(0, 20))
472 self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:5:count.txt 10:10:count.txt\n", c.portable_manifest_text())
474 def test_write_large_rewrite(self):
475 keep = ArvadosFileWriterTestCase.MockKeep({})
476 api = ArvadosFileWriterTestCase.MockApi({"name":"test_write_large",
477 "manifest_text": ". 3dc0d4bc21f48060bedcb2c91af4f906+32892003 a5de24f4417cfba9d5825eadc2f4ca49+67108000 0:3:count.txt 32892006:67107997:count.txt 3:32892000:count.txt\n",
478 "replication_desired":None},
479 {"uuid":"zzzzz-4zz18-mockcollection0",
480 "manifest_text": ". 3dc0d4bc21f48060bedcb2c91af4f906+32892003 a5de24f4417cfba9d5825eadc2f4ca49+67108000 0:3:count.txt 32892006:67107997:count.txt 3:32892000:count.txt\n",
481 "portable_data_hash":"217665c6b713e1b78dfba7ebd42344db+156"})
482 with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
483 api_client=api, keep_client=keep) as c:
484 writer = c.open("count.txt", "rb+")
485 text = b''.join([b"0123456789" for a in range(0, 100)])
486 for b in range(0, 100000):
488 writer.seek(0, os.SEEK_SET)
490 self.assertEqual(writer.size(), 100000000)
492 self.assertIsNone(c.manifest_locator())
493 self.assertTrue(c.modified())
494 c.save_new("test_write_large")
495 self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
496 self.assertFalse(c.modified())
498 def test_create(self):
499 keep = ArvadosFileWriterTestCase.MockKeep({})
500 api = ArvadosFileWriterTestCase.MockApi({
501 "name":"test_create",
502 "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
503 "replication_desired":None,
505 "uuid":"zzzzz-4zz18-mockcollection0",
506 "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
507 "portable_data_hash":"7a461a8c58601798f690f8b368ac4423+51",
509 with Collection(api_client=api, keep_client=keep) as c:
510 writer = c.open("count.txt", "wb+")
511 self.assertEqual(writer.size(), 0)
512 writer.write("01234567")
513 self.assertEqual(writer.size(), 8)
515 self.assertIsNone(c.manifest_locator())
516 self.assertTrue(c.modified())
517 self.assertIsNone(keep.get("2e9ec317e197819358fbc43afca7d837+8"))
518 c.save_new("test_create")
519 self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
520 self.assertFalse(c.modified())
521 self.assertEqual(b"01234567", keep.get("2e9ec317e197819358fbc43afca7d837+8"))
524 def test_create_subdir(self):
525 keep = ArvadosFileWriterTestCase.MockKeep({})
526 api = ArvadosFileWriterTestCase.MockApi({"name":"test_create",
527 "manifest_text":"./foo/bar 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
528 "replication_desired":None},
529 {"uuid":"zzzzz-4zz18-mockcollection0",
530 "manifest_text":"./foo/bar 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
531 "portable_data_hash":"1b02aaa62528d28a5be41651cbb9d7c7+59"})
532 with Collection(api_client=api, keep_client=keep) as c:
533 self.assertIsNone(c.api_response())
534 writer = c.open("foo/bar/count.txt", "wb+")
535 writer.write("01234567")
536 self.assertFalse(c.committed())
537 c.save_new("test_create")
538 self.assertTrue(c.committed())
539 self.assertEqual(c.api_response(), api.response)
541 def test_overwrite(self):
542 keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
543 api = ArvadosFileWriterTestCase.MockApi({"name":"test_overwrite",
544 "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
545 "replication_desired":None},
546 {"uuid":"zzzzz-4zz18-mockcollection0",
547 "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n",
548 "portable_data_hash":"7a461a8c58601798f690f8b368ac4423+51"})
549 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
550 api_client=api, keep_client=keep) as c:
551 writer = c.open("count.txt", "wb+")
552 self.assertEqual(writer.size(), 0)
553 writer.write("01234567")
554 self.assertEqual(writer.size(), 8)
556 self.assertIsNone(c.manifest_locator())
557 self.assertTrue(c.modified())
558 c.save_new("test_overwrite")
559 self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
560 self.assertFalse(c.modified())
562 def test_file_not_found(self):
563 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n') as c:
564 with self.assertRaises(IOError):
565 writer = c.open("nocount.txt", "rb")
567 def test_cannot_open_directory(self):
568 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n') as c:
569 with self.assertRaises(IOError):
570 writer = c.open(".", "rb")
572 def test_create_multiple(self):
573 keep = ArvadosFileWriterTestCase.MockKeep({})
574 api = ArvadosFileWriterTestCase.MockApi({"name":"test_create_multiple",
575 "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:8:count1.txt 8:8:count2.txt\n",
576 "replication_desired":None},
577 {"uuid":"zzzzz-4zz18-mockcollection0",
578 "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:8:count1.txt 8:8:count2.txt\n",
579 "portable_data_hash":"71e7bb6c00d31fc2b4364199fd97be08+102"})
580 with Collection(api_client=api, keep_client=keep) as c:
581 w1 = c.open("count1.txt", "wb")
582 w2 = c.open("count2.txt", "wb")
585 self.assertEqual(w1.size(), 8)
586 self.assertEqual(w2.size(), 8)
588 self.assertIsNone(c.manifest_locator())
589 self.assertTrue(c.modified())
590 self.assertIsNone(keep.get("2e9ec317e197819358fbc43afca7d837+8"))
591 c.save_new("test_create_multiple")
592 self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
593 self.assertFalse(c.modified())
594 self.assertEqual(b"01234567", keep.get("2e9ec317e197819358fbc43afca7d837+8"))
597 class ArvadosFileReaderTestCase(StreamFileReaderTestCase):
598 class MockParent(object):
599 class MockBlockMgr(object):
600 def __init__(self, blocks, nocache):
602 self.nocache = nocache
604 def block_prefetch(self, loc):
607 def get_block_contents(self, loc, num_retries=0, cache_only=False):
608 if self.nocache and cache_only:
610 return self.blocks[loc]
612 def __init__(self, blocks, nocache):
614 self.nocache = nocache
615 self.lock = arvados.arvfile.NoopLock()
617 def root_collection(self):
620 def _my_block_manager(self):
621 return ArvadosFileReaderTestCase.MockParent.MockBlockMgr(self.blocks, self.nocache)
624 def make_count_reader(self, nocache=False):
628 for d in [b'01234', b'34567', b'67890']:
629 loc = tutil.str_keep_locator(d)
631 stream.append(Range(loc, n, len(d)))
633 af = ArvadosFile(ArvadosFileReaderTestCase.MockParent(blocks, nocache), "count.txt", stream=stream, segments=[Range(1, 0, 3), Range(6, 3, 3), Range(11, 6, 3)])
634 return ArvadosFileReader(af, mode="rb")
636 def test_read_block_crossing_behavior(self):
637 # read() needs to return all the data requested if possible, even if it
638 # crosses uncached blocks: https://arvados.org/issues/5856
639 sfile = self.make_count_reader(nocache=True)
640 self.assertEqual(b'12345678', sfile.read(8))
642 def test_successive_reads(self):
643 # Override StreamFileReaderTestCase.test_successive_reads
644 sfile = self.make_count_reader(nocache=True)
645 self.assertEqual(b'1234', sfile.read(4))
646 self.assertEqual(b'5678', sfile.read(4))
647 self.assertEqual(b'9', sfile.read(4))
648 self.assertEqual(b'', sfile.read(4))
650 def test_tell_after_block_read(self):
651 # Override StreamFileReaderTestCase.test_tell_after_block_read
652 sfile = self.make_count_reader(nocache=True)
653 self.assertEqual(b'12345678', sfile.read(8))
654 self.assertEqual(8, sfile.tell())
656 def test_prefetch(self):
657 keep = ArvadosFileWriterTestCase.MockKeep({
658 "2e9ec317e197819358fbc43afca7d837+8": b"01234567",
659 "e8dc4081b13434b45189a720b77b6818+8": b"abcdefgh",
661 with Collection(". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:16:count.txt\n", keep_client=keep) as c:
662 r = c.open("count.txt", "rb")
663 self.assertEqual(b"0123", r.read(4))
664 self.assertIn("2e9ec317e197819358fbc43afca7d837+8", keep.requests)
665 self.assertIn("e8dc4081b13434b45189a720b77b6818+8", keep.requests)
667 def test__eq__from_manifest(self):
668 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
669 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c2:
670 self.assertTrue(c1["count1.txt"] == c2["count1.txt"])
671 self.assertFalse(c1["count1.txt"] != c2["count1.txt"])
673 def test__eq__from_writes(self):
674 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
675 with Collection() as c2:
676 f = c2.open("count1.txt", "wb")
677 f.write("0123456789")
679 self.assertTrue(c1["count1.txt"] == c2["count1.txt"])
680 self.assertFalse(c1["count1.txt"] != c2["count1.txt"])
682 def test__ne__(self):
683 with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
684 with Collection() as c2:
685 f = c2.open("count1.txt", "wb")
686 f.write("1234567890")
688 self.assertTrue(c1["count1.txt"] != c2["count1.txt"])
689 self.assertFalse(c1["count1.txt"] == c2["count1.txt"])
692 class ArvadosFileReadTestCase(unittest.TestCase, StreamRetryTestMixin):
693 def reader_for(self, coll_name, **kwargs):
697 for d in self.manifest_for(coll_name).split():
700 segments.append(Range(n, n, k.size))
701 stream.append(Range(d, n, k.size))
706 blockmanager = arvados.arvfile._BlockManager(self.keep_client())
707 blockmanager.prefetch_enabled = False
708 col = Collection(keep_client=self.keep_client(), block_manager=blockmanager)
709 af = ArvadosFile(col, "test",
712 kwargs.setdefault('mode', 'rb')
713 return ArvadosFileReader(af, **kwargs)
715 def read_for_test(self, reader, byte_count, **kwargs):
716 return reader.read(byte_count, **kwargs)
719 class ArvadosFileReadFromTestCase(ArvadosFileReadTestCase):
720 def read_for_test(self, reader, byte_count, **kwargs):
721 return reader.readfrom(0, byte_count, **kwargs)
724 class ArvadosFileReadAllTestCase(ArvadosFileReadTestCase):
725 def read_for_test(self, reader, byte_count, **kwargs):
726 return b''.join(reader.readall(**kwargs))
729 class ArvadosFileReadAllDecompressedTestCase(ArvadosFileReadTestCase):
730 def read_for_test(self, reader, byte_count, **kwargs):
731 return b''.join(reader.readall_decompressed(**kwargs))
734 class ArvadosFileReadlinesTestCase(ArvadosFileReadTestCase):
735 def read_for_test(self, reader, byte_count, **kwargs):
736 return ''.join(reader.readlines(**kwargs)).encode()
739 class ArvadosFileTestCase(unittest.TestCase):
740 def datetime_to_hex(self, dt):
741 return hex(int(time.mktime(dt.timetuple())))[2:]
743 def test_permission_expired(self):
744 base_manifest = ". 781e5e245d69b566979b86e28d23f2c7+10+A715fd31f8111894f717eb1003c1b0216799dd9ec@{} 0:10:count.txt\n"
745 now = datetime.datetime.now()
746 a_week_ago = now - datetime.timedelta(days=7)
747 a_month_ago = now - datetime.timedelta(days=30)
748 a_week_from_now = now + datetime.timedelta(days=7)
749 with Collection(base_manifest.format(self.datetime_to_hex(a_week_from_now))) as c:
750 self.assertFalse(c.find('count.txt').permission_expired())
751 with Collection(base_manifest.format(self.datetime_to_hex(a_week_ago))) as c:
752 f = c.find('count.txt')
753 self.assertTrue(f.permission_expired())
754 self.assertTrue(f.permission_expired(a_week_from_now))
755 self.assertFalse(f.permission_expired(a_month_ago))
758 class BlockManagerTest(unittest.TestCase):
759 def test_bufferblock_append(self):
760 keep = ArvadosFileWriterTestCase.MockKeep({})
761 with arvados.arvfile._BlockManager(keep) as blockmanager:
762 bufferblock = blockmanager.alloc_bufferblock()
763 bufferblock.append("foo")
765 self.assertEqual(bufferblock.size(), 3)
766 self.assertEqual(bufferblock.buffer_view[0:3], b"foo")
767 self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3")
769 bufferblock.append("bar")
771 self.assertEqual(bufferblock.size(), 6)
772 self.assertEqual(bufferblock.buffer_view[0:6], b"foobar")
773 self.assertEqual(bufferblock.locator(), "3858f62230ac3c915f300c664312c63f+6")
775 bufferblock.set_state(arvados.arvfile._BufferBlock.PENDING)
776 with self.assertRaises(arvados.errors.AssertionError):
777 bufferblock.append("bar")
779 def test_bufferblock_dup(self):
780 keep = ArvadosFileWriterTestCase.MockKeep({})
781 with arvados.arvfile._BlockManager(keep) as blockmanager:
782 bufferblock = blockmanager.alloc_bufferblock()
783 bufferblock.append("foo")
785 self.assertEqual(bufferblock.size(), 3)
786 self.assertEqual(bufferblock.buffer_view[0:3], b"foo")
787 self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3")
788 bufferblock.set_state(arvados.arvfile._BufferBlock.PENDING)
790 bufferblock2 = blockmanager.dup_block(bufferblock, None)
791 self.assertNotEqual(bufferblock.blockid, bufferblock2.blockid)
793 bufferblock2.append("bar")
795 self.assertEqual(bufferblock2.size(), 6)
796 self.assertEqual(bufferblock2.buffer_view[0:6], b"foobar")
797 self.assertEqual(bufferblock2.locator(), "3858f62230ac3c915f300c664312c63f+6")
799 self.assertEqual(bufferblock.size(), 3)
800 self.assertEqual(bufferblock.buffer_view[0:3], b"foo")
801 self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3")
803 def test_bufferblock_get(self):
804 keep = ArvadosFileWriterTestCase.MockKeep({
805 "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
807 with arvados.arvfile._BlockManager(keep) as blockmanager:
808 bufferblock = blockmanager.alloc_bufferblock()
809 bufferblock.append("foo")
811 self.assertEqual(blockmanager.get_block_contents("781e5e245d69b566979b86e28d23f2c7+10", 1), b"0123456789")
812 self.assertEqual(blockmanager.get_block_contents(bufferblock.blockid, 1), b"foo")
814 def test_bufferblock_commit(self):
815 mockkeep = mock.MagicMock()
816 with arvados.arvfile._BlockManager(mockkeep) as blockmanager:
817 bufferblock = blockmanager.alloc_bufferblock()
818 bufferblock.owner = mock.MagicMock()
819 def flush(sync=None):
820 blockmanager.commit_bufferblock(bufferblock, sync)
821 bufferblock.owner.flush.side_effect = flush
822 bufferblock.append("foo")
823 blockmanager.commit_all()
824 self.assertTrue(bufferblock.owner.flush.called)
825 self.assertTrue(mockkeep.put.called)
826 self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.COMMITTED)
827 self.assertIsNone(bufferblock.buffer_view)
829 def test_bufferblock_commit_pending(self):
831 mockkeep = mock.MagicMock()
832 mockkeep.put.side_effect = lambda x: time.sleep(1)
833 with arvados.arvfile._BlockManager(mockkeep) as blockmanager:
834 bufferblock = blockmanager.alloc_bufferblock()
835 bufferblock.append("foo")
837 blockmanager.commit_bufferblock(bufferblock, False)
838 self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.PENDING)
840 blockmanager.commit_bufferblock(bufferblock, True)
841 self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.COMMITTED)
844 def test_bufferblock_commit_with_error(self):
845 mockkeep = mock.MagicMock()
846 mockkeep.put.side_effect = arvados.errors.KeepWriteError("fail")
847 with arvados.arvfile._BlockManager(mockkeep) as blockmanager:
848 bufferblock = blockmanager.alloc_bufferblock()
849 bufferblock.owner = mock.MagicMock()
850 def flush(sync=None):
851 blockmanager.commit_bufferblock(bufferblock, sync)
852 bufferblock.owner.flush.side_effect = flush
853 bufferblock.append("foo")
854 with self.assertRaises(arvados.errors.KeepWriteError) as err:
855 blockmanager.commit_all()
856 self.assertTrue(bufferblock.owner.flush.called)
857 self.assertEqual(str(err.exception), "Error writing some blocks: block acbd18db4cc2f85cedef654fccc4a4d8+3 raised KeepWriteError (fail)")
858 self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.ERROR)