3 # ARVADOS_API_TOKEN=abc ARVADOS_API_HOST=arvados.local python -m unittest discover
17 class TestResumableWriter(arvados.ResumableCollectionWriter):
18 KEEP_BLOCK_SIZE = 1024 # PUT to Keep every 1K.
21 self.saved_states = []
22 return super(TestResumableWriter, self).__init__()
24 def checkpoint_state(self):
25 self.saved_states.append(self.dump_state(copy.deepcopy))
28 assert self.saved_states, "resumable writer did not save any state"
29 return self.saved_states[-1]
32 class ArvadosCollectionsTest(unittest.TestCase):
33 def _make_tmpdir(self):
34 self._tempdirs.append(tempfile.mkdtemp())
35 return self._tempdirs[-1]
38 self._orig_keep_local_store = os.environ.get('KEEP_LOCAL_STORE')
40 os.environ['KEEP_LOCAL_STORE'] = self._make_tmpdir()
43 for workdir in self._tempdirs:
44 shutil.rmtree(workdir, ignore_errors=True)
45 if self._orig_keep_local_store is None:
46 del os.environ['KEEP_LOCAL_STORE']
48 os.environ['KEEP_LOCAL_STORE'] = self._orig_keep_local_store
50 def write_foo_bar_baz(self):
51 cw = arvados.CollectionWriter()
52 self.assertEqual(cw.current_stream_name(), '.',
53 'current_stream_name() should be "." now')
54 cw.set_current_file_name('foo.txt')
56 self.assertEqual(cw.current_file_name(), 'foo.txt',
57 'current_file_name() should be foo.txt now')
58 cw.start_new_file('bar.txt')
60 cw.start_new_stream('baz')
62 cw.set_current_file_name('baz.txt')
65 def test_keep_local_store(self):
66 self.assertEqual(arvados.Keep.put('foo'), 'acbd18db4cc2f85cedef654fccc4a4d8+3', 'wrong md5 hash from Keep.put')
67 self.assertEqual(arvados.Keep.get('acbd18db4cc2f85cedef654fccc4a4d8+3'), 'foo', 'wrong data from Keep.get')
69 def test_local_collection_writer(self):
70 self.assertEqual(self.write_foo_bar_baz(),
71 'd6c3b8e571f1b81ebb150a45ed06c884+114',
72 "wrong locator hash for files foo, bar, baz")
74 def test_local_collection_reader(self):
75 self.write_foo_bar_baz()
76 cr = arvados.CollectionReader('d6c3b8e571f1b81ebb150a45ed06c884+114+Xzizzle')
78 for s in cr.all_streams():
79 for f in s.all_files():
80 got += [[f.size(), f.stream_name(), f.name(), f.read(2**26)]]
81 expected = [[3, '.', 'bar.txt', 'bar'],
82 [3, '.', 'foo.txt', 'foo'],
83 [3, './baz', 'baz.txt', 'baz']]
86 stream0 = cr.all_streams()[0]
87 self.assertEqual(stream0.readfrom(0, 0),
89 'reading zero bytes should have returned empty string')
90 self.assertEqual(stream0.readfrom(0, 2**26),
92 'reading entire stream failed')
93 self.assertEqual(stream0.readfrom(2**26, 0),
95 'reading zero bytes should have returned empty string')
97 def _test_subset(self, collection, expected):
98 cr = arvados.CollectionReader(collection)
99 for s in cr.all_streams():
103 got = [f.size(), f.stream_name(), f.name(), "".join(f.readall(2**26))]
104 self.assertEqual(got,
106 'all_files|as_manifest did not preserve manifest contents: got %s expected %s' % (got, ex))
108 def test_collection_manifest_subset(self):
109 self.write_foo_bar_baz()
110 self._test_subset('d6c3b8e571f1b81ebb150a45ed06c884+114',
111 [[3, '.', 'bar.txt', 'bar'],
112 [3, '.', 'foo.txt', 'foo'],
113 [3, './baz', 'baz.txt', 'baz']])
114 self._test_subset((". %s %s 0:3:foo.txt 3:3:bar.txt\n" %
115 (arvados.Keep.put("foo"),
116 arvados.Keep.put("bar"))),
117 [[3, '.', 'bar.txt', 'bar'],
118 [3, '.', 'foo.txt', 'foo']])
119 self._test_subset((". %s %s 0:2:fo.txt 2:4:obar.txt\n" %
120 (arvados.Keep.put("foo"),
121 arvados.Keep.put("bar"))),
122 [[2, '.', 'fo.txt', 'fo'],
123 [4, '.', 'obar.txt', 'obar']])
124 self._test_subset((". %s %s 0:2:fo.txt 2:0:zero.txt 2:2:ob.txt 4:2:ar.txt\n" %
125 (arvados.Keep.put("foo"),
126 arvados.Keep.put("bar"))),
127 [[2, '.', 'ar.txt', 'ar'],
128 [2, '.', 'fo.txt', 'fo'],
129 [2, '.', 'ob.txt', 'ob'],
130 [0, '.', 'zero.txt', '']])
132 def _test_readline(self, what_in, what_out):
133 cw = arvados.CollectionWriter()
134 cw.start_new_file('test.txt')
137 cr = arvados.CollectionReader(test1)
139 for x in list(cr.all_files())[0].readlines():
141 self.assertEqual(got,
143 "readlines did not split lines correctly: %s" % got)
145 def test_collection_readline(self):
146 self._test_readline("\na\nbcd\n\nefg\nz",
147 ["\n", "a\n", "bcd\n", "\n", "efg\n", "z"])
148 self._test_readline("ab\ncd\n",
151 def test_collection_empty_file(self):
152 cw = arvados.CollectionWriter()
153 cw.start_new_file('zero.txt')
156 self.assertEqual(cw.manifest_text(), ". 0:0:zero.txt\n")
157 self.check_manifest_file_sizes(cw.manifest_text(), [0])
158 cw = arvados.CollectionWriter()
159 cw.start_new_file('zero.txt')
161 cw.start_new_file('one.txt')
163 cw.start_new_stream('foo')
164 cw.start_new_file('zero.txt')
166 self.check_manifest_file_sizes(cw.manifest_text(), [1,0,0])
168 def check_manifest_file_sizes(self, manifest_text, expect_sizes):
169 cr = arvados.CollectionReader(manifest_text)
171 for f in cr.all_files():
172 got_sizes += [f.size()]
173 self.assertEqual(got_sizes, expect_sizes, "got wrong file sizes %s, expected %s" % (got_sizes, expect_sizes))
175 def test_collection_bz2_decompression(self):
178 for x in xrange(0, 18):
180 compressed_data_in = bz2.compress(data_in)
181 cw = arvados.CollectionWriter()
182 cw.start_new_file('test.bz2')
183 cw.write(compressed_data_in)
184 bz2_manifest = cw.manifest_text()
186 cr = arvados.CollectionReader(bz2_manifest)
189 for x in list(cr.all_files())[0].readlines():
190 self.assertEqual(x, "abc\n", "decompression returned wrong data: %s" % x)
192 self.assertEqual(got,
194 "decompression returned %d lines instead of %d" % (got, n_lines_in))
196 def test_collection_gzip_decompression(self):
199 for x in xrange(0, 18):
201 p = subprocess.Popen(["gzip", "-1cn"],
202 stdout=subprocess.PIPE,
203 stdin=subprocess.PIPE,
204 stderr=subprocess.PIPE,
205 shell=False, close_fds=True)
206 compressed_data_in, stderrdata = p.communicate(data_in)
208 cw = arvados.CollectionWriter()
209 cw.start_new_file('test.gz')
210 cw.write(compressed_data_in)
211 gzip_manifest = cw.manifest_text()
213 cr = arvados.CollectionReader(gzip_manifest)
215 for x in list(cr.all_files())[0].readlines():
216 self.assertEqual(x, "abc\n", "decompression returned wrong data: %s" % x)
218 self.assertEqual(got,
220 "decompression returned %d lines instead of %d" % (got, n_lines_in))
222 def data_file(self, filename):
224 basedir = os.path.dirname(__file__)
227 return open(os.path.join(basedir, 'data', filename))
229 def test_normalized_collection(self):
230 m1 = """. 5348b82a029fd9e971a811ce1f71360b+43 0:43:md5sum.txt
231 . 085c37f02916da1cad16f93c54d899b7+41 0:41:md5sum.txt
232 . 8b22da26f9f433dea0a10e5ec66d73ba+43 0:43:md5sum.txt"""
233 self.assertEqual(arvados.CollectionReader(m1).manifest_text(),
234 """. 5348b82a029fd9e971a811ce1f71360b+43 085c37f02916da1cad16f93c54d899b7+41 8b22da26f9f433dea0a10e5ec66d73ba+43 0:127:md5sum.txt
237 m2 = """. 204e43b8a1185621ca55a94839582e6f+67108864 b9677abbac956bd3e86b1deb28dfac03+67108864 fc15aff2a762b13f521baf042140acec+67108864 323d2a3ce20370c4ca1d3462a344f8fd+25885655 0:227212247:var-GS000016015-ASM.tsv.bz2
239 self.assertEqual(arvados.CollectionReader(m2).manifest_text(), m2)
241 m3 = """. 5348b82a029fd9e971a811ce1f71360b+43 3:40:md5sum.txt
242 . 085c37f02916da1cad16f93c54d899b7+41 0:41:md5sum.txt
243 . 8b22da26f9f433dea0a10e5ec66d73ba+43 0:43:md5sum.txt"""
244 self.assertEqual(arvados.CollectionReader(m3).manifest_text(),
245 """. 5348b82a029fd9e971a811ce1f71360b+43 085c37f02916da1cad16f93c54d899b7+41 8b22da26f9f433dea0a10e5ec66d73ba+43 3:124:md5sum.txt
248 m4 = """. 204e43b8a1185621ca55a94839582e6f+67108864 0:3:foo/bar
249 ./zzz 204e43b8a1185621ca55a94839582e6f+67108864 0:999:zzz
250 ./foo 323d2a3ce20370c4ca1d3462a344f8fd+25885655 0:3:bar"""
251 self.assertEqual(arvados.CollectionReader(m4).manifest_text(),
252 """./foo 204e43b8a1185621ca55a94839582e6f+67108864 323d2a3ce20370c4ca1d3462a344f8fd+25885655 0:3:bar 67108864:3:bar
253 ./zzz 204e43b8a1185621ca55a94839582e6f+67108864 0:999:zzz
256 m5 = """. 204e43b8a1185621ca55a94839582e6f+67108864 0:3:foo/bar
257 ./zzz 204e43b8a1185621ca55a94839582e6f+67108864 0:999:zzz
258 ./foo 204e43b8a1185621ca55a94839582e6f+67108864 3:3:bar"""
259 self.assertEqual(arvados.CollectionReader(m5).manifest_text(),
260 """./foo 204e43b8a1185621ca55a94839582e6f+67108864 0:6:bar
261 ./zzz 204e43b8a1185621ca55a94839582e6f+67108864 0:999:zzz
264 with self.data_file('1000G_ref_manifest') as f6:
266 self.assertEqual(arvados.CollectionReader(m6).manifest_text(), m6)
268 with self.data_file('jlake_manifest') as f7:
270 self.assertEqual(arvados.CollectionReader(m7).manifest_text(), m7)
272 m8 = """./a\\040b\\040c 59ca0efa9f5633cb0371bbc0355478d8+13 0:13:hello\\040world.txt
274 self.assertEqual(arvados.CollectionReader(m8).manifest_text(), m8)
276 def test_locators_and_ranges(self):
277 blocks2 = [['a', 10, 0],
284 self.assertEqual(arvados.locators_and_ranges(blocks2, 2, 2), [['a', 10, 2, 2]])
285 self.assertEqual(arvados.locators_and_ranges(blocks2, 12, 2), [['b', 10, 2, 2]])
286 self.assertEqual(arvados.locators_and_ranges(blocks2, 22, 2), [['c', 10, 2, 2]])
287 self.assertEqual(arvados.locators_and_ranges(blocks2, 32, 2), [['d', 10, 2, 2]])
288 self.assertEqual(arvados.locators_and_ranges(blocks2, 42, 2), [['e', 10, 2, 2]])
289 self.assertEqual(arvados.locators_and_ranges(blocks2, 52, 2), [['f', 10, 2, 2]])
290 self.assertEqual(arvados.locators_and_ranges(blocks2, 62, 2), [])
291 self.assertEqual(arvados.locators_and_ranges(blocks2, -2, 2), [])
293 self.assertEqual(arvados.locators_and_ranges(blocks2, 0, 2), [['a', 10, 0, 2]])
294 self.assertEqual(arvados.locators_and_ranges(blocks2, 10, 2), [['b', 10, 0, 2]])
295 self.assertEqual(arvados.locators_and_ranges(blocks2, 20, 2), [['c', 10, 0, 2]])
296 self.assertEqual(arvados.locators_and_ranges(blocks2, 30, 2), [['d', 10, 0, 2]])
297 self.assertEqual(arvados.locators_and_ranges(blocks2, 40, 2), [['e', 10, 0, 2]])
298 self.assertEqual(arvados.locators_and_ranges(blocks2, 50, 2), [['f', 10, 0, 2]])
299 self.assertEqual(arvados.locators_and_ranges(blocks2, 60, 2), [])
300 self.assertEqual(arvados.locators_and_ranges(blocks2, -2, 2), [])
302 self.assertEqual(arvados.locators_and_ranges(blocks2, 9, 2), [['a', 10, 9, 1], ['b', 10, 0, 1]])
303 self.assertEqual(arvados.locators_and_ranges(blocks2, 19, 2), [['b', 10, 9, 1], ['c', 10, 0, 1]])
304 self.assertEqual(arvados.locators_and_ranges(blocks2, 29, 2), [['c', 10, 9, 1], ['d', 10, 0, 1]])
305 self.assertEqual(arvados.locators_and_ranges(blocks2, 39, 2), [['d', 10, 9, 1], ['e', 10, 0, 1]])
306 self.assertEqual(arvados.locators_and_ranges(blocks2, 49, 2), [['e', 10, 9, 1], ['f', 10, 0, 1]])
307 self.assertEqual(arvados.locators_and_ranges(blocks2, 59, 2), [['f', 10, 9, 1]])
310 blocks3 = [['a', 10, 0],
318 self.assertEqual(arvados.locators_and_ranges(blocks3, 2, 2), [['a', 10, 2, 2]])
319 self.assertEqual(arvados.locators_and_ranges(blocks3, 12, 2), [['b', 10, 2, 2]])
320 self.assertEqual(arvados.locators_and_ranges(blocks3, 22, 2), [['c', 10, 2, 2]])
321 self.assertEqual(arvados.locators_and_ranges(blocks3, 32, 2), [['d', 10, 2, 2]])
322 self.assertEqual(arvados.locators_and_ranges(blocks3, 42, 2), [['e', 10, 2, 2]])
323 self.assertEqual(arvados.locators_and_ranges(blocks3, 52, 2), [['f', 10, 2, 2]])
324 self.assertEqual(arvados.locators_and_ranges(blocks3, 62, 2), [['g', 10, 2, 2]])
327 blocks = [['a', 10, 0],
330 self.assertEqual(arvados.locators_and_ranges(blocks, 1, 0), [])
331 self.assertEqual(arvados.locators_and_ranges(blocks, 0, 5), [['a', 10, 0, 5]])
332 self.assertEqual(arvados.locators_and_ranges(blocks, 3, 5), [['a', 10, 3, 5]])
333 self.assertEqual(arvados.locators_and_ranges(blocks, 0, 10), [['a', 10, 0, 10]])
335 self.assertEqual(arvados.locators_and_ranges(blocks, 0, 11), [['a', 10, 0, 10],
337 self.assertEqual(arvados.locators_and_ranges(blocks, 1, 11), [['a', 10, 1, 9],
339 self.assertEqual(arvados.locators_and_ranges(blocks, 0, 25), [['a', 10, 0, 10],
342 self.assertEqual(arvados.locators_and_ranges(blocks, 0, 30), [['a', 10, 0, 10],
345 self.assertEqual(arvados.locators_and_ranges(blocks, 1, 30), [['a', 10, 1, 9],
348 self.assertEqual(arvados.locators_and_ranges(blocks, 0, 31), [['a', 10, 0, 10],
352 self.assertEqual(arvados.locators_and_ranges(blocks, 15, 5), [['b', 15, 5, 5]])
354 self.assertEqual(arvados.locators_and_ranges(blocks, 8, 17), [['a', 10, 8, 2],
357 self.assertEqual(arvados.locators_and_ranges(blocks, 8, 20), [['a', 10, 8, 2],
361 self.assertEqual(arvados.locators_and_ranges(blocks, 26, 2), [['c', 5, 1, 2]])
363 self.assertEqual(arvados.locators_and_ranges(blocks, 9, 15), [['a', 10, 9, 1],
365 self.assertEqual(arvados.locators_and_ranges(blocks, 10, 15), [['b', 15, 0, 15]])
366 self.assertEqual(arvados.locators_and_ranges(blocks, 11, 15), [['b', 15, 1, 14],
369 class MockStreamReader(object):
370 def __init__(self, content):
371 self.content = content
373 def readfrom(self, start, size):
374 return self.content[start:start+size]
376 def test_file_stream(self):
377 content = 'abcdefghijklmnopqrstuvwxyz0123456789'
378 msr = self.MockStreamReader(content)
379 segments = [[0, 10, 0],
383 sfr = arvados.StreamFileReader(msr, segments, "test")
385 self.assertEqual(sfr.name(), "test")
386 self.assertEqual(sfr.size(), 30)
388 self.assertEqual(sfr.readfrom(0, 30), content[0:30])
389 self.assertEqual(sfr.readfrom(2, 30), content[2:30])
391 self.assertEqual(sfr.readfrom(2, 8), content[2:10])
392 self.assertEqual(sfr.readfrom(0, 10), content[0:10])
394 self.assertEqual(sfr.tell(), 0)
395 self.assertEqual(sfr.read(5), content[0:5])
396 self.assertEqual(sfr.tell(), 5)
397 self.assertEqual(sfr.read(5), content[5:10])
398 self.assertEqual(sfr.tell(), 10)
399 self.assertEqual(sfr.read(5), content[10:15])
400 self.assertEqual(sfr.tell(), 15)
401 self.assertEqual(sfr.read(5), content[15:20])
402 self.assertEqual(sfr.tell(), 20)
403 self.assertEqual(sfr.read(5), content[20:25])
404 self.assertEqual(sfr.tell(), 25)
405 self.assertEqual(sfr.read(5), content[25:30])
406 self.assertEqual(sfr.tell(), 30)
407 self.assertEqual(sfr.read(5), '')
408 self.assertEqual(sfr.tell(), 30)
410 segments = [[26, 10, 0],
414 sfr = arvados.StreamFileReader(msr, segments, "test")
416 self.assertEqual(sfr.size(), 30)
418 self.assertEqual(sfr.readfrom(0, 30), content[26:36] + content[0:20])
419 self.assertEqual(sfr.readfrom(2, 30), content[28:36] + content[0:20])
421 self.assertEqual(sfr.readfrom(2, 8), content[28:36])
422 self.assertEqual(sfr.readfrom(0, 10), content[26:36])
424 self.assertEqual(sfr.tell(), 0)
425 self.assertEqual(sfr.read(5), content[26:31])
426 self.assertEqual(sfr.tell(), 5)
427 self.assertEqual(sfr.read(5), content[31:36])
428 self.assertEqual(sfr.tell(), 10)
429 self.assertEqual(sfr.read(5), content[0:5])
430 self.assertEqual(sfr.tell(), 15)
431 self.assertEqual(sfr.read(5), content[5:10])
432 self.assertEqual(sfr.tell(), 20)
433 self.assertEqual(sfr.read(5), content[10:15])
434 self.assertEqual(sfr.tell(), 25)
435 self.assertEqual(sfr.read(5), content[15:20])
436 self.assertEqual(sfr.tell(), 30)
437 self.assertEqual(sfr.read(5), '')
438 self.assertEqual(sfr.tell(), 30)
441 class MockKeep(object):
442 def __init__(self, content):
443 self.content = content
445 def get(self, locator):
446 return self.content[locator]
448 def test_stream_reader(self):
449 keepblocks = {'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa+10': 'abcdefghij',
450 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb+15': 'klmnopqrstuvwxy',
451 'cccccccccccccccccccccccccccccccc+5': 'z0123'}
452 mk = self.MockKeep(keepblocks)
454 sr = arvados.StreamReader([".", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa+10", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb+15", "cccccccccccccccccccccccccccccccc+5", "0:30:foo"], mk)
456 content = 'abcdefghijklmnopqrstuvwxyz0123456789'
458 self.assertEqual(sr.readfrom(0, 30), content[0:30])
459 self.assertEqual(sr.readfrom(2, 30), content[2:30])
461 self.assertEqual(sr.readfrom(2, 8), content[2:10])
462 self.assertEqual(sr.readfrom(0, 10), content[0:10])
464 self.assertEqual(sr.readfrom(0, 5), content[0:5])
465 self.assertEqual(sr.readfrom(5, 5), content[5:10])
466 self.assertEqual(sr.readfrom(10, 5), content[10:15])
467 self.assertEqual(sr.readfrom(15, 5), content[15:20])
468 self.assertEqual(sr.readfrom(20, 5), content[20:25])
469 self.assertEqual(sr.readfrom(25, 5), content[25:30])
470 self.assertEqual(sr.readfrom(30, 5), '')
472 def test_extract_file(self):
473 m1 = """. 5348b82a029fd9e971a811ce1f71360b+43 0:43:md5sum.txt
474 . 085c37f02916da1cad16f93c54d899b7+41 0:41:md6sum.txt
475 . 8b22da26f9f433dea0a10e5ec66d73ba+43 0:43:md7sum.txt
476 . 085c37f02916da1cad16f93c54d899b7+41 5348b82a029fd9e971a811ce1f71360b+43 8b22da26f9f433dea0a10e5ec66d73ba+43 47:80:md8sum.txt
477 . 085c37f02916da1cad16f93c54d899b7+41 5348b82a029fd9e971a811ce1f71360b+43 8b22da26f9f433dea0a10e5ec66d73ba+43 40:80:md9sum.txt"""
479 m2 = arvados.CollectionReader(m1)
481 self.assertEqual(m2.manifest_text(),
482 ". 5348b82a029fd9e971a811ce1f71360b+43 085c37f02916da1cad16f93c54d899b7+41 8b22da26f9f433dea0a10e5ec66d73ba+43 0:43:md5sum.txt 43:41:md6sum.txt 84:43:md7sum.txt 6:37:md8sum.txt 84:43:md8sum.txt 83:1:md9sum.txt 0:43:md9sum.txt 84:36:md9sum.txt\n")
484 self.assertEqual(arvados.CollectionReader(m1).all_streams()[0].files()['md5sum.txt'].as_manifest(),
485 ". 5348b82a029fd9e971a811ce1f71360b+43 0:43:md5sum.txt\n")
486 self.assertEqual(arvados.CollectionReader(m1).all_streams()[0].files()['md6sum.txt'].as_manifest(),
487 ". 085c37f02916da1cad16f93c54d899b7+41 0:41:md6sum.txt\n")
488 self.assertEqual(arvados.CollectionReader(m1).all_streams()[0].files()['md7sum.txt'].as_manifest(),
489 ". 8b22da26f9f433dea0a10e5ec66d73ba+43 0:43:md7sum.txt\n")
490 self.assertEqual(arvados.CollectionReader(m1).all_streams()[0].files()['md9sum.txt'].as_manifest(),
491 ". 085c37f02916da1cad16f93c54d899b7+41 5348b82a029fd9e971a811ce1f71360b+43 8b22da26f9f433dea0a10e5ec66d73ba+43 40:80:md9sum.txt\n")
493 def build_directory_tree(self, tree=['basefile', 'subdir/subfile']):
494 tree_root = self._make_tmpdir()
496 path = os.path.join(tree_root, leaf)
498 os.makedirs(os.path.dirname(path))
499 except OSError as error:
500 if error.errno != errno.EEXIST:
502 with open(path, 'w') as tmpfile:
506 def test_write_directory_tree(self):
507 cwriter = arvados.CollectionWriter()
508 cwriter.write_directory_tree(self.build_directory_tree())
509 self.assertEqual(cwriter.manifest_text(),
510 """. c5110c5ac93202d8e0f9e381f22bac0f+8 0:8:basefile
511 ./subdir 1ca4dec89403084bf282ad31e6cf7972+14 0:14:subfile\n""")
513 def test_write_named_directory_tree(self):
514 cwriter = arvados.CollectionWriter()
515 cwriter.write_directory_tree(self.build_directory_tree(), 'root')
517 cwriter.manifest_text(),
518 """./root c5110c5ac93202d8e0f9e381f22bac0f+8 0:8:basefile
519 ./root/subdir 1ca4dec89403084bf282ad31e6cf7972+14 0:14:subfile\n""")
521 def test_write_directory_tree_in_one_stream(self):
522 cwriter = arvados.CollectionWriter()
523 cwriter.write_directory_tree(self.build_directory_tree(),
524 max_manifest_depth=0)
525 self.assertEqual(cwriter.manifest_text(),
526 """. 4ace875ffdc6824a04950f06858f4465+22 0:8:basefile
527 ./subdir 4ace875ffdc6824a04950f06858f4465+22 8:14:subfile\n""")
529 def test_write_directory_tree_with_limited_recursion(self):
530 cwriter = arvados.CollectionWriter()
531 cwriter.write_directory_tree(
532 self.build_directory_tree(['f1', 'd1/f2', 'd1/d2/f3']),
533 max_manifest_depth=1)
534 self.assertEqual(cwriter.manifest_text(),
535 """. bd19836ddb62c11c55ab251ccaca5645+2 0:2:f1
536 ./d1 50170217e5b04312024aa5cd42934494+13 8:5:f2
537 ./d1/d2 50170217e5b04312024aa5cd42934494+13 0:8:f3\n""")
539 def make_test_file(self, text="test"):
540 testfile = tempfile.NamedTemporaryFile()
545 def test_write_one_file(self):
546 cwriter = arvados.CollectionWriter()
547 with self.make_test_file() as testfile:
548 cwriter.write_file(testfile.name)
550 cwriter.manifest_text(),
551 ". 098f6bcd4621d373cade4e832627b4f6+4 0:4:{}\n".format(
552 os.path.basename(testfile.name)))
554 def test_write_named_file(self):
555 cwriter = arvados.CollectionWriter()
556 with self.make_test_file() as testfile:
557 cwriter.write_file(testfile.name, 'foo')
558 self.assertEqual(cwriter.manifest_text(),
559 ". 098f6bcd4621d373cade4e832627b4f6+4 0:4:foo\n")
561 def test_write_multiple_files(self):
562 cwriter = arvados.CollectionWriter()
564 with self.make_test_file(letter) as testfile:
565 cwriter.write_file(testfile.name, letter)
567 cwriter.manifest_text(),
568 ". 902fbdd2b1df0c4f70b4a5d23525e932+3 0:1:A 1:1:B 2:1:C\n")
570 def test_checkpoint_after_put(self):
571 cwriter = TestResumableWriter()
572 with self.make_test_file(
573 't' * (cwriter.KEEP_BLOCK_SIZE + 10)) as testfile:
574 testpath = os.path.realpath(testfile.name)
575 cwriter.write_file(testpath, 'test')
576 for state in cwriter.saved_states:
577 if state.get('_current_file') == (testpath,
578 cwriter.KEEP_BLOCK_SIZE):
581 self.fail("can't find state immediately after PUT to Keep")
582 self.assertIn('d45107e93f9052fa88a82fc08bb1d316+1024', # 't' * 1024
583 state['_current_stream_locators'])
585 def test_basic_resume(self):
586 cwriter = TestResumableWriter()
587 with self.make_test_file() as testfile:
588 cwriter.write_file(testfile.name, 'test')
589 last_state = cwriter.last_state()
590 resumed = TestResumableWriter.from_state(last_state)
591 self.assertEquals(cwriter.manifest_text(), resumed.manifest_text(),
592 "resumed CollectionWriter had different manifest")
594 def test_resume_fails_when_missing_dependency(self):
595 cwriter = TestResumableWriter()
596 with self.make_test_file() as testfile:
597 cwriter.write_file(testfile.name, 'test')
598 self.assertRaises(arvados.errors.StaleWriterStateError,
599 TestResumableWriter.from_state,
600 cwriter.last_state())
602 def test_resume_fails_when_dependency_mtime_changed(self):
603 cwriter = TestResumableWriter()
604 with self.make_test_file() as testfile:
605 cwriter.write_file(testfile.name, 'test')
606 os.utime(testfile.name, (0, 0))
607 self.assertRaises(arvados.errors.StaleWriterStateError,
608 TestResumableWriter.from_state,
609 cwriter.last_state())
611 def test_resume_fails_when_dependency_is_nonfile(self):
612 cwriter = TestResumableWriter()
613 cwriter.write_file('/dev/null', 'empty')
614 self.assertRaises(arvados.errors.StaleWriterStateError,
615 TestResumableWriter.from_state,
616 cwriter.last_state())
618 def test_resume_fails_when_dependency_size_changed(self):
619 cwriter = TestResumableWriter()
620 with self.make_test_file() as testfile:
621 cwriter.write_file(testfile.name, 'test')
622 orig_mtime = os.fstat(testfile.fileno()).st_mtime
623 testfile.write('extra')
625 os.utime(testfile.name, (orig_mtime, orig_mtime))
626 self.assertRaises(arvados.errors.StaleWriterStateError,
627 TestResumableWriter.from_state,
628 cwriter.last_state())
630 def test_successful_resumes(self):
631 # FIXME: This is more of an integration test than a unit test.
632 cwriter = TestResumableWriter()
633 source_tree = self.build_directory_tree()
634 with open(os.path.join(source_tree, 'long'), 'w') as longfile:
635 longfile.write('t' * (cwriter.KEEP_BLOCK_SIZE + 10))
636 cwriter.write_directory_tree(source_tree)
637 # A state for each file, plus a fourth for mid-longfile.
638 self.assertGreater(len(cwriter.saved_states), 3,
639 "CollectionWriter didn't save enough states to test")
641 for state in cwriter.saved_states:
642 new_writer = TestResumableWriter.from_state(state)
643 manifests = [writer.manifest_text()
644 for writer in (cwriter, new_writer)]
646 manifests[0], manifests[1],
647 "\n".join(["manifest mismatch after resuming from state:",
648 pprint.pformat(state), ""] + manifests))
650 def test_arbitrary_objects_not_resumable(self):
651 cwriter = TestResumableWriter()
652 with open('/dev/null') as badfile:
653 self.assertRaises(arvados.errors.AssertionError,
654 cwriter.write_file, badfile)
656 def test_arbitrary_writes_not_resumable(self):
657 cwriter = TestResumableWriter()
658 self.assertRaises(arvados.errors.AssertionError,
659 cwriter.write, "badtext")
662 if __name__ == '__main__':