1 from __future__ import absolute_import
3 from builtins import object
16 from . import run_test_server
17 from arvados._ranges import Range, LocatorAndRange
18 from arvados.collection import Collection, CollectionReader
19 from . import arvados_testutil as tutil
21 class TestResumableWriter(arvados.ResumableCollectionWriter):
22 KEEP_BLOCK_SIZE = 1024 # PUT to Keep every 1K.
24 def current_state(self):
25 return self.dump_state(copy.deepcopy)
28 class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
29 tutil.ArvadosBaseTestCase):
34 super(ArvadosCollectionsTest, cls).setUpClass()
35 run_test_server.authorize_with('active')
36 cls.api_client = arvados.api('v1')
37 cls.keep_client = arvados.KeepClient(api_client=cls.api_client,
38 local_store=cls.local_store)
40 def write_foo_bar_baz(self):
41 cw = arvados.CollectionWriter(self.api_client)
42 self.assertEqual(cw.current_stream_name(), '.',
43 'current_stream_name() should be "." now')
44 cw.set_current_file_name('foo.txt')
46 self.assertEqual(cw.current_file_name(), 'foo.txt',
47 'current_file_name() should be foo.txt now')
48 cw.start_new_file('bar.txt')
50 cw.start_new_stream('baz')
52 cw.set_current_file_name('baz.txt')
53 self.assertEqual(cw.manifest_text(),
54 ". 3858f62230ac3c915f300c664312c63f+6 0:3:foo.txt 3:3:bar.txt\n" +
55 "./baz 73feffa4b7f6bb68e44cf984c85f6e88+3 0:3:baz.txt\n",
56 "wrong manifest: got {}".format(cw.manifest_text()))
58 return cw.portable_data_hash()
60 def test_pdh_is_native_str(self):
61 pdh = self.write_foo_bar_baz()
62 self.assertEqual(type(''), type(pdh))
64 def test_keep_local_store(self):
65 self.assertEqual(self.keep_client.put(b'foo'), 'acbd18db4cc2f85cedef654fccc4a4d8+3', 'wrong md5 hash from Keep.put')
66 self.assertEqual(self.keep_client.get('acbd18db4cc2f85cedef654fccc4a4d8+3'), b'foo', 'wrong data from Keep.get')
68 def test_local_collection_writer(self):
69 self.assertEqual(self.write_foo_bar_baz(),
70 '23ca013983d6239e98931cc779e68426+114',
71 'wrong locator hash: ' + self.write_foo_bar_baz())
73 def test_local_collection_reader(self):
74 foobarbaz = self.write_foo_bar_baz()
75 cr = arvados.CollectionReader(
76 foobarbaz + '+Xzizzle', self.api_client)
78 for s in cr.all_streams():
79 for f in s.all_files():
80 got += [[f.size(), f.stream_name(), f.name(), f.read(2**26)]]
81 expected = [[3, '.', 'foo.txt', b'foo'],
82 [3, '.', 'bar.txt', b'bar'],
83 [3, './baz', 'baz.txt', b'baz']]
86 stream0 = cr.all_streams()[0]
87 self.assertEqual(stream0.readfrom(0, 0),
89 'reading zero bytes should have returned empty string')
90 self.assertEqual(stream0.readfrom(0, 2**26),
92 'reading entire stream failed')
93 self.assertEqual(stream0.readfrom(2**26, 0),
95 'reading zero bytes should have returned empty string')
97 def _test_subset(self, collection, expected):
98 cr = arvados.CollectionReader(collection, self.api_client)
99 for s in cr.all_streams():
103 got = [f.size(), f.stream_name(), f.name(), "".join(f.readall(2**26))]
104 self.assertEqual(got,
106 'all_files|as_manifest did not preserve manifest contents: got %s expected %s' % (got, ex))
108 def test_collection_manifest_subset(self):
109 foobarbaz = self.write_foo_bar_baz()
110 self._test_subset(foobarbaz,
111 [[3, '.', 'bar.txt', b'bar'],
112 [3, '.', 'foo.txt', b'foo'],
113 [3, './baz', 'baz.txt', b'baz']])
114 self._test_subset((". %s %s 0:3:foo.txt 3:3:bar.txt\n" %
115 (self.keep_client.put(b"foo"),
116 self.keep_client.put(b"bar"))),
117 [[3, '.', 'bar.txt', b'bar'],
118 [3, '.', 'foo.txt', b'foo']])
119 self._test_subset((". %s %s 0:2:fo.txt 2:4:obar.txt\n" %
120 (self.keep_client.put(b"foo"),
121 self.keep_client.put(b"bar"))),
122 [[2, '.', 'fo.txt', b'fo'],
123 [4, '.', 'obar.txt', b'obar']])
124 self._test_subset((". %s %s 0:2:fo.txt 2:0:zero.txt 2:2:ob.txt 4:2:ar.txt\n" %
125 (self.keep_client.put(b"foo"),
126 self.keep_client.put(b"bar"))),
127 [[2, '.', 'ar.txt', b'ar'],
128 [2, '.', 'fo.txt', b'fo'],
129 [2, '.', 'ob.txt', b'ob'],
130 [0, '.', 'zero.txt', b'']])
132 def test_collection_empty_file(self):
133 cw = arvados.CollectionWriter(self.api_client)
134 cw.start_new_file('zero.txt')
137 self.assertEqual(cw.manifest_text(), ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:zero.txt\n")
138 self.check_manifest_file_sizes(cw.manifest_text(), [0])
139 cw = arvados.CollectionWriter(self.api_client)
140 cw.start_new_file('zero.txt')
142 cw.start_new_file('one.txt')
144 cw.start_new_stream('foo')
145 cw.start_new_file('zero.txt')
147 self.check_manifest_file_sizes(cw.manifest_text(), [0,1,0])
149 def test_no_implicit_normalize(self):
150 cw = arvados.CollectionWriter(self.api_client)
151 cw.start_new_file('b')
153 cw.start_new_file('a')
155 self.check_manifest_file_sizes(cw.manifest_text(), [1,0])
156 self.check_manifest_file_sizes(
157 arvados.CollectionReader(
158 cw.manifest_text()).manifest_text(normalize=True),
161 def check_manifest_file_sizes(self, manifest_text, expect_sizes):
162 cr = arvados.CollectionReader(manifest_text, self.api_client)
164 for f in cr.all_files():
165 got_sizes += [f.size()]
166 self.assertEqual(got_sizes, expect_sizes, "got wrong file sizes %s, expected %s" % (got_sizes, expect_sizes))
168 def test_normalized_collection(self):
169 m1 = """. 5348b82a029fd9e971a811ce1f71360b+43 0:43:md5sum.txt
170 . 085c37f02916da1cad16f93c54d899b7+41 0:41:md5sum.txt
171 . 8b22da26f9f433dea0a10e5ec66d73ba+43 0:43:md5sum.txt
173 self.assertEqual(arvados.CollectionReader(m1, self.api_client).manifest_text(normalize=True),
174 """. 5348b82a029fd9e971a811ce1f71360b+43 085c37f02916da1cad16f93c54d899b7+41 8b22da26f9f433dea0a10e5ec66d73ba+43 0:127:md5sum.txt
177 m2 = """. 204e43b8a1185621ca55a94839582e6f+67108864 b9677abbac956bd3e86b1deb28dfac03+67108864 fc15aff2a762b13f521baf042140acec+67108864 323d2a3ce20370c4ca1d3462a344f8fd+25885655 0:227212247:var-GS000016015-ASM.tsv.bz2
179 self.assertEqual(arvados.CollectionReader(m2, self.api_client).manifest_text(normalize=True), m2)
181 m3 = """. 5348b82a029fd9e971a811ce1f71360b+43 3:40:md5sum.txt
182 . 085c37f02916da1cad16f93c54d899b7+41 0:41:md5sum.txt
183 . 8b22da26f9f433dea0a10e5ec66d73ba+43 0:43:md5sum.txt
185 self.assertEqual(arvados.CollectionReader(m3, self.api_client).manifest_text(normalize=True),
186 """. 5348b82a029fd9e971a811ce1f71360b+43 085c37f02916da1cad16f93c54d899b7+41 8b22da26f9f433dea0a10e5ec66d73ba+43 3:124:md5sum.txt
189 m4 = """. 204e43b8a1185621ca55a94839582e6f+67108864 0:3:foo/bar
190 ./zzz 204e43b8a1185621ca55a94839582e6f+67108864 0:999:zzz
191 ./foo 323d2a3ce20370c4ca1d3462a344f8fd+25885655 0:3:bar
193 self.assertEqual(arvados.CollectionReader(m4, self.api_client).manifest_text(normalize=True),
194 """./foo 204e43b8a1185621ca55a94839582e6f+67108864 323d2a3ce20370c4ca1d3462a344f8fd+25885655 0:3:bar 67108864:3:bar
195 ./zzz 204e43b8a1185621ca55a94839582e6f+67108864 0:999:zzz
198 m5 = """. 204e43b8a1185621ca55a94839582e6f+67108864 0:3:foo/bar
199 ./zzz 204e43b8a1185621ca55a94839582e6f+67108864 0:999:zzz
200 ./foo 204e43b8a1185621ca55a94839582e6f+67108864 3:3:bar
202 self.assertEqual(arvados.CollectionReader(m5, self.api_client).manifest_text(normalize=True),
203 """./foo 204e43b8a1185621ca55a94839582e6f+67108864 0:6:bar
204 ./zzz 204e43b8a1185621ca55a94839582e6f+67108864 0:999:zzz
207 with self.data_file('1000G_ref_manifest') as f6:
209 self.assertEqual(arvados.CollectionReader(m6, self.api_client).manifest_text(normalize=True), m6)
211 with self.data_file('jlake_manifest') as f7:
213 self.assertEqual(arvados.CollectionReader(m7, self.api_client).manifest_text(normalize=True), m7)
215 m8 = """./a\\040b\\040c 59ca0efa9f5633cb0371bbc0355478d8+13 0:13:hello\\040world.txt
217 self.assertEqual(arvados.CollectionReader(m8, self.api_client).manifest_text(normalize=True), m8)
219 def test_locators_and_ranges(self):
220 blocks2 = [Range('a', 0, 10),
227 self.assertEqual(arvados.locators_and_ranges(blocks2, 2, 2), [LocatorAndRange('a', 10, 2, 2)])
228 self.assertEqual(arvados.locators_and_ranges(blocks2, 12, 2), [LocatorAndRange('b', 10, 2, 2)])
229 self.assertEqual(arvados.locators_and_ranges(blocks2, 22, 2), [LocatorAndRange('c', 10, 2, 2)])
230 self.assertEqual(arvados.locators_and_ranges(blocks2, 32, 2), [LocatorAndRange('d', 10, 2, 2)])
231 self.assertEqual(arvados.locators_and_ranges(blocks2, 42, 2), [LocatorAndRange('e', 10, 2, 2)])
232 self.assertEqual(arvados.locators_and_ranges(blocks2, 52, 2), [LocatorAndRange('f', 10, 2, 2)])
233 self.assertEqual(arvados.locators_and_ranges(blocks2, 62, 2), [])
234 self.assertEqual(arvados.locators_and_ranges(blocks2, -2, 2), [])
236 self.assertEqual(arvados.locators_and_ranges(blocks2, 0, 2), [LocatorAndRange('a', 10, 0, 2)])
237 self.assertEqual(arvados.locators_and_ranges(blocks2, 10, 2), [LocatorAndRange('b', 10, 0, 2)])
238 self.assertEqual(arvados.locators_and_ranges(blocks2, 20, 2), [LocatorAndRange('c', 10, 0, 2)])
239 self.assertEqual(arvados.locators_and_ranges(blocks2, 30, 2), [LocatorAndRange('d', 10, 0, 2)])
240 self.assertEqual(arvados.locators_and_ranges(blocks2, 40, 2), [LocatorAndRange('e', 10, 0, 2)])
241 self.assertEqual(arvados.locators_and_ranges(blocks2, 50, 2), [LocatorAndRange('f', 10, 0, 2)])
242 self.assertEqual(arvados.locators_and_ranges(blocks2, 60, 2), [])
243 self.assertEqual(arvados.locators_and_ranges(blocks2, -2, 2), [])
245 self.assertEqual(arvados.locators_and_ranges(blocks2, 9, 2), [LocatorAndRange('a', 10, 9, 1), LocatorAndRange('b', 10, 0, 1)])
246 self.assertEqual(arvados.locators_and_ranges(blocks2, 19, 2), [LocatorAndRange('b', 10, 9, 1), LocatorAndRange('c', 10, 0, 1)])
247 self.assertEqual(arvados.locators_and_ranges(blocks2, 29, 2), [LocatorAndRange('c', 10, 9, 1), LocatorAndRange('d', 10, 0, 1)])
248 self.assertEqual(arvados.locators_and_ranges(blocks2, 39, 2), [LocatorAndRange('d', 10, 9, 1), LocatorAndRange('e', 10, 0, 1)])
249 self.assertEqual(arvados.locators_and_ranges(blocks2, 49, 2), [LocatorAndRange('e', 10, 9, 1), LocatorAndRange('f', 10, 0, 1)])
250 self.assertEqual(arvados.locators_and_ranges(blocks2, 59, 2), [LocatorAndRange('f', 10, 9, 1)])
253 blocks3 = [Range('a', 0, 10),
261 self.assertEqual(arvados.locators_and_ranges(blocks3, 2, 2), [LocatorAndRange('a', 10, 2, 2)])
262 self.assertEqual(arvados.locators_and_ranges(blocks3, 12, 2), [LocatorAndRange('b', 10, 2, 2)])
263 self.assertEqual(arvados.locators_and_ranges(blocks3, 22, 2), [LocatorAndRange('c', 10, 2, 2)])
264 self.assertEqual(arvados.locators_and_ranges(blocks3, 32, 2), [LocatorAndRange('d', 10, 2, 2)])
265 self.assertEqual(arvados.locators_and_ranges(blocks3, 42, 2), [LocatorAndRange('e', 10, 2, 2)])
266 self.assertEqual(arvados.locators_and_ranges(blocks3, 52, 2), [LocatorAndRange('f', 10, 2, 2)])
267 self.assertEqual(arvados.locators_and_ranges(blocks3, 62, 2), [LocatorAndRange('g', 10, 2, 2)])
270 blocks = [Range('a', 0, 10),
273 self.assertEqual(arvados.locators_and_ranges(blocks, 1, 0), [])
274 self.assertEqual(arvados.locators_and_ranges(blocks, 0, 5), [LocatorAndRange('a', 10, 0, 5)])
275 self.assertEqual(arvados.locators_and_ranges(blocks, 3, 5), [LocatorAndRange('a', 10, 3, 5)])
276 self.assertEqual(arvados.locators_and_ranges(blocks, 0, 10), [LocatorAndRange('a', 10, 0, 10)])
278 self.assertEqual(arvados.locators_and_ranges(blocks, 0, 11), [LocatorAndRange('a', 10, 0, 10),
279 LocatorAndRange('b', 15, 0, 1)])
280 self.assertEqual(arvados.locators_and_ranges(blocks, 1, 11), [LocatorAndRange('a', 10, 1, 9),
281 LocatorAndRange('b', 15, 0, 2)])
282 self.assertEqual(arvados.locators_and_ranges(blocks, 0, 25), [LocatorAndRange('a', 10, 0, 10),
283 LocatorAndRange('b', 15, 0, 15)])
285 self.assertEqual(arvados.locators_and_ranges(blocks, 0, 30), [LocatorAndRange('a', 10, 0, 10),
286 LocatorAndRange('b', 15, 0, 15),
287 LocatorAndRange('c', 5, 0, 5)])
288 self.assertEqual(arvados.locators_and_ranges(blocks, 1, 30), [LocatorAndRange('a', 10, 1, 9),
289 LocatorAndRange('b', 15, 0, 15),
290 LocatorAndRange('c', 5, 0, 5)])
291 self.assertEqual(arvados.locators_and_ranges(blocks, 0, 31), [LocatorAndRange('a', 10, 0, 10),
292 LocatorAndRange('b', 15, 0, 15),
293 LocatorAndRange('c', 5, 0, 5)])
295 self.assertEqual(arvados.locators_and_ranges(blocks, 15, 5), [LocatorAndRange('b', 15, 5, 5)])
297 self.assertEqual(arvados.locators_and_ranges(blocks, 8, 17), [LocatorAndRange('a', 10, 8, 2),
298 LocatorAndRange('b', 15, 0, 15)])
300 self.assertEqual(arvados.locators_and_ranges(blocks, 8, 20), [LocatorAndRange('a', 10, 8, 2),
301 LocatorAndRange('b', 15, 0, 15),
302 LocatorAndRange('c', 5, 0, 3)])
304 self.assertEqual(arvados.locators_and_ranges(blocks, 26, 2), [LocatorAndRange('c', 5, 1, 2)])
306 self.assertEqual(arvados.locators_and_ranges(blocks, 9, 15), [LocatorAndRange('a', 10, 9, 1),
307 LocatorAndRange('b', 15, 0, 14)])
308 self.assertEqual(arvados.locators_and_ranges(blocks, 10, 15), [LocatorAndRange('b', 15, 0, 15)])
309 self.assertEqual(arvados.locators_and_ranges(blocks, 11, 15), [LocatorAndRange('b', 15, 1, 14),
310 LocatorAndRange('c', 5, 0, 1)])
312 class MockKeep(object):
313 def __init__(self, content, num_retries=0):
314 self.content = content
316 def get(self, locator, num_retries=0):
317 return self.content[locator]
319 def test_stream_reader(self):
321 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa+10': b'abcdefghij',
322 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb+15': b'klmnopqrstuvwxy',
323 'cccccccccccccccccccccccccccccccc+5': b'z0123',
325 mk = self.MockKeep(keepblocks)
327 sr = arvados.StreamReader([".", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa+10", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb+15", "cccccccccccccccccccccccccccccccc+5", "0:30:foo"], mk)
329 content = b'abcdefghijklmnopqrstuvwxyz0123456789'
331 self.assertEqual(sr.readfrom(0, 30), content[0:30])
332 self.assertEqual(sr.readfrom(2, 30), content[2:30])
334 self.assertEqual(sr.readfrom(2, 8), content[2:10])
335 self.assertEqual(sr.readfrom(0, 10), content[0:10])
337 self.assertEqual(sr.readfrom(0, 5), content[0:5])
338 self.assertEqual(sr.readfrom(5, 5), content[5:10])
339 self.assertEqual(sr.readfrom(10, 5), content[10:15])
340 self.assertEqual(sr.readfrom(15, 5), content[15:20])
341 self.assertEqual(sr.readfrom(20, 5), content[20:25])
342 self.assertEqual(sr.readfrom(25, 5), content[25:30])
343 self.assertEqual(sr.readfrom(30, 5), b'')
345 def test_extract_file(self):
346 m1 = """. 5348b82a029fd9e971a811ce1f71360b+43 0:43:md5sum.txt
347 . 085c37f02916da1cad16f93c54d899b7+41 0:41:md6sum.txt
348 . 8b22da26f9f433dea0a10e5ec66d73ba+43 0:43:md7sum.txt
349 . 085c37f02916da1cad16f93c54d899b7+41 5348b82a029fd9e971a811ce1f71360b+43 8b22da26f9f433dea0a10e5ec66d73ba+43 47:80:md8sum.txt
350 . 085c37f02916da1cad16f93c54d899b7+41 5348b82a029fd9e971a811ce1f71360b+43 8b22da26f9f433dea0a10e5ec66d73ba+43 40:80:md9sum.txt
353 m2 = arvados.CollectionReader(m1, self.api_client).manifest_text(normalize=True)
356 ". 5348b82a029fd9e971a811ce1f71360b+43 085c37f02916da1cad16f93c54d899b7+41 8b22da26f9f433dea0a10e5ec66d73ba+43 0:43:md5sum.txt 43:41:md6sum.txt 84:43:md7sum.txt 6:37:md8sum.txt 84:43:md8sum.txt 83:1:md9sum.txt 0:43:md9sum.txt 84:36:md9sum.txt\n")
357 files = arvados.CollectionReader(
358 m2, self.api_client).all_streams()[0].files()
360 self.assertEqual(files['md5sum.txt'].as_manifest(),
361 ". 5348b82a029fd9e971a811ce1f71360b+43 0:43:md5sum.txt\n")
362 self.assertEqual(files['md6sum.txt'].as_manifest(),
363 ". 085c37f02916da1cad16f93c54d899b7+41 0:41:md6sum.txt\n")
364 self.assertEqual(files['md7sum.txt'].as_manifest(),
365 ". 8b22da26f9f433dea0a10e5ec66d73ba+43 0:43:md7sum.txt\n")
366 self.assertEqual(files['md9sum.txt'].as_manifest(),
367 ". 085c37f02916da1cad16f93c54d899b7+41 5348b82a029fd9e971a811ce1f71360b+43 8b22da26f9f433dea0a10e5ec66d73ba+43 40:80:md9sum.txt\n")
369 def test_write_directory_tree(self):
370 cwriter = arvados.CollectionWriter(self.api_client)
371 cwriter.write_directory_tree(self.build_directory_tree(
372 ['basefile', 'subdir/subfile']))
373 self.assertEqual(cwriter.manifest_text(),
374 """. c5110c5ac93202d8e0f9e381f22bac0f+8 0:8:basefile
375 ./subdir 1ca4dec89403084bf282ad31e6cf7972+14 0:14:subfile\n""")
377 def test_write_named_directory_tree(self):
378 cwriter = arvados.CollectionWriter(self.api_client)
379 cwriter.write_directory_tree(self.build_directory_tree(
380 ['basefile', 'subdir/subfile']), 'root')
382 cwriter.manifest_text(),
383 """./root c5110c5ac93202d8e0f9e381f22bac0f+8 0:8:basefile
384 ./root/subdir 1ca4dec89403084bf282ad31e6cf7972+14 0:14:subfile\n""")
386 def test_write_directory_tree_in_one_stream(self):
387 cwriter = arvados.CollectionWriter(self.api_client)
388 cwriter.write_directory_tree(self.build_directory_tree(
389 ['basefile', 'subdir/subfile']), max_manifest_depth=0)
390 self.assertEqual(cwriter.manifest_text(),
391 """. 4ace875ffdc6824a04950f06858f4465+22 0:8:basefile 8:14:subdir/subfile\n""")
393 def test_write_directory_tree_with_limited_recursion(self):
394 cwriter = arvados.CollectionWriter(self.api_client)
395 cwriter.write_directory_tree(
396 self.build_directory_tree(['f1', 'd1/f2', 'd1/d2/f3']),
397 max_manifest_depth=1)
398 self.assertEqual(cwriter.manifest_text(),
399 """. bd19836ddb62c11c55ab251ccaca5645+2 0:2:f1
400 ./d1 50170217e5b04312024aa5cd42934494+13 0:8:d2/f3 8:5:f2\n""")
402 def test_write_directory_tree_with_zero_recursion(self):
403 cwriter = arvados.CollectionWriter(self.api_client)
404 content = 'd1/d2/f3d1/f2f1'
405 blockhash = tutil.str_keep_locator(content)
406 cwriter.write_directory_tree(
407 self.build_directory_tree(['f1', 'd1/f2', 'd1/d2/f3']),
408 max_manifest_depth=0)
410 cwriter.manifest_text(),
411 ". {} 0:8:d1/d2/f3 8:5:d1/f2 13:2:f1\n".format(blockhash))
413 def test_write_one_file(self):
414 cwriter = arvados.CollectionWriter(self.api_client)
415 with self.make_test_file() as testfile:
416 cwriter.write_file(testfile.name)
418 cwriter.manifest_text(),
419 ". 098f6bcd4621d373cade4e832627b4f6+4 0:4:{}\n".format(
420 os.path.basename(testfile.name)))
422 def test_write_named_file(self):
423 cwriter = arvados.CollectionWriter(self.api_client)
424 with self.make_test_file() as testfile:
425 cwriter.write_file(testfile.name, 'foo')
426 self.assertEqual(cwriter.manifest_text(),
427 ". 098f6bcd4621d373cade4e832627b4f6+4 0:4:foo\n")
429 def test_write_multiple_files(self):
430 cwriter = arvados.CollectionWriter(self.api_client)
432 with self.make_test_file(letter.encode()) as testfile:
433 cwriter.write_file(testfile.name, letter)
435 cwriter.manifest_text(),
436 ". 902fbdd2b1df0c4f70b4a5d23525e932+3 0:1:A 1:1:B 2:1:C\n")
438 def test_basic_resume(self):
439 cwriter = TestResumableWriter()
440 with self.make_test_file() as testfile:
441 cwriter.write_file(testfile.name, 'test')
442 resumed = TestResumableWriter.from_state(cwriter.current_state())
443 self.assertEqual(cwriter.manifest_text(), resumed.manifest_text(),
444 "resumed CollectionWriter had different manifest")
446 def test_resume_fails_when_missing_dependency(self):
447 cwriter = TestResumableWriter()
448 with self.make_test_file() as testfile:
449 cwriter.write_file(testfile.name, 'test')
450 self.assertRaises(arvados.errors.StaleWriterStateError,
451 TestResumableWriter.from_state,
452 cwriter.current_state())
454 def test_resume_fails_when_dependency_mtime_changed(self):
455 cwriter = TestResumableWriter()
456 with self.make_test_file() as testfile:
457 cwriter.write_file(testfile.name, 'test')
458 os.utime(testfile.name, (0, 0))
459 self.assertRaises(arvados.errors.StaleWriterStateError,
460 TestResumableWriter.from_state,
461 cwriter.current_state())
463 def test_resume_fails_when_dependency_is_nonfile(self):
464 cwriter = TestResumableWriter()
465 cwriter.write_file('/dev/null', 'empty')
466 self.assertRaises(arvados.errors.StaleWriterStateError,
467 TestResumableWriter.from_state,
468 cwriter.current_state())
470 def test_resume_fails_when_dependency_size_changed(self):
471 cwriter = TestResumableWriter()
472 with self.make_test_file() as testfile:
473 cwriter.write_file(testfile.name, 'test')
474 orig_mtime = os.fstat(testfile.fileno()).st_mtime
475 testfile.write(b'extra')
477 os.utime(testfile.name, (orig_mtime, orig_mtime))
478 self.assertRaises(arvados.errors.StaleWriterStateError,
479 TestResumableWriter.from_state,
480 cwriter.current_state())
482 def test_resume_fails_with_expired_locator(self):
483 cwriter = TestResumableWriter()
484 state = cwriter.current_state()
485 # Add an expired locator to the state.
486 state['_current_stream_locators'].append(''.join([
487 'a' * 32, '+1+A', 'b' * 40, '@', '10000000']))
488 self.assertRaises(arvados.errors.StaleWriterStateError,
489 TestResumableWriter.from_state, state)
491 def test_arbitrary_objects_not_resumable(self):
492 cwriter = TestResumableWriter()
493 with open('/dev/null') as badfile:
494 self.assertRaises(arvados.errors.AssertionError,
495 cwriter.write_file, badfile)
497 def test_arbitrary_writes_not_resumable(self):
498 cwriter = TestResumableWriter()
499 self.assertRaises(arvados.errors.AssertionError,
500 cwriter.write, "badtext")
502 def test_read_arbitrary_data_with_collection_reader(self):
503 # arv-get relies on this to do "arv-get {keep-locator} -".
504 self.write_foo_bar_baz()
507 arvados.CollectionReader(
508 '3858f62230ac3c915f300c664312c63f+6'
512 class CollectionTestMixin(tutil.ApiClientMock):
513 API_COLLECTIONS = run_test_server.fixture('collections')
514 DEFAULT_COLLECTION = API_COLLECTIONS['foo_file']
515 DEFAULT_DATA_HASH = DEFAULT_COLLECTION['portable_data_hash']
516 DEFAULT_MANIFEST = DEFAULT_COLLECTION['manifest_text']
517 DEFAULT_UUID = DEFAULT_COLLECTION['uuid']
518 ALT_COLLECTION = API_COLLECTIONS['bar_file']
519 ALT_DATA_HASH = ALT_COLLECTION['portable_data_hash']
520 ALT_MANIFEST = ALT_COLLECTION['manifest_text']
522 def api_client_mock(self, status=200):
523 client = super(CollectionTestMixin, self).api_client_mock()
524 self.mock_keep_services(client, status=status, service_type='proxy', count=1)
529 class CollectionReaderTestCase(unittest.TestCase, CollectionTestMixin):
530 def mock_get_collection(self, api_mock, code, fixturename):
531 body = self.API_COLLECTIONS.get(fixturename)
532 self._mock_api_call(api_mock.collections().get, code, body)
534 def api_client_mock(self, status=200):
535 client = super(CollectionReaderTestCase, self).api_client_mock()
536 self.mock_get_collection(client, status, 'foo_file')
539 def test_init_no_default_retries(self):
540 client = self.api_client_mock(200)
541 reader = arvados.CollectionReader(self.DEFAULT_UUID, api_client=client)
542 reader.manifest_text()
543 client.collections().get().execute.assert_called_with(num_retries=0)
545 def test_uuid_init_success(self):
546 client = self.api_client_mock(200)
547 reader = arvados.CollectionReader(self.DEFAULT_UUID, api_client=client,
549 self.assertEqual(self.DEFAULT_COLLECTION['manifest_text'],
550 reader.manifest_text())
551 client.collections().get().execute.assert_called_with(num_retries=3)
553 def test_uuid_init_failure_raises_api_error(self):
554 client = self.api_client_mock(500)
555 with self.assertRaises(arvados.errors.ApiError):
556 reader = arvados.CollectionReader(self.DEFAULT_UUID, api_client=client)
558 def test_locator_init(self):
559 client = self.api_client_mock(200)
560 # Ensure Keep will not return anything if asked.
561 with tutil.mock_keep_responses(None, 404):
562 reader = arvados.CollectionReader(self.DEFAULT_DATA_HASH,
564 self.assertEqual(self.DEFAULT_MANIFEST, reader.manifest_text())
566 def test_locator_init_fallback_to_keep(self):
567 # crunch-job needs this to read manifests that have only ever
568 # been written to Keep.
569 client = self.api_client_mock(200)
570 self.mock_get_collection(client, 404, None)
571 with tutil.mock_keep_responses(self.DEFAULT_MANIFEST, 200):
572 reader = arvados.CollectionReader(self.DEFAULT_DATA_HASH,
574 self.assertEqual(self.DEFAULT_MANIFEST, reader.manifest_text())
576 def test_uuid_init_no_fallback_to_keep(self):
577 # Do not look up a collection UUID in Keep.
578 client = self.api_client_mock(404)
579 with tutil.mock_keep_responses(self.DEFAULT_MANIFEST, 200):
580 with self.assertRaises(arvados.errors.ApiError):
581 reader = arvados.CollectionReader(self.DEFAULT_UUID,
584 def test_try_keep_first_if_permission_hint(self):
585 # To verify that CollectionReader tries Keep first here, we
586 # mock API server to return the wrong data.
587 client = self.api_client_mock(200)
588 with tutil.mock_keep_responses(self.ALT_MANIFEST, 200):
591 arvados.CollectionReader(
592 self.ALT_DATA_HASH + '+Affffffffffffffffffffffffffffffffffffffff@fedcba98',
593 api_client=client).manifest_text())
595 def test_init_num_retries_propagated(self):
596 # More of an integration test...
597 client = self.api_client_mock(200)
598 reader = arvados.CollectionReader(self.DEFAULT_UUID, api_client=client,
600 with tutil.mock_keep_responses('foo', 500, 500, 200):
601 self.assertEqual(b'foo',
602 b''.join(f.read(9) for f in reader.all_files()))
604 def test_read_nonnormalized_manifest_with_collection_reader(self):
605 # client should be able to use CollectionReader on a manifest without normalizing it
606 client = self.api_client_mock(500)
607 nonnormal = ". acbd18db4cc2f85cedef654fccc4a4d8+3+Aabadbadbee@abeebdee 0:3:foo.txt 1:0:bar.txt 0:3:foo.txt\n"
608 reader = arvados.CollectionReader(
610 api_client=client, num_retries=0)
611 # Ensure stripped_manifest() doesn't mangle our manifest in
612 # any way other than stripping hints.
614 re.sub('\+[^\d\s\+]+', '', nonnormal),
615 reader.stripped_manifest())
616 # Ensure stripped_manifest() didn't mutate our reader.
617 self.assertEqual(nonnormal, reader.manifest_text())
618 # Ensure the files appear in the order given in the manifest.
620 [[6, '.', 'foo.txt'],
621 [0, '.', 'bar.txt']],
622 [[f.size(), f.stream_name(), f.name()]
623 for f in reader.all_streams()[0].all_files()])
625 def test_read_empty_collection(self):
626 client = self.api_client_mock(200)
627 self.mock_get_collection(client, 200, 'empty')
628 reader = arvados.CollectionReader('d41d8cd98f00b204e9800998ecf8427e+0',
630 self.assertEqual('', reader.manifest_text())
632 def test_api_response(self):
633 client = self.api_client_mock()
634 reader = arvados.CollectionReader(self.DEFAULT_UUID, api_client=client)
635 self.assertEqual(self.DEFAULT_COLLECTION, reader.api_response())
637 def test_api_response_with_collection_from_keep(self):
638 client = self.api_client_mock()
639 self.mock_get_collection(client, 404, 'foo')
640 with tutil.mock_keep_responses(self.DEFAULT_MANIFEST, 200):
641 reader = arvados.CollectionReader(self.DEFAULT_DATA_HASH,
643 api_response = reader.api_response()
644 self.assertIsNone(api_response)
646 def check_open_file(self, coll_file, stream_name, file_name, file_size):
647 self.assertFalse(coll_file.closed, "returned file is not open")
648 self.assertEqual(stream_name, coll_file.stream_name())
649 self.assertEqual(file_name, coll_file.name)
650 self.assertEqual(file_size, coll_file.size())
652 def test_open_collection_file_one_argument(self):
653 client = self.api_client_mock(200)
654 reader = arvados.CollectionReader(self.DEFAULT_UUID, api_client=client)
655 cfile = reader.open('./foo', 'rb')
656 self.check_open_file(cfile, '.', 'foo', 3)
658 def test_open_deep_file(self):
659 coll_name = 'collection_with_files_in_subdir'
660 client = self.api_client_mock(200)
661 self.mock_get_collection(client, 200, coll_name)
662 reader = arvados.CollectionReader(
663 self.API_COLLECTIONS[coll_name]['uuid'], api_client=client)
664 cfile = reader.open('./subdir2/subdir3/file2_in_subdir3.txt', 'rb')
665 self.check_open_file(cfile, './subdir2/subdir3', 'file2_in_subdir3.txt',
668 def test_open_nonexistent_stream(self):
669 client = self.api_client_mock(200)
670 reader = arvados.CollectionReader(self.DEFAULT_UUID, api_client=client)
671 self.assertRaises(IOError, reader.open, './nonexistent/foo')
673 def test_open_nonexistent_file(self):
674 client = self.api_client_mock(200)
675 reader = arvados.CollectionReader(self.DEFAULT_UUID, api_client=client)
676 self.assertRaises(IOError, reader.open, 'nonexistent')
680 class CollectionWriterTestCase(unittest.TestCase, CollectionTestMixin):
681 def mock_keep(self, body, *codes, **headers):
682 headers.setdefault('x-keep-replicas-stored', 2)
683 return tutil.mock_keep_responses(body, *codes, **headers)
685 def foo_writer(self, **kwargs):
686 kwargs.setdefault('api_client', self.api_client_mock())
687 writer = arvados.CollectionWriter(**kwargs)
688 writer.start_new_file('foo')
692 def test_write_whole_collection(self):
693 writer = self.foo_writer()
694 with self.mock_keep(self.DEFAULT_DATA_HASH, 200, 200):
695 self.assertEqual(self.DEFAULT_DATA_HASH, writer.finish())
697 def test_write_no_default(self):
698 writer = self.foo_writer()
699 with self.mock_keep(None, 500):
700 with self.assertRaises(arvados.errors.KeepWriteError):
703 def test_write_insufficient_replicas_via_proxy(self):
704 writer = self.foo_writer(replication=3)
705 with self.mock_keep(None, 200, **{'x-keep-replicas-stored': 2}):
706 with self.assertRaises(arvados.errors.KeepWriteError):
707 writer.manifest_text()
709 def test_write_insufficient_replicas_via_disks(self):
710 client = mock.MagicMock(name='api_client')
713 **{'x-keep-replicas-stored': 1}) as keepmock:
714 self.mock_keep_services(client, status=200, service_type='disk', count=2)
715 writer = self.foo_writer(api_client=client, replication=3)
716 with self.assertRaises(arvados.errors.KeepWriteError):
717 writer.manifest_text()
719 def test_write_three_replicas(self):
720 client = mock.MagicMock(name='api_client')
722 "", 500, 500, 500, 200, 200, 200,
723 **{'x-keep-replicas-stored': 1}) as keepmock:
724 self.mock_keep_services(client, status=200, service_type='disk', count=6)
725 writer = self.foo_writer(api_client=client, replication=3)
726 writer.manifest_text()
727 self.assertEqual(6, keepmock.call_count)
729 def test_write_whole_collection_through_retries(self):
730 writer = self.foo_writer(num_retries=2)
731 with self.mock_keep(self.DEFAULT_DATA_HASH,
732 500, 500, 200, 500, 500, 200):
733 self.assertEqual(self.DEFAULT_DATA_HASH, writer.finish())
735 def test_flush_data_retries(self):
736 writer = self.foo_writer(num_retries=2)
737 foo_hash = self.DEFAULT_MANIFEST.split()[1]
738 with self.mock_keep(foo_hash, 500, 200):
740 self.assertEqual(self.DEFAULT_MANIFEST, writer.manifest_text())
742 def test_one_open(self):
743 client = self.api_client_mock()
744 writer = arvados.CollectionWriter(client)
745 with writer.open('out') as out_file:
746 self.assertEqual('.', writer.current_stream_name())
747 self.assertEqual('out', writer.current_file_name())
748 out_file.write(b'test data')
749 data_loc = tutil.str_keep_locator('test data')
750 self.assertTrue(out_file.closed, "writer file not closed after context")
751 self.assertRaises(ValueError, out_file.write, 'extra text')
752 with self.mock_keep(data_loc, 200) as keep_mock:
753 self.assertEqual(". {} 0:9:out\n".format(data_loc),
754 writer.manifest_text())
756 def test_open_writelines(self):
757 client = self.api_client_mock()
758 writer = arvados.CollectionWriter(client)
759 with writer.open('six') as out_file:
760 out_file.writelines(['12', '34', '56'])
761 data_loc = tutil.str_keep_locator('123456')
762 with self.mock_keep(data_loc, 200) as keep_mock:
763 self.assertEqual(". {} 0:6:six\n".format(data_loc),
764 writer.manifest_text())
766 def test_open_flush(self):
767 client = self.api_client_mock()
768 data_loc1 = tutil.str_keep_locator('flush1')
769 data_loc2 = tutil.str_keep_locator('flush2')
770 with self.mock_keep((data_loc1, 200), (data_loc2, 200)) as keep_mock:
771 writer = arvados.CollectionWriter(client)
772 with writer.open('flush_test') as out_file:
773 out_file.write(b'flush1')
775 out_file.write(b'flush2')
776 self.assertEqual(". {} {} 0:12:flush_test\n".format(data_loc1,
778 writer.manifest_text())
780 def test_two_opens_same_stream(self):
781 client = self.api_client_mock()
782 writer = arvados.CollectionWriter(client)
783 with writer.open('.', '1') as out_file:
784 out_file.write(b'1st')
785 with writer.open('.', '2') as out_file:
786 out_file.write(b'2nd')
787 data_loc = tutil.str_keep_locator('1st2nd')
788 with self.mock_keep(data_loc, 200) as keep_mock:
789 self.assertEqual(". {} 0:3:1 3:3:2\n".format(data_loc),
790 writer.manifest_text())
792 def test_two_opens_two_streams(self):
793 client = self.api_client_mock()
794 data_loc1 = tutil.str_keep_locator('file')
795 data_loc2 = tutil.str_keep_locator('indir')
796 with self.mock_keep((data_loc1, 200), (data_loc2, 200)) as keep_mock:
797 writer = arvados.CollectionWriter(client)
798 with writer.open('file') as out_file:
799 out_file.write(b'file')
800 with writer.open('./dir', 'indir') as out_file:
801 out_file.write(b'indir')
802 expected = ". {} 0:4:file\n./dir {} 0:5:indir\n".format(
803 data_loc1, data_loc2)
804 self.assertEqual(expected, writer.manifest_text())
806 def test_dup_open_fails(self):
807 client = self.api_client_mock()
808 writer = arvados.CollectionWriter(client)
809 file1 = writer.open('one')
810 self.assertRaises(arvados.errors.AssertionError, writer.open, 'two')
813 class CollectionMethods(run_test_server.TestCaseWithServers):
815 def test_keys_values_items_support_indexing(self):
817 with c.open('foo', 'wb') as f:
819 with c.open('bar', 'wb') as f:
821 self.assertEqual(2, len(c.keys()))
822 if sys.version_info < (3, 0):
823 # keys() supports indexing only for python2 callers.
828 self.assertEqual(2, len(c.values()))
831 self.assertEqual(2, len(c.items()))
832 self.assertEqual(fn0, c.items()[0][0])
833 self.assertEqual(fn1, c.items()[1][0])
836 class CollectionOpenModes(run_test_server.TestCaseWithServers):
838 def test_open_binary_modes(self):
840 for mode in ['wb', 'wb+', 'ab', 'ab+']:
841 with c.open('foo', mode) as f:
844 def test_open_invalid_modes(self):
846 for mode in ['+r', 'aa', '++', 'r+b', 'beer', '', None]:
847 with self.assertRaises(Exception):
850 def test_open_text_modes(self):
852 with c.open('foo', 'wb') as f:
854 for mode in ['r', 'rt', 'r+', 'rt+', 'w', 'wt', 'a', 'at']:
855 if sys.version_info >= (3, 0):
856 with self.assertRaises(NotImplementedError):
859 with c.open('foo', mode) as f:
860 if mode[0] == 'r' and '+' not in mode:
861 self.assertEqual('foo', f.read(3))
864 f.seek(-3, os.SEEK_CUR)
865 self.assertEqual('bar', f.read(3))
868 class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
870 def test_replication_desired_kept_on_load(self):
871 m = '. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n'
872 c1 = Collection(m, replication_desired=1)
874 loc = c1.manifest_locator()
876 self.assertEqual(c1.manifest_text, c2.manifest_text)
877 self.assertEqual(c1.replication_desired, c2.replication_desired)
879 def test_replication_desired_not_loaded_if_provided(self):
880 m = '. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n'
881 c1 = Collection(m, replication_desired=1)
883 loc = c1.manifest_locator()
884 c2 = Collection(loc, replication_desired=2)
885 self.assertEqual(c1.manifest_text, c2.manifest_text)
886 self.assertNotEqual(c1.replication_desired, c2.replication_desired)
888 def test_init_manifest(self):
889 m1 = """. 5348b82a029fd9e971a811ce1f71360b+43 0:43:md5sum.txt
890 . 085c37f02916da1cad16f93c54d899b7+41 0:41:md5sum.txt
891 . 8b22da26f9f433dea0a10e5ec66d73ba+43 0:43:md5sum.txt
893 self.assertEqual(m1, CollectionReader(m1).manifest_text(normalize=False))
894 self.assertEqual(". 5348b82a029fd9e971a811ce1f71360b+43 085c37f02916da1cad16f93c54d899b7+41 8b22da26f9f433dea0a10e5ec66d73ba+43 0:127:md5sum.txt\n", CollectionReader(m1).manifest_text(normalize=True))
896 def test_init_manifest_with_collision(self):
897 m1 = """. 5348b82a029fd9e971a811ce1f71360b+43 0:43:md5sum.txt
898 ./md5sum.txt 085c37f02916da1cad16f93c54d899b7+41 0:41:md5sum.txt
900 with self.assertRaises(arvados.errors.ArgumentError):
901 self.assertEqual(m1, CollectionReader(m1))
903 def test_init_manifest_with_error(self):
904 m1 = """. 0:43:md5sum.txt"""
905 with self.assertRaises(arvados.errors.ArgumentError):
906 self.assertEqual(m1, CollectionReader(m1))
908 def test_remove(self):
909 c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n')
910 self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n", c.portable_manifest_text())
911 self.assertIn("count1.txt", c)
912 c.remove("count1.txt")
913 self.assertNotIn("count1.txt", c)
914 self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n", c.portable_manifest_text())
915 with self.assertRaises(arvados.errors.ArgumentError):
919 c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n')
920 self.assertIs(c.find("."), c)
921 self.assertIs(c.find("./count1.txt"), c["count1.txt"])
922 self.assertIs(c.find("count1.txt"), c["count1.txt"])
923 with self.assertRaises(IOError):
925 with self.assertRaises(arvados.errors.ArgumentError):
927 self.assertIs(c.find("./nonexistant.txt"), None)
928 self.assertIs(c.find("./nonexistantsubdir/nonexistant.txt"), None)
930 def test_remove_in_subdir(self):
931 c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
932 c.remove("foo/count2.txt")
933 self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", c.portable_manifest_text())
935 def test_remove_empty_subdir(self):
936 c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
937 c.remove("foo/count2.txt")
939 self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", c.portable_manifest_text())
941 def test_remove_nonempty_subdir(self):
942 c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
943 with self.assertRaises(IOError):
945 c.remove("foo", recursive=True)
946 self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", c.portable_manifest_text())
948 def test_copy_to_file_in_dir(self):
949 c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
950 c.copy("count1.txt", "foo/count2.txt")
951 self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n", c.portable_manifest_text())
953 def test_copy_file(self):
954 c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
955 c.copy("count1.txt", "count2.txt")
956 self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n", c.portable_manifest_text())
958 def test_copy_to_existing_dir(self):
959 c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
960 c.copy("count1.txt", "foo")
961 self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n", c.portable_manifest_text())
963 def test_copy_to_new_dir(self):
964 c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
965 c.copy("count1.txt", "foo/")
966 self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", c.portable_manifest_text())
968 def test_rename_file(self):
969 c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
970 c.rename("count1.txt", "count2.txt")
971 self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n", c.manifest_text())
973 def test_move_file_to_dir(self):
974 c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
976 c.rename("count1.txt", "foo/count2.txt")
977 self.assertEqual("./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n", c.manifest_text())
979 def test_move_file_to_other(self):
980 c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
982 c2.rename("count1.txt", "count2.txt", source_collection=c1)
983 self.assertEqual("", c1.manifest_text())
984 self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n", c2.manifest_text())
986 def test_clone(self):
987 c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
989 self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n", cl.portable_manifest_text())
991 def test_diff_del_add(self):
992 c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
993 c2 = Collection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
995 self.assertEqual(sorted(d), [
996 ('add', './count1.txt', c1["count1.txt"]),
997 ('del', './count2.txt', c2["count2.txt"]),
1000 self.assertEqual(sorted(d), [
1001 ('add', './count2.txt', c2["count2.txt"]),
1002 ('del', './count1.txt', c1["count1.txt"]),
1004 self.assertNotEqual(c1.portable_manifest_text(), c2.portable_manifest_text())
1006 self.assertEqual(c1.portable_manifest_text(), c2.portable_manifest_text())
1008 def test_diff_same(self):
1009 c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
1010 c2 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
1012 self.assertEqual(d, [('tok', './count1.txt', c2["count1.txt"], c1["count1.txt"])])
1014 self.assertEqual(d, [('tok', './count1.txt', c2["count1.txt"], c1["count1.txt"])])
1016 self.assertEqual(c1.portable_manifest_text(), c2.portable_manifest_text())
1018 self.assertEqual(c1.portable_manifest_text(), c2.portable_manifest_text())
1020 def test_diff_mod(self):
1021 c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
1022 c2 = Collection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt\n')
1024 self.assertEqual(d, [('mod', './count1.txt', c2["count1.txt"], c1["count1.txt"])])
1026 self.assertEqual(d, [('mod', './count1.txt', c1["count1.txt"], c2["count1.txt"])])
1028 self.assertNotEqual(c1.portable_manifest_text(), c2.portable_manifest_text())
1030 self.assertEqual(c1.portable_manifest_text(), c2.portable_manifest_text())
1032 def test_diff_add(self):
1033 c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
1034 c2 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt 10:20:count2.txt\n')
1036 self.assertEqual(sorted(d), [
1037 ('del', './count2.txt', c2["count2.txt"]),
1038 ('tok', './count1.txt', c2["count1.txt"], c1["count1.txt"]),
1041 self.assertEqual(sorted(d), [
1042 ('add', './count2.txt', c2["count2.txt"]),
1043 ('tok', './count1.txt', c2["count1.txt"], c1["count1.txt"]),
1046 self.assertNotEqual(c1.portable_manifest_text(), c2.portable_manifest_text())
1048 self.assertEqual(c1.portable_manifest_text(), c2.portable_manifest_text())
1050 def test_diff_add_in_subcollection(self):
1051 c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
1052 c2 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
1054 self.assertEqual(sorted(d), [
1055 ('del', './foo', c2["foo"]),
1056 ('tok', './count1.txt', c2["count1.txt"], c1["count1.txt"]),
1059 self.assertEqual(sorted(d), [
1060 ('add', './foo', c2["foo"]),
1061 ('tok', './count1.txt', c2["count1.txt"], c1["count1.txt"]),
1063 self.assertNotEqual(c1.portable_manifest_text(), c2.portable_manifest_text())
1065 self.assertEqual(c1.portable_manifest_text(), c2.portable_manifest_text())
1067 def test_diff_del_add_in_subcollection(self):
1068 c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
1069 c2 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:3:count3.txt\n')
1071 self.assertEqual(sorted(d), [
1072 ('add', './foo/count2.txt', c1.find("foo/count2.txt")),
1073 ('del', './foo/count3.txt', c2.find("foo/count3.txt")),
1074 ('tok', './count1.txt', c2["count1.txt"], c1["count1.txt"]),
1077 self.assertEqual(sorted(d), [
1078 ('add', './foo/count3.txt', c2.find("foo/count3.txt")),
1079 ('del', './foo/count2.txt', c1.find("foo/count2.txt")),
1080 ('tok', './count1.txt', c2["count1.txt"], c1["count1.txt"]),
1083 self.assertNotEqual(c1.portable_manifest_text(), c2.portable_manifest_text())
1085 self.assertEqual(c1.portable_manifest_text(), c2.portable_manifest_text())
1087 def test_diff_mod_in_subcollection(self):
1088 c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
1089 c2 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:3:foo\n')
1091 self.assertEqual(sorted(d), [
1092 ('mod', './foo', c2["foo"], c1["foo"]),
1093 ('tok', './count1.txt', c2["count1.txt"], c1["count1.txt"]),
1096 self.assertEqual(sorted(d), [
1097 ('mod', './foo', c1["foo"], c2["foo"]),
1098 ('tok', './count1.txt', c2["count1.txt"], c1["count1.txt"]),
1101 self.assertNotEqual(c1.portable_manifest_text(), c2.portable_manifest_text())
1103 self.assertEqual(c1.portable_manifest_text(), c2.portable_manifest_text())
1105 def test_conflict_keep_local_change(self):
1106 c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
1107 c2 = Collection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
1109 self.assertEqual(sorted(d), [
1110 ('add', './count2.txt', c2["count2.txt"]),
1111 ('del', './count1.txt', c1["count1.txt"]),
1113 f = c1.open("count1.txt", "wb")
1116 # c1 changed, so it should not be deleted.
1118 self.assertEqual(c1.portable_manifest_text(), ". 95ebc3c7b3b9f1d2c40fec14415d3cb8+5 5348b82a029fd9e971a811ce1f71360b+43 0:5:count1.txt 5:10:count2.txt\n")
1120 def test_conflict_mod(self):
1121 c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt')
1122 c2 = Collection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt')
1124 self.assertEqual(d, [('mod', './count1.txt', c1["count1.txt"], c2["count1.txt"])])
1125 f = c1.open("count1.txt", "wb")
1128 # c1 changed, so c2 mod will go to a conflict file
1131 c1.portable_manifest_text(),
1132 r"\. 95ebc3c7b3b9f1d2c40fec14415d3cb8\+5 5348b82a029fd9e971a811ce1f71360b\+43 0:5:count1\.txt 5:10:count1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~$")
1134 def test_conflict_add(self):
1135 c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
1136 c2 = Collection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt\n')
1138 self.assertEqual(sorted(d), [
1139 ('add', './count1.txt', c2["count1.txt"]),
1140 ('del', './count2.txt', c1["count2.txt"]),
1142 f = c1.open("count1.txt", "wb")
1145 # c1 added count1.txt, so c2 add will go to a conflict file
1148 c1.portable_manifest_text(),
1149 r"\. 95ebc3c7b3b9f1d2c40fec14415d3cb8\+5 5348b82a029fd9e971a811ce1f71360b\+43 0:5:count1\.txt 5:10:count1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~$")
1151 def test_conflict_del(self):
1152 c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt')
1153 c2 = Collection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt')
1155 self.assertEqual(d, [('mod', './count1.txt', c1["count1.txt"], c2["count1.txt"])])
1156 c1.remove("count1.txt")
1158 # c1 deleted, so c2 mod will go to a conflict file
1161 c1.portable_manifest_text(),
1162 r"\. 5348b82a029fd9e971a811ce1f71360b\+43 0:10:count1\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~$")
1164 def test_notify(self):
1167 c1.subscribe(lambda event, collection, name, item: events.append((event, collection, name, item)))
1168 f = c1.open("foo.txt", "wb")
1169 self.assertEqual(events[0], (arvados.collection.ADD, c1, "foo.txt", f.arvadosfile))
1171 def test_open_w(self):
1172 c1 = Collection(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n")
1173 self.assertEqual(c1["count1.txt"].size(), 10)
1174 c1.open("count1.txt", "wb").close()
1175 self.assertEqual(c1["count1.txt"].size(), 0)
1178 class NewCollectionTestCaseWithServersAndTokens(run_test_server.TestCaseWithServers):
1183 self.keep_put = getattr(arvados.keep.KeepClient, 'put')
1185 def test_repacked_block_submission_get_permission_token(self):
1187 Make sure that those blocks that are committed after repacking small ones,
1188 get their permission tokens assigned on the collection manifest.
1190 def wrapped_keep_put(*args, **kwargs):
1191 # Simulate slow put operations
1193 return self.keep_put(*args, **kwargs)
1195 re_locator = "[0-9a-f]{32}\+\d+\+A[a-f0-9]{40}@[a-f0-9]{8}"
1197 with mock.patch('arvados.keep.KeepClient.put', autospec=True) as mocked_put:
1198 mocked_put.side_effect = wrapped_keep_put
1200 # Write 70 files ~1MiB each so we force to produce 1 big block by repacking
1201 # small ones before finishing the upload.
1203 f = c.open("file_{}.txt".format(i), 'wb')
1204 f.write(random.choice('abcdefghijklmnopqrstuvwxyz') * (2**20+i))
1205 f.close(flush=False)
1206 # We should get 2 blocks with their tokens
1207 self.assertEqual(len(re.findall(re_locator, c.manifest_text())), 2)
1210 class NewCollectionTestCaseWithServers(run_test_server.TestCaseWithServers):
1211 def test_get_manifest_text_only_committed(self):
1213 with c.open("count.txt", "wb") as f:
1214 # One file committed
1215 with c.open("foo.txt", "wb") as foo:
1217 foo.flush() # Force block commit
1218 f.write(b"0123456789")
1219 # Other file not committed. Block not written to keep yet.
1221 c._get_manifest_text(".",
1224 only_committed=True),
1225 '. acbd18db4cc2f85cedef654fccc4a4d8+3 0:0:count.txt 0:3:foo.txt\n')
1226 # And now with the file closed...
1227 f.flush() # Force block commit
1229 c._get_manifest_text(".",
1232 only_committed=True),
1233 ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:10:count.txt 10:3:foo.txt\n")
1235 def test_only_small_blocks_are_packed_together(self):
1237 # Write a couple of small files,
1238 f = c.open("count.txt", "wb")
1239 f.write(b"0123456789")
1240 f.close(flush=False)
1241 foo = c.open("foo.txt", "wb")
1243 foo.close(flush=False)
1244 # Then, write a big file, it shouldn't be packed with the ones above
1245 big = c.open("bigfile.txt", "wb")
1246 big.write(b"x" * 1024 * 1024 * 33) # 33 MB > KEEP_BLOCK_SIZE/2
1247 big.close(flush=False)
1249 c.manifest_text("."),
1250 '. 2d303c138c118af809f39319e5d507e9+34603008 a8430a058b8fbf408e1931b794dbd6fb+13 0:34603008:bigfile.txt 34603008:10:count.txt 34603018:3:foo.txt\n')
1252 def test_flush_after_small_block_packing(self):
1254 # Write a couple of small files,
1255 f = c.open("count.txt", "wb")
1256 f.write(b"0123456789")
1257 f.close(flush=False)
1258 foo = c.open("foo.txt", "wb")
1260 foo.close(flush=False)
1264 '. a8430a058b8fbf408e1931b794dbd6fb+13 0:10:count.txt 10:3:foo.txt\n')
1266 f = c.open("count.txt", "rb+")
1271 '. a8430a058b8fbf408e1931b794dbd6fb+13 0:10:count.txt 10:3:foo.txt\n')
1273 def test_write_after_small_block_packing2(self):
1275 # Write a couple of small files,
1276 f = c.open("count.txt", "wb")
1277 f.write(b"0123456789")
1278 f.close(flush=False)
1279 foo = c.open("foo.txt", "wb")
1281 foo.close(flush=False)
1285 '. a8430a058b8fbf408e1931b794dbd6fb+13 0:10:count.txt 10:3:foo.txt\n')
1287 f = c.open("count.txt", "rb+")
1289 f.close(flush=False)
1293 '. 900150983cd24fb0d6963f7d28e17f72+3 a8430a058b8fbf408e1931b794dbd6fb+13 0:3:count.txt 6:7:count.txt 13:3:foo.txt\n')
1296 def test_small_block_packing_with_overwrite(self):
1298 c.open("b1", "wb").close()
1299 c["b1"].writeto(0, b"b1", 0)
1301 c.open("b2", "wb").close()
1302 c["b2"].writeto(0, b"b2", 0)
1304 c["b1"].writeto(0, b"1b", 0)
1306 self.assertEquals(c.manifest_text(), ". ed4f3f67c70b02b29c50ce1ea26666bd+4 0:2:b1 2:2:b2\n")
1307 self.assertEquals(c["b1"].manifest_text(), ". ed4f3f67c70b02b29c50ce1ea26666bd+4 0:2:b1\n")
1308 self.assertEquals(c["b2"].manifest_text(), ". ed4f3f67c70b02b29c50ce1ea26666bd+4 2:2:b2\n")
1311 class CollectionCreateUpdateTest(run_test_server.TestCaseWithServers):
1315 def create_count_txt(self):
1316 # Create an empty collection, save it to the API server, then write a
1317 # file, but don't save it.
1320 c.save_new("CollectionCreateUpdateTest", ensure_unique_name=True)
1321 self.assertEqual(c.portable_data_hash(), "d41d8cd98f00b204e9800998ecf8427e+0")
1322 self.assertEqual(c.api_response()["portable_data_hash"], "d41d8cd98f00b204e9800998ecf8427e+0" )
1324 with c.open("count.txt", "wb") as f:
1325 f.write(b"0123456789")
1327 self.assertEqual(c.portable_manifest_text(), ". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n")
1331 def test_create_and_save(self):
1332 c = self.create_count_txt()
1336 r"^\. 781e5e245d69b566979b86e28d23f2c7\+10\+A[a-f0-9]{40}@[a-f0-9]{8} 0:10:count\.txt$",)
1338 def test_create_and_save_new(self):
1339 c = self.create_count_txt()
1343 r"^\. 781e5e245d69b566979b86e28d23f2c7\+10\+A[a-f0-9]{40}@[a-f0-9]{8} 0:10:count\.txt$",)
1345 def test_create_diff_apply(self):
1346 c1 = self.create_count_txt()
1349 c2 = Collection(c1.manifest_locator())
1350 with c2.open("count.txt", "wb") as f:
1355 self.assertEqual(diff[0], (arvados.collection.MOD, u'./count.txt', c1["count.txt"], c2["count.txt"]))
1358 self.assertEqual(c1.portable_data_hash(), c2.portable_data_hash())
1360 def test_diff_apply_with_token(self):
1361 baseline = CollectionReader(". 781e5e245d69b566979b86e28d23f2c7+10+A715fd31f8111894f717eb1003c1b0216799dd9ec@54f5dd1a 0:10:count.txt\n")
1362 c = Collection(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n")
1363 other = CollectionReader(". 7ac66c0f148de9519b8bd264312c4d64+7+A715fd31f8111894f717eb1003c1b0216799dd9ec@54f5dd1a 0:7:count.txt\n")
1365 diff = baseline.diff(other)
1366 self.assertEqual(diff, [('mod', u'./count.txt', c["count.txt"], other["count.txt"])])
1370 self.assertEqual(c.manifest_text(), ". 7ac66c0f148de9519b8bd264312c4d64+7+A715fd31f8111894f717eb1003c1b0216799dd9ec@54f5dd1a 0:7:count.txt\n")
1373 def test_create_and_update(self):
1374 c1 = self.create_count_txt()
1377 c2 = arvados.collection.Collection(c1.manifest_locator())
1378 with c2.open("count.txt", "wb") as f:
1383 self.assertNotEqual(c1.portable_data_hash(), c2.portable_data_hash())
1385 self.assertEqual(c1.portable_data_hash(), c2.portable_data_hash())
1388 def test_create_and_update_with_conflict(self):
1389 c1 = self.create_count_txt()
1392 with c1.open("count.txt", "wb") as f:
1395 c2 = arvados.collection.Collection(c1.manifest_locator())
1396 with c2.open("count.txt", "wb") as f:
1404 r"\. e65075d550f9b5bf9992fa1d71a131be\+3\S* 7ac66c0f148de9519b8bd264312c4d64\+7\S* 0:3:count\.txt 3:7:count\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~$")
1406 def test_pdh_is_native_str(self):
1407 c1 = self.create_count_txt()
1408 pdh = c1.portable_data_hash()
1409 self.assertEqual(type(''), type(pdh))
1412 if __name__ == '__main__':