X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/d1327f9456b561c059c18c76f63391728ce1780d..ebb2559b3a09636ff687316bbe512e0e8a86b168:/sdk/python/tests/test_collections.py diff --git a/sdk/python/tests/test_collections.py b/sdk/python/tests/test_collections.py index ac7dd1b9f6..0e3d5e13f1 100644 --- a/sdk/python/tests/test_collections.py +++ b/sdk/python/tests/test_collections.py @@ -4,7 +4,6 @@ import arvados import copy -import hashlib import mock import os import pprint @@ -395,7 +394,7 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers, def test_write_directory_tree_with_zero_recursion(self): cwriter = arvados.CollectionWriter(self.api_client) content = 'd1/d2/f3d1/f2f1' - blockhash = hashlib.md5(content).hexdigest() + '+' + str(len(content)) + blockhash = tutil.str_keep_locator(content) cwriter.write_directory_tree( self.build_directory_tree(['f1', 'd1/f2', 'd1/d2/f3']), max_manifest_depth=0) @@ -739,7 +738,7 @@ class CollectionWriterTestCase(unittest.TestCase, CollectionTestMixin): self.assertEqual('.', writer.current_stream_name()) self.assertEqual('out', writer.current_file_name()) out_file.write('test data') - data_loc = hashlib.md5('test data').hexdigest() + '+9' + data_loc = tutil.str_keep_locator('test data') self.assertTrue(out_file.closed, "writer file not closed after context") self.assertRaises(ValueError, out_file.write, 'extra text') with self.mock_keep(data_loc, 200) as keep_mock: @@ -751,15 +750,15 @@ class CollectionWriterTestCase(unittest.TestCase, CollectionTestMixin): writer = arvados.CollectionWriter(client) with writer.open('six') as out_file: out_file.writelines(['12', '34', '56']) - data_loc = hashlib.md5('123456').hexdigest() + '+6' + data_loc = tutil.str_keep_locator('123456') with self.mock_keep(data_loc, 200) as keep_mock: self.assertEqual(". {} 0:6:six\n".format(data_loc), writer.manifest_text()) def test_open_flush(self): client = self.api_client_mock() - data_loc1 = hashlib.md5('flush1').hexdigest() + '+6' - data_loc2 = hashlib.md5('flush2').hexdigest() + '+6' + data_loc1 = tutil.str_keep_locator('flush1') + data_loc2 = tutil.str_keep_locator('flush2') with self.mock_keep((data_loc1, 200), (data_loc2, 200)) as keep_mock: writer = arvados.CollectionWriter(client) with writer.open('flush_test') as out_file: @@ -777,15 +776,15 @@ class CollectionWriterTestCase(unittest.TestCase, CollectionTestMixin): out_file.write('1st') with writer.open('.', '2') as out_file: out_file.write('2nd') - data_loc = hashlib.md5('1st2nd').hexdigest() + '+6' + data_loc = tutil.str_keep_locator('1st2nd') with self.mock_keep(data_loc, 200) as keep_mock: self.assertEqual(". {} 0:3:1 3:3:2\n".format(data_loc), writer.manifest_text()) def test_two_opens_two_streams(self): client = self.api_client_mock() - data_loc1 = hashlib.md5('file').hexdigest() + '+4' - data_loc2 = hashlib.md5('indir').hexdigest() + '+5' + data_loc1 = tutil.str_keep_locator('file') + data_loc2 = tutil.str_keep_locator('indir') with self.mock_keep((data_loc1, 200), (data_loc2, 200)) as keep_mock: writer = arvados.CollectionWriter(client) with writer.open('file') as out_file: @@ -805,6 +804,24 @@ class CollectionWriterTestCase(unittest.TestCase, CollectionTestMixin): class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin): + def test_replication_desired_kept_on_load(self): + m = '. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n' + c1 = Collection(m, replication_desired=1) + c1.save_new() + loc = c1.manifest_locator() + c2 = Collection(loc) + self.assertEqual(c1.manifest_text, c2.manifest_text) + self.assertEqual(c1.replication_desired, c2.replication_desired) + + def test_replication_desired_not_loaded_if_provided(self): + m = '. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n' + c1 = Collection(m, replication_desired=1) + c1.save_new() + loc = c1.manifest_locator() + c2 = Collection(loc, replication_desired=2) + self.assertEqual(c1.manifest_text, c2.manifest_text) + self.assertNotEqual(c1.replication_desired, c2.replication_desired) + def test_init_manifest(self): m1 = """. 5348b82a029fd9e971a811ce1f71360b+43 0:43:md5sum.txt . 085c37f02916da1cad16f93c54d899b7+41 0:41:md5sum.txt @@ -844,6 +861,8 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin): c.find("/.") with self.assertRaises(arvados.errors.ArgumentError): c.find("") + self.assertIs(c.find("./nonexistant.txt"), None) + self.assertIs(c.find("./nonexistantsubdir/nonexistant.txt"), None) def test_remove_in_subdir(self): c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n') @@ -923,9 +942,9 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin): c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n') c2 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n') d = c2.diff(c1) - self.assertEqual(d, []) + self.assertEqual(d, [('tok', './count1.txt', c2["count1.txt"], c1["count1.txt"])]) d = c1.diff(c2) - self.assertEqual(d, []) + self.assertEqual(d, [('tok', './count1.txt', c2["count1.txt"], c1["count1.txt"])]) self.assertEqual(c1.portable_manifest_text(), c2.portable_manifest_text()) c1.apply(d) @@ -947,9 +966,11 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin): c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n') c2 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt 10:20:count2.txt\n') d = c2.diff(c1) - self.assertEqual(d, [('del', './count2.txt', c2["count2.txt"])]) + self.assertEqual(d, [('del', './count2.txt', c2["count2.txt"]), + ('tok', './count1.txt', c2["count1.txt"], c1["count1.txt"])]) d = c1.diff(c2) - self.assertEqual(d, [('add', './count2.txt', c2["count2.txt"])]) + self.assertEqual(d, [('add', './count2.txt', c2["count2.txt"]), + ('tok', './count1.txt', c2["count1.txt"], c1["count1.txt"])]) self.assertNotEqual(c1.portable_manifest_text(), c2.portable_manifest_text()) c1.apply(d) @@ -959,9 +980,11 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin): c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n') c2 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n') d = c2.diff(c1) - self.assertEqual(d, [('del', './foo', c2["foo"])]) + self.assertEqual(d, [('del', './foo', c2["foo"]), + ('tok', './count1.txt', c2["count1.txt"], c1["count1.txt"])]) d = c1.diff(c2) - self.assertEqual(d, [('add', './foo', c2["foo"])]) + self.assertEqual(d, [('add', './foo', c2["foo"]), + ('tok', './count1.txt', c2["count1.txt"], c1["count1.txt"])]) self.assertNotEqual(c1.portable_manifest_text(), c2.portable_manifest_text()) c1.apply(d) @@ -973,10 +996,12 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin): d = c2.diff(c1) self.assertEqual(d, [('del', './foo/count3.txt', c2.find("foo/count3.txt")), - ('add', './foo/count2.txt', c1.find("foo/count2.txt"))]) + ('add', './foo/count2.txt', c1.find("foo/count2.txt")), + ('tok', './count1.txt', c2["count1.txt"], c1["count1.txt"])]) d = c1.diff(c2) self.assertEqual(d, [('del', './foo/count2.txt', c1.find("foo/count2.txt")), - ('add', './foo/count3.txt', c2.find("foo/count3.txt"))]) + ('add', './foo/count3.txt', c2.find("foo/count3.txt")), + ('tok', './count1.txt', c2["count1.txt"], c1["count1.txt"])]) self.assertNotEqual(c1.portable_manifest_text(), c2.portable_manifest_text()) c1.apply(d) @@ -986,9 +1011,11 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin): c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n') c2 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:3:foo\n') d = c2.diff(c1) - self.assertEqual(d, [('mod', './foo', c2["foo"], c1["foo"])]) + self.assertEqual(d, [('mod', './foo', c2["foo"], c1["foo"]), + ('tok', './count1.txt', c2["count1.txt"], c1["count1.txt"])]) d = c1.diff(c2) - self.assertEqual(d, [('mod', './foo', c1["foo"], c2["foo"])]) + self.assertEqual(d, [('mod', './foo', c1["foo"], c2["foo"]), + ('tok', './count1.txt', c2["count1.txt"], c1["count1.txt"])]) self.assertNotEqual(c1.portable_manifest_text(), c2.portable_manifest_text()) c1.apply(d) @@ -1057,6 +1084,49 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin): self.assertEqual(c1["count1.txt"].size(), 0) +class NewCollectionTestCaseWithServers(run_test_server.TestCaseWithServers): + def test_get_manifest_text_only_committed(self): + c = Collection() + with c.open("count.txt", "w") as f: + # One file committed + with c.open("foo.txt", "w") as foo: + foo.write("foo") + foo.flush() # Force block commit + f.write("0123456789") + # Other file not committed. Block not written to keep yet. + self.assertEqual( + c._get_manifest_text(".", + strip=False, + normalize=False, + only_committed=True), + '. acbd18db4cc2f85cedef654fccc4a4d8+3 0:0:count.txt 0:3:foo.txt\n') + # And now with the file closed... + f.flush() # Force block commit + self.assertEqual( + c._get_manifest_text(".", + strip=False, + normalize=False, + only_committed=True), + ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:10:count.txt 10:3:foo.txt\n") + + def test_only_small_blocks_are_packed_together(self): + c = Collection() + # Write a couple of small files, + f = c.open("count.txt", "w") + f.write("0123456789") + f.close(flush=False) + foo = c.open("foo.txt", "w") + foo.write("foo") + foo.close(flush=False) + # Then, write a big file, it shouldn't be packed with the ones above + big = c.open("bigfile.txt", "w") + big.write("x" * 1024 * 1024 * 33) # 33 MB > KEEP_BLOCK_SIZE/2 + big.close(flush=False) + self.assertEqual( + c.manifest_text("."), + '. 2d303c138c118af809f39319e5d507e9+34603008 a8430a058b8fbf408e1931b794dbd6fb+13 0:34603008:bigfile.txt 34603008:10:count.txt 34603018:3:foo.txt\n') + + class CollectionCreateUpdateTest(run_test_server.TestCaseWithServers): MAIN_SERVER = {} KEEP_SERVER = {}