X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/31e10ca041ac1cfd296b24d4878264f6af50d64b..ebb2559b3a09636ff687316bbe512e0e8a86b168:/sdk/python/tests/test_collections.py

diff --git a/sdk/python/tests/test_collections.py b/sdk/python/tests/test_collections.py
index bc3c0bf1af..0e3d5e13f1 100644
--- a/sdk/python/tests/test_collections.py
+++ b/sdk/python/tests/test_collections.py
@@ -804,6 +804,24 @@ class CollectionWriterTestCase(unittest.TestCase, CollectionTestMixin):
 
 class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
 
+    def test_replication_desired_kept_on_load(self):
+        m = '. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n'
+        c1 = Collection(m, replication_desired=1)
+        c1.save_new()
+        loc = c1.manifest_locator()
+        c2 = Collection(loc)
+        self.assertEqual(c1.manifest_text, c2.manifest_text)
+        self.assertEqual(c1.replication_desired, c2.replication_desired)
+
+    def test_replication_desired_not_loaded_if_provided(self):
+        m = '. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n'
+        c1 = Collection(m, replication_desired=1)
+        c1.save_new()
+        loc = c1.manifest_locator()
+        c2 = Collection(loc, replication_desired=2)
+        self.assertEqual(c1.manifest_text, c2.manifest_text)
+        self.assertNotEqual(c1.replication_desired, c2.replication_desired)
+
     def test_init_manifest(self):
         m1 = """. 5348b82a029fd9e971a811ce1f71360b+43 0:43:md5sum.txt
 . 085c37f02916da1cad16f93c54d899b7+41 0:41:md5sum.txt
@@ -843,6 +861,8 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
             c.find("/.")
         with self.assertRaises(arvados.errors.ArgumentError):
             c.find("")
+        self.assertIs(c.find("./nonexistant.txt"), None)
+        self.assertIs(c.find("./nonexistantsubdir/nonexistant.txt"), None)
 
     def test_remove_in_subdir(self):
         c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
@@ -922,9 +942,9 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
         c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
         c2 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
         d = c2.diff(c1)
-        self.assertEqual(d, [])
+        self.assertEqual(d, [('tok', './count1.txt', c2["count1.txt"], c1["count1.txt"])])
         d = c1.diff(c2)
-        self.assertEqual(d, [])
+        self.assertEqual(d, [('tok', './count1.txt', c2["count1.txt"], c1["count1.txt"])])
 
         self.assertEqual(c1.portable_manifest_text(), c2.portable_manifest_text())
         c1.apply(d)
@@ -946,9 +966,11 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
         c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
         c2 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt 10:20:count2.txt\n')
         d = c2.diff(c1)
-        self.assertEqual(d, [('del', './count2.txt', c2["count2.txt"])])
+        self.assertEqual(d, [('del', './count2.txt', c2["count2.txt"]),
+                             ('tok', './count1.txt', c2["count1.txt"], c1["count1.txt"])])
         d = c1.diff(c2)
-        self.assertEqual(d, [('add', './count2.txt', c2["count2.txt"])])
+        self.assertEqual(d, [('add', './count2.txt', c2["count2.txt"]),
+                             ('tok', './count1.txt', c2["count1.txt"], c1["count1.txt"])])
 
         self.assertNotEqual(c1.portable_manifest_text(), c2.portable_manifest_text())
         c1.apply(d)
@@ -958,9 +980,11 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
         c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
         c2 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
         d = c2.diff(c1)
-        self.assertEqual(d, [('del', './foo', c2["foo"])])
+        self.assertEqual(d, [('del', './foo', c2["foo"]),
+                             ('tok', './count1.txt', c2["count1.txt"], c1["count1.txt"])])
         d = c1.diff(c2)
-        self.assertEqual(d, [('add', './foo', c2["foo"])])
+        self.assertEqual(d, [('add', './foo', c2["foo"]),
+                             ('tok', './count1.txt', c2["count1.txt"], c1["count1.txt"])])
 
         self.assertNotEqual(c1.portable_manifest_text(), c2.portable_manifest_text())
         c1.apply(d)
@@ -972,10 +996,12 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
 
         d = c2.diff(c1)
         self.assertEqual(d, [('del', './foo/count3.txt', c2.find("foo/count3.txt")),
-                             ('add', './foo/count2.txt', c1.find("foo/count2.txt"))])
+                             ('add', './foo/count2.txt', c1.find("foo/count2.txt")),
+                             ('tok', './count1.txt', c2["count1.txt"], c1["count1.txt"])])
         d = c1.diff(c2)
         self.assertEqual(d, [('del', './foo/count2.txt', c1.find("foo/count2.txt")),
-                             ('add', './foo/count3.txt', c2.find("foo/count3.txt"))])
+                             ('add', './foo/count3.txt', c2.find("foo/count3.txt")),
+                             ('tok', './count1.txt', c2["count1.txt"], c1["count1.txt"])])
 
         self.assertNotEqual(c1.portable_manifest_text(), c2.portable_manifest_text())
         c1.apply(d)
@@ -985,9 +1011,11 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
         c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
         c2 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:3:foo\n')
         d = c2.diff(c1)
-        self.assertEqual(d, [('mod', './foo', c2["foo"], c1["foo"])])
+        self.assertEqual(d, [('mod', './foo', c2["foo"], c1["foo"]),
+                             ('tok', './count1.txt', c2["count1.txt"], c1["count1.txt"])])
         d = c1.diff(c2)
-        self.assertEqual(d, [('mod', './foo', c1["foo"], c2["foo"])])
+        self.assertEqual(d, [('mod', './foo', c1["foo"], c2["foo"]),
+                             ('tok', './count1.txt', c2["count1.txt"], c1["count1.txt"])])
 
         self.assertNotEqual(c1.portable_manifest_text(), c2.portable_manifest_text())
         c1.apply(d)
@@ -1056,6 +1084,49 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
         self.assertEqual(c1["count1.txt"].size(), 0)
 
 
+class NewCollectionTestCaseWithServers(run_test_server.TestCaseWithServers):
+    def test_get_manifest_text_only_committed(self):
+        c = Collection()
+        with c.open("count.txt", "w") as f:
+            # One file committed
+            with c.open("foo.txt", "w") as foo:
+                foo.write("foo")
+                foo.flush() # Force block commit
+            f.write("0123456789")
+            # Other file not committed. Block not written to keep yet.
+            self.assertEqual(
+                c._get_manifest_text(".",
+                                     strip=False,
+                                     normalize=False,
+                                     only_committed=True),
+                '. acbd18db4cc2f85cedef654fccc4a4d8+3 0:0:count.txt 0:3:foo.txt\n')
+            # And now with the file closed...
+            f.flush() # Force block commit
+        self.assertEqual(
+            c._get_manifest_text(".",
+                                 strip=False,
+                                 normalize=False,
+                                 only_committed=True),
+            ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:10:count.txt 10:3:foo.txt\n")
+
+    def test_only_small_blocks_are_packed_together(self):
+        c = Collection()
+        # Write a couple of small files, 
+        f = c.open("count.txt", "w")
+        f.write("0123456789")
+        f.close(flush=False)
+        foo = c.open("foo.txt", "w")
+        foo.write("foo")
+        foo.close(flush=False)
+        # Then, write a big file, it shouldn't be packed with the ones above
+        big = c.open("bigfile.txt", "w")
+        big.write("x" * 1024 * 1024 * 33) # 33 MB > KEEP_BLOCK_SIZE/2
+        big.close(flush=False)
+        self.assertEqual(
+            c.manifest_text("."),
+            '. 2d303c138c118af809f39319e5d507e9+34603008 a8430a058b8fbf408e1931b794dbd6fb+13 0:34603008:bigfile.txt 34603008:10:count.txt 34603018:3:foo.txt\n')
+
+
 class CollectionCreateUpdateTest(run_test_server.TestCaseWithServers):
     MAIN_SERVER = {}
     KEEP_SERVER = {}