5692: Collection.manifest_text(strip=False) will flush open files and wait for

[arvados.git] / sdk / python / tests / test_arvfile.py

diff --git a/sdk/python/tests/test_arvfile.py b/sdk/python/tests/test_arvfile.py
index e63cf85b217c94261956875ebbbe5de94cd0165a..a8da5577b95c3c77decaa1e1bf6b5ded3bce8ee6 100644 (file)
--- a/sdk/python/tests/test_arvfile.py
+++ b/sdk/python/tests/test_arvfile.py
@@ -77,10 +77,10 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
             writer.seek(0, os.SEEK_SET)
             self.assertEqual("01234567", writer.read(12))
 
-            self.assertEqual(None, c._manifest_locator)
+            self.assertEqual(None, c.manifest_locator())
             self.assertEqual(True, c.modified())
             c.save_new("test_truncate")
-            self.assertEqual("zzzzz-4zz18-mockcollection0", c._manifest_locator)
+            self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
             self.assertEqual(False, c.modified())
 
     def test_write_to_end(self):
@@ -104,12 +104,12 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
             writer.seek(5, os.SEEK_SET)
             self.assertEqual("56789foo", writer.read(8))
 
-            self.assertEqual(None, c._manifest_locator)
+            self.assertEqual(None, c.manifest_locator())
             self.assertEqual(True, c.modified())
             self.assertEqual(None, keep.get("acbd18db4cc2f85cedef654fccc4a4d8+3"))
 
             c.save_new("test_append")
-            self.assertEqual("zzzzz-4zz18-mockcollection0", c._manifest_locator)
+            self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
             self.assertEqual(False, c.modified())
             self.assertEqual("foo", keep.get("acbd18db4cc2f85cedef654fccc4a4d8+3"))
 
@@ -142,6 +142,23 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
             self.assertEqual("foo3456789", writer.readfrom(0, 13))
             self.assertEqual(". acbd18db4cc2f85cedef654fccc4a4d8+3 781e5e245d69b566979b86e28d23f2c7+10 0:3:count.txt 6:7:count.txt\n", c.manifest_text())
 
+    def test_write_empty(self):
+        keep = ArvadosFileWriterTestCase.MockKeep({})
+        with Collection(keep_client=keep) as c:
+            writer = c.open("count.txt", "w")
+            self.assertEqual(writer.size(), 0)
+            self.assertEqual(". d41d8cd98f00b204e9800998ecf8427e+0 0:0:count.txt\n", c.manifest_text())
+
+    def test_get_manifest_text_commits(self):
+        keep = ArvadosFileWriterTestCase.MockKeep({})
+        with Collection(keep_client=keep) as c:
+            writer = c.open("count.txt", "w")
+            writer.write("0123456789")
+            self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.manifest_text(strip=True))
+            self.assertNotIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks)
+            self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.manifest_text(strip=False))
+            self.assertIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks)
+
     def test_write_in_middle(self):
         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
@@ -204,10 +221,10 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
                 writer.write(text)
             self.assertEqual(writer.size(), 100000000)
 
-            self.assertEqual(None, c._manifest_locator)
+            self.assertEqual(None, c.manifest_locator())
             self.assertEqual(True, c.modified())
             c.save_new("test_write_large")
-            self.assertEqual("zzzzz-4zz18-mockcollection0", c._manifest_locator)
+            self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
             self.assertEqual(False, c.modified())
 
     def test_rewrite_on_empty_file(self):
@@ -218,10 +235,14 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
             for b in xrange(0, 10):
                 writer.seek(0, os.SEEK_SET)
                 writer.write("0123456789")
-            writer.arvadosfile._repack_writes()
+
+            self.assertEqual(writer.size(), 10)
+            self.assertEqual("0123456789", writer.readfrom(0, 20))
+            self.assertEqual(". 7a08b07e84641703e5f2c836aa59a170+100 90:10:count.txt\n", c.manifest_text(strip=True))
+            writer.flush()
             self.assertEqual(writer.size(), 10)
             self.assertEqual("0123456789", writer.readfrom(0, 20))
-            self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n", c.manifest_text())
+            self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n", c.manifest_text(strip=True))
 
     def test_rewrite_append_existing_file(self):
         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
@@ -231,10 +252,15 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
             for b in xrange(0, 10):
                 writer.seek(10, os.SEEK_SET)
                 writer.write("abcdefghij")
-            writer.arvadosfile._repack_writes()
+
             self.assertEqual(writer.size(), 20)
             self.assertEqual("0123456789abcdefghij", writer.readfrom(0, 20))
-            self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:20:count.txt\n", c.manifest_text())
+            self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 ae5f43bab79cf0be33f025fa97ae7398+100 0:10:count.txt 100:10:count.txt\n", c.manifest_text(strip=True))
+
+            writer.arvadosfile.flush()
+            self.assertEqual(writer.size(), 20)
+            self.assertEqual("0123456789abcdefghij", writer.readfrom(0, 20))
+            self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:20:count.txt\n", c.manifest_text(strip=True))
 
     def test_rewrite_over_existing_file(self):
         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
@@ -244,10 +270,16 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
             for b in xrange(0, 10):
                 writer.seek(5, os.SEEK_SET)
                 writer.write("abcdefghij")
-            writer.arvadosfile._repack_writes()
+
             self.assertEqual(writer.size(), 15)
             self.assertEqual("01234abcdefghij", writer.readfrom(0, 20))
-            self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:5:count.txt 10:10:count.txt\n", c.manifest_text())
+            self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 ae5f43bab79cf0be33f025fa97ae7398+100 0:5:count.txt 100:10:count.txt\n", c.manifest_text(strip=True))
+
+            writer.arvadosfile.flush()
+
+            self.assertEqual(writer.size(), 15)
+            self.assertEqual("01234abcdefghij", writer.readfrom(0, 20))
+            self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:5:count.txt 10:10:count.txt\n", c.manifest_text(strip=True))
 
     def test_write_large_rewrite(self):
         keep = ArvadosFileWriterTestCase.MockKeep({})
@@ -265,10 +297,10 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
             writer.write("foo")
             self.assertEqual(writer.size(), 100000000)
 
-            self.assertEqual(None, c._manifest_locator)
+            self.assertEqual(None, c.manifest_locator())
             self.assertEqual(True, c.modified())
             c.save_new("test_write_large")
-            self.assertEqual("zzzzz-4zz18-mockcollection0", c._manifest_locator)
+            self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
             self.assertEqual(False, c.modified())
 
     def test_create(self):
@@ -283,11 +315,11 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
             writer.write("01234567")
             self.assertEqual(writer.size(), 8)
 
-            self.assertEqual(None, c._manifest_locator)
+            self.assertEqual(None, c.manifest_locator())
             self.assertEqual(True, c.modified())
             self.assertEqual(None, keep.get("2e9ec317e197819358fbc43afca7d837+8"))
             c.save_new("test_create")
-            self.assertEqual("zzzzz-4zz18-mockcollection0", c._manifest_locator)
+            self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
             self.assertEqual(False, c.modified())
             self.assertEqual("01234567", keep.get("2e9ec317e197819358fbc43afca7d837+8"))
 
@@ -318,10 +350,10 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
             writer.write("01234567")
             self.assertEqual(writer.size(), 8)
 
-            self.assertEqual(None, c._manifest_locator)
+            self.assertEqual(None, c.manifest_locator())
             self.assertEqual(True, c.modified())
             c.save_new("test_overwrite")
-            self.assertEqual("zzzzz-4zz18-mockcollection0", c._manifest_locator)
+            self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
             self.assertEqual(False, c.modified())
 
     def test_file_not_found(self):
@@ -348,11 +380,11 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
             self.assertEqual(w1.size(), 8)
             self.assertEqual(w2.size(), 8)
 
-            self.assertEqual(None, c._manifest_locator)
+            self.assertEqual(None, c.manifest_locator())
             self.assertEqual(True, c.modified())
             self.assertEqual(None, keep.get("2e9ec317e197819358fbc43afca7d837+8"))
             c.save_new("test_create_multiple")
-            self.assertEqual("zzzzz-4zz18-mockcollection0", c._manifest_locator)
+            self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
             self.assertEqual(False, c.modified())
             self.assertEqual("01234567", keep.get("2e9ec317e197819358fbc43afca7d837+8"))
 
@@ -383,9 +415,6 @@ class ArvadosFileReaderTestCase(StreamFileReaderTestCase):
         def _my_block_manager(self):
             return ArvadosFileReaderTestCase.MockParent.MockBlockMgr(self.blocks, self.nocache)
 
-        def sync_mode(self):
-            return SYNC_READONLY
-
 
     def make_count_reader(self, nocache=False):
         stream = []
@@ -460,8 +489,10 @@ class ArvadosFileReadTestCase(unittest.TestCase, StreamRetryTestMixin):
                 n += k.size
             except ValueError:
                 pass
-        col = Collection(keep_client=self.keep_client())
-        col._my_block_manager().prefetch_enabled = False
+
+        blockmanager = arvados.arvfile._BlockManager(self.keep_client())
+        blockmanager.prefetch_enabled = False
+        col = Collection(keep_client=self.keep_client(), block_manager=blockmanager)
         af = ArvadosFile(col,
                          stream=stream,
                          segments=segments)
@@ -548,6 +579,9 @@ class BlockManagerTest(unittest.TestCase):
         mockkeep = mock.MagicMock()
         blockmanager = arvados.arvfile._BlockManager(mockkeep)
         bufferblock = blockmanager.alloc_bufferblock()
+        bufferblock.owner = ArvadosFile(Collection(block_manager=blockmanager),
+                                        [Range(bufferblock.blockid, 0, 3)], [Range(0, 0, 3)])
+        bufferblock.owner._current_bblock = bufferblock
         bufferblock.append("foo")
         blockmanager.commit_all()
         self.assertTrue(mockkeep.put.called)
@@ -560,8 +594,11 @@ class BlockManagerTest(unittest.TestCase):
         mockkeep.put.side_effect = arvados.errors.KeepWriteError("fail")
         blockmanager = arvados.arvfile._BlockManager(mockkeep)
         bufferblock = blockmanager.alloc_bufferblock()
+        bufferblock.owner = ArvadosFile(Collection(block_manager=blockmanager),
+                                        [Range(bufferblock.blockid, 0, 3)], [Range(0, 0, 3)])
+        bufferblock.owner._current_bblock = bufferblock
         bufferblock.append("foo")
         with self.assertRaises(arvados.errors.KeepWriteError) as err:
             blockmanager.commit_all()
-        self.assertEqual(str(err.exception), "Error writing some blocks: acbd18db4cc2f85cedef654fccc4a4d8+3 raised KeepWriteError (fail)")
+        self.assertEqual(str(err.exception), "Error writing some blocks: block acbd18db4cc2f85cedef654fccc4a4d8+3 raised KeepWriteError (fail)")
         self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.PENDING)