X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/de7c71aac6a8e93f84d515e42859cce674eab009..8eb65b21a7b7eea1f987d7fec9cb8ac65de2aeb2:/sdk/python/tests/test_arvfile.py diff --git a/sdk/python/tests/test_arvfile.py b/sdk/python/tests/test_arvfile.py index a8da5577b9..1701aa43ca 100644 --- a/sdk/python/tests/test_arvfile.py +++ b/sdk/python/tests/test_arvfile.py @@ -128,7 +128,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase): writer.write("world") self.assertEqual(writer.read(20), "0123456789helloworld") - self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 fc5e038d38a57032085441e7fe7010b0+10 0:20:count.txt\n", c.manifest_text()) + self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 fc5e038d38a57032085441e7fe7010b0+10 0:20:count.txt\n", c.portable_manifest_text()) def test_write_at_beginning(self): keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"}) @@ -140,25 +140,37 @@ class ArvadosFileWriterTestCase(unittest.TestCase): writer.write("foo") self.assertEqual(writer.size(), 10) self.assertEqual("foo3456789", writer.readfrom(0, 13)) - self.assertEqual(". acbd18db4cc2f85cedef654fccc4a4d8+3 781e5e245d69b566979b86e28d23f2c7+10 0:3:count.txt 6:7:count.txt\n", c.manifest_text()) + self.assertEqual(". acbd18db4cc2f85cedef654fccc4a4d8+3 781e5e245d69b566979b86e28d23f2c7+10 0:3:count.txt 6:7:count.txt\n", c.portable_manifest_text()) def test_write_empty(self): keep = ArvadosFileWriterTestCase.MockKeep({}) with Collection(keep_client=keep) as c: writer = c.open("count.txt", "w") self.assertEqual(writer.size(), 0) - self.assertEqual(". d41d8cd98f00b204e9800998ecf8427e+0 0:0:count.txt\n", c.manifest_text()) + self.assertEqual(". d41d8cd98f00b204e9800998ecf8427e+0 0:0:count.txt\n", c.portable_manifest_text()) - def test_get_manifest_text_commits(self): + def test_save_manifest_text(self): keep = ArvadosFileWriterTestCase.MockKeep({}) with Collection(keep_client=keep) as c: writer = c.open("count.txt", "w") writer.write("0123456789") - self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.manifest_text(strip=True)) + self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.portable_manifest_text()) self.assertNotIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks) - self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.manifest_text(strip=False)) + + self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.save_new(create_collection_record=False)) self.assertIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks) + def test_get_manifest_text_commits(self): + keep = ArvadosFileWriterTestCase.MockKeep({}) + with Collection(keep_client=keep) as c: + writer = c.open("count.txt", "w") + writer.write("0123456789") + self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.portable_manifest_text()) + self.assertNotIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks) + self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.manifest_text()) + self.assertIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks) + + def test_write_in_middle(self): keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"}) with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', @@ -169,7 +181,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase): writer.write("foo") self.assertEqual(writer.size(), 10) self.assertEqual("012foo6789", writer.readfrom(0, 13)) - self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:count.txt 10:3:count.txt 6:4:count.txt\n", c.manifest_text()) + self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:count.txt 10:3:count.txt 6:4:count.txt\n", c.portable_manifest_text()) def test_write_at_end(self): keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"}) @@ -181,7 +193,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase): writer.write("foo") self.assertEqual(writer.size(), 10) self.assertEqual("0123456foo", writer.readfrom(0, 13)) - self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:7:count.txt 10:3:count.txt\n", c.manifest_text()) + self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:7:count.txt 10:3:count.txt\n", c.portable_manifest_text()) def test_write_across_segment_boundary(self): keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"}) @@ -193,7 +205,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase): writer.write("foobar") self.assertEqual(writer.size(), 20) self.assertEqual("0123456foobar34", writer.readfrom(0, 15)) - self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 3858f62230ac3c915f300c664312c63f+6 0:7:count.txt 10:6:count.txt 3:7:count.txt\n", c.manifest_text()) + self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 3858f62230ac3c915f300c664312c63f+6 0:7:count.txt 10:6:count.txt 3:7:count.txt\n", c.portable_manifest_text()) def test_write_across_several_segments(self): keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"}) @@ -205,7 +217,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase): writer.write("abcdefg") self.assertEqual(writer.size(), 12) self.assertEqual("01abcdefg123", writer.readfrom(0, 15)) - self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 7ac66c0f148de9519b8bd264312c4d64+7 0:2:count.txt 10:7:count.txt 1:3:count.txt\n", c.manifest_text()) + self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 7ac66c0f148de9519b8bd264312c4d64+7 0:2:count.txt 10:7:count.txt 1:3:count.txt\n", c.portable_manifest_text()) def test_write_large(self): keep = ArvadosFileWriterTestCase.MockKeep({}) @@ -238,11 +250,11 @@ class ArvadosFileWriterTestCase(unittest.TestCase): self.assertEqual(writer.size(), 10) self.assertEqual("0123456789", writer.readfrom(0, 20)) - self.assertEqual(". 7a08b07e84641703e5f2c836aa59a170+100 90:10:count.txt\n", c.manifest_text(strip=True)) + self.assertEqual(". 7a08b07e84641703e5f2c836aa59a170+100 90:10:count.txt\n", c.portable_manifest_text()) writer.flush() self.assertEqual(writer.size(), 10) self.assertEqual("0123456789", writer.readfrom(0, 20)) - self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n", c.manifest_text(strip=True)) + self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n", c.portable_manifest_text()) def test_rewrite_append_existing_file(self): keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"}) @@ -255,12 +267,12 @@ class ArvadosFileWriterTestCase(unittest.TestCase): self.assertEqual(writer.size(), 20) self.assertEqual("0123456789abcdefghij", writer.readfrom(0, 20)) - self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 ae5f43bab79cf0be33f025fa97ae7398+100 0:10:count.txt 100:10:count.txt\n", c.manifest_text(strip=True)) + self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 ae5f43bab79cf0be33f025fa97ae7398+100 0:10:count.txt 100:10:count.txt\n", c.portable_manifest_text()) writer.arvadosfile.flush() self.assertEqual(writer.size(), 20) self.assertEqual("0123456789abcdefghij", writer.readfrom(0, 20)) - self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:20:count.txt\n", c.manifest_text(strip=True)) + self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:20:count.txt\n", c.portable_manifest_text()) def test_rewrite_over_existing_file(self): keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"}) @@ -273,13 +285,13 @@ class ArvadosFileWriterTestCase(unittest.TestCase): self.assertEqual(writer.size(), 15) self.assertEqual("01234abcdefghij", writer.readfrom(0, 20)) - self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 ae5f43bab79cf0be33f025fa97ae7398+100 0:5:count.txt 100:10:count.txt\n", c.manifest_text(strip=True)) + self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 ae5f43bab79cf0be33f025fa97ae7398+100 0:5:count.txt 100:10:count.txt\n", c.portable_manifest_text()) writer.arvadosfile.flush() self.assertEqual(writer.size(), 15) self.assertEqual("01234abcdefghij", writer.readfrom(0, 20)) - self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:5:count.txt 10:10:count.txt\n", c.manifest_text(strip=True)) + self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:5:count.txt 10:10:count.txt\n", c.portable_manifest_text()) def test_write_large_rewrite(self): keep = ArvadosFileWriterTestCase.MockKeep({}) @@ -428,20 +440,25 @@ class ArvadosFileReaderTestCase(StreamFileReaderTestCase): af = ArvadosFile(ArvadosFileReaderTestCase.MockParent(blocks, nocache), stream=stream, segments=[Range(1, 0, 3), Range(6, 3, 3), Range(11, 6, 3)]) return ArvadosFileReader(af, "count.txt") - def test_read_returns_first_block(self): - # read() calls will be aligned on block boundaries - see #3663. + def test_read_block_crossing_behavior(self): + # read() needs to return all the data requested if possible, even if it + # crosses uncached blocks: https://arvados.org/issues/5856 sfile = self.make_count_reader(nocache=True) - self.assertEqual('123', sfile.read(10)) + self.assertEqual('12345678', sfile.read(8)) def test_successive_reads(self): + # Override StreamFileReaderTestCase.test_successive_reads sfile = self.make_count_reader(nocache=True) - for expect in ['123', '456', '789', '']: - self.assertEqual(expect, sfile.read(10)) + self.assertEqual('1234', sfile.read(4)) + self.assertEqual('5678', sfile.read(4)) + self.assertEqual('9', sfile.read(4)) + self.assertEqual('', sfile.read(4)) def test_tell_after_block_read(self): + # Override StreamFileReaderTestCase.test_tell_after_block_read sfile = self.make_count_reader(nocache=True) - sfile.read(5) - self.assertEqual(3, sfile.tell()) + self.assertEqual('12345678', sfile.read(8)) + self.assertEqual(8, sfile.tell()) def test_prefetch(self): keep = ArvadosFileWriterTestCase.MockKeep({"2e9ec317e197819358fbc43afca7d837+8": "01234567", "e8dc4081b13434b45189a720b77b6818+8": "abcdefgh"}) @@ -579,11 +596,11 @@ class BlockManagerTest(unittest.TestCase): mockkeep = mock.MagicMock() blockmanager = arvados.arvfile._BlockManager(mockkeep) bufferblock = blockmanager.alloc_bufferblock() - bufferblock.owner = ArvadosFile(Collection(block_manager=blockmanager), - [Range(bufferblock.blockid, 0, 3)], [Range(0, 0, 3)]) - bufferblock.owner._current_bblock = bufferblock + bufferblock.owner = mock.MagicMock() + bufferblock.owner.flush.side_effect = lambda: blockmanager.commit_bufferblock(bufferblock) bufferblock.append("foo") blockmanager.commit_all() + self.assertTrue(bufferblock.owner.flush.called) self.assertTrue(mockkeep.put.called) self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.COMMITTED) self.assertIsNone(bufferblock.buffer_view) @@ -594,11 +611,11 @@ class BlockManagerTest(unittest.TestCase): mockkeep.put.side_effect = arvados.errors.KeepWriteError("fail") blockmanager = arvados.arvfile._BlockManager(mockkeep) bufferblock = blockmanager.alloc_bufferblock() - bufferblock.owner = ArvadosFile(Collection(block_manager=blockmanager), - [Range(bufferblock.blockid, 0, 3)], [Range(0, 0, 3)]) - bufferblock.owner._current_bblock = bufferblock + bufferblock.owner = mock.MagicMock() + bufferblock.owner.flush.side_effect = lambda: blockmanager.commit_bufferblock(bufferblock) bufferblock.append("foo") with self.assertRaises(arvados.errors.KeepWriteError) as err: blockmanager.commit_all() + self.assertTrue(bufferblock.owner.flush.called) self.assertEqual(str(err.exception), "Error writing some blocks: block acbd18db4cc2f85cedef654fccc4a4d8+3 raised KeepWriteError (fail)") self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.PENDING)