X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/31d31c010bb6b5170e3962fdd50c6d393cfe6076..1e3eacb0ca6f2228f50f13514c7577a149a707e6:/sdk/python/tests/test_arvfile.py diff --git a/sdk/python/tests/test_arvfile.py b/sdk/python/tests/test_arvfile.py index 6990aa68b8..086fa542a2 100644 --- a/sdk/python/tests/test_arvfile.py +++ b/sdk/python/tests/test_arvfile.py @@ -1,3 +1,7 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + from __future__ import absolute_import from builtins import hex from builtins import str @@ -79,7 +83,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase): "portable_data_hash":"7fcd0eaac3aad4c31a6a0e756475da92+52", }) with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', - api_client=api, keep_client=keep) as c: + api_client=api, keep_client=keep) as c: writer = c.open("count.txt", "rb+") self.assertEqual(writer.size(), 10) self.assertEqual(b"0123456789", writer.read(12)) @@ -101,18 +105,23 @@ class ArvadosFileWriterTestCase(unittest.TestCase): def test_truncate2(self): - keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"}) - api = ArvadosFileWriterTestCase.MockApi({"name":"test_truncate2", - "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 7f614da9329cd3aebf59b91aadc30bf0+67108864 0:12:count.txt\n", - "replication_desired":None}, - {"uuid":"zzzzz-4zz18-mockcollection0", - "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 7f614da9329cd3aebf59b91aadc30bf0+67108864 0:12:count.txt\n", - "portable_data_hash":"272da898abdf86ddc71994835e3155f8+95"}) + keep = ArvadosFileWriterTestCase.MockKeep({ + "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789", + }) + api = ArvadosFileWriterTestCase.MockApi({ + "name": "test_truncate2", + "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 7f614da9329cd3aebf59b91aadc30bf0+67108864 0:12:count.txt\n", + "replication_desired": None, + }, { + "uuid": "zzzzz-4zz18-mockcollection0", + "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 7f614da9329cd3aebf59b91aadc30bf0+67108864 0:12:count.txt\n", + "portable_data_hash": "272da898abdf86ddc71994835e3155f8+95", + }) with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', - api_client=api, keep_client=keep) as c: - writer = c.open("count.txt", "r+") + api_client=api, keep_client=keep) as c: + writer = c.open("count.txt", "rb+") self.assertEqual(writer.size(), 10) - self.assertEqual("0123456789", writer.read(12)) + self.assertEqual(b"0123456789", writer.read(12)) # extend file size writer.truncate(12) @@ -128,30 +137,35 @@ class ArvadosFileWriterTestCase(unittest.TestCase): self.assertFalse(c.modified()) def test_truncate3(self): - keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789", - "a925576942e94b2ef57a066101b48876+10": "abcdefghij"}) - api = ArvadosFileWriterTestCase.MockApi({"name":"test_truncate", - "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n", - "replication_desired":None}, - {"uuid":"zzzzz-4zz18-mockcollection0", - "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n", - "portable_data_hash":"7fcd0eaac3aad4c31a6a0e756475da92+52"}) + keep = ArvadosFileWriterTestCase.MockKeep({ + "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789", + "a925576942e94b2ef57a066101b48876+10": b"abcdefghij", + }) + api = ArvadosFileWriterTestCase.MockApi({ + "name": "test_truncate", + "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n", + "replication_desired": None, + }, { + "uuid": "zzzzz-4zz18-mockcollection0", + "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n", + "portable_data_hash": "7fcd0eaac3aad4c31a6a0e756475da92+52", + }) with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:20:count.txt\n', - api_client=api, keep_client=keep) as c: - writer = c.open("count.txt", "r+") + api_client=api, keep_client=keep) as c: + writer = c.open("count.txt", "rb+") self.assertEqual(writer.size(), 20) - self.assertEqual("0123456789ab", writer.read(12)) + self.assertEqual(b"0123456789ab", writer.read(12)) self.assertEqual(12, writer.tell()) writer.truncate(8) # Make sure reading off the end doesn't break self.assertEqual(12, writer.tell()) - self.assertEqual("", writer.read(12)) + self.assertEqual(b"", writer.read(12)) self.assertEqual(writer.size(), 8) self.assertEqual(2, writer.seek(-10, os.SEEK_CUR)) - self.assertEqual("234567", writer.read(12)) + self.assertEqual(b"234567", writer.read(12)) self.assertIsNone(c.manifest_locator()) self.assertTrue(c.modified()) @@ -203,19 +217,41 @@ class ArvadosFileWriterTestCase(unittest.TestCase): keep = ArvadosFileWriterTestCase.MockKeep({ "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789", }) - c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', keep_client=keep) - writer = c.open("count.txt", "ab+") - self.assertEqual(writer.read(20), b"0123456789") - writer.seek(0, os.SEEK_SET) + for (mode, convert) in ( + ('a+', lambda data: data.decode(encoding='utf-8')), + ('at+', lambda data: data.decode(encoding='utf-8')), + ('ab+', lambda data: data)): + c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', keep_client=keep) + writer = c.open("count.txt", mode) + self.assertEqual(writer.read(20), convert(b"0123456789")) - writer.write("hello") - self.assertEqual(writer.read(20), b"0123456789hello") - writer.seek(0, os.SEEK_SET) + writer.seek(0, os.SEEK_SET) + writer.write(convert(b"hello")) + self.assertEqual(writer.read(), convert(b"")) + if 'b' in mode: + writer.seek(-5, os.SEEK_CUR) + self.assertEqual(writer.read(3), convert(b"hel")) + self.assertEqual(writer.read(), convert(b"lo")) + else: + with self.assertRaises(IOError): + writer.seek(-5, os.SEEK_CUR) + with self.assertRaises(IOError): + writer.seek(-3, os.SEEK_END) + writer.seek(0, os.SEEK_SET) + writer.read(7) + self.assertEqual(7, writer.tell()) + self.assertEqual(7, writer.seek(7, os.SEEK_SET)) - writer.write("world") - self.assertEqual(writer.read(20), b"0123456789helloworld") + writer.seek(0, os.SEEK_SET) + self.assertEqual(writer.read(), convert(b"0123456789hello")) - self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 fc5e038d38a57032085441e7fe7010b0+10 0:20:count.txt\n", c.portable_manifest_text()) + writer.seek(0) + writer.write(convert(b"world")) + self.assertEqual(writer.read(), convert(b"")) + writer.seek(0) + self.assertEqual(writer.read(), convert(b"0123456789helloworld")) + + self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 fc5e038d38a57032085441e7fe7010b0+10 0:20:count.txt\n", c.portable_manifest_text()) def test_write_at_beginning(self): keep = ArvadosFileWriterTestCase.MockKeep({ @@ -352,10 +388,10 @@ class ArvadosFileWriterTestCase(unittest.TestCase): api = ArvadosFileWriterTestCase.MockApi({}, {}) with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt', api_client=api, keep_client=keep) as c: - writer = c.open("count.txt", "r+") + writer = c.open("count.txt", "rb+") self.assertEqual(writer.size(), 0) - text = "0123456789" + text = b"0123456789" writer.seek(2) writer.write(text) self.assertEqual(writer.size(), 12) @@ -370,7 +406,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase): api = ArvadosFileWriterTestCase.MockApi({}, {}) with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt', api_client=api, keep_client=keep) as c: - writer = c.open("count.txt", "r+") + writer = c.open("count.txt", "rb+") self.assertEqual(writer.size(), 0) text = "0123456789" @@ -387,28 +423,28 @@ class ArvadosFileWriterTestCase(unittest.TestCase): api = ArvadosFileWriterTestCase.MockApi({}, {}) for r in [[0, 1, 2, 3, 4], [4, 3, 2, 1, 0], [3, 2, 0, 4, 1]]: with Collection() as c: - writer = c.open("count.txt", "r+") + writer = c.open("count.txt", "rb+") self.assertEqual(writer.size(), 0) for i in r: w = ("%s" % i) * 10 writer.seek(i*10) - writer.write(w) + writer.write(w.encode()) writer.seek(0) - self.assertEqual(writer.read(), "00000000001111111111222222222233333333334444444444") + self.assertEqual(writer.read(), b"00000000001111111111222222222233333333334444444444") def test_sparse_write4(self): keep = ArvadosFileWriterTestCase.MockKeep({}) api = ArvadosFileWriterTestCase.MockApi({}, {}) for r in [[0, 1, 2, 4], [4, 2, 1, 0], [2, 0, 4, 1]]: with Collection() as c: - writer = c.open("count.txt", "r+") + writer = c.open("count.txt", "rb+") self.assertEqual(writer.size(), 0) for i in r: w = ("%s" % i) * 10 writer.seek(i*10) - writer.write(w) + writer.write(w.encode()) writer.seek(0) self.assertEqual(writer.read(), b"000000000011111111112222222222\x00\x00\x00\x00\x00\x00\x00\x00\x00\x004444444444") @@ -815,7 +851,7 @@ class BlockManagerTest(unittest.TestCase): mockkeep = mock.MagicMock() with arvados.arvfile._BlockManager(mockkeep) as blockmanager: bufferblock = blockmanager.alloc_bufferblock() - bufferblock.owner = mock.MagicMock() + bufferblock.owner = mock.MagicMock(spec=arvados.arvfile.ArvadosFile) def flush(sync=None): blockmanager.commit_bufferblock(bufferblock, sync) bufferblock.owner.flush.side_effect = flush @@ -829,7 +865,7 @@ class BlockManagerTest(unittest.TestCase): def test_bufferblock_commit_pending(self): # Test for bug #7225 mockkeep = mock.MagicMock() - mockkeep.put.side_effect = lambda x: time.sleep(1) + mockkeep.put.side_effect = lambda *args, **kwargs: time.sleep(1) with arvados.arvfile._BlockManager(mockkeep) as blockmanager: bufferblock = blockmanager.alloc_bufferblock() bufferblock.append("foo") @@ -846,7 +882,7 @@ class BlockManagerTest(unittest.TestCase): mockkeep.put.side_effect = arvados.errors.KeepWriteError("fail") with arvados.arvfile._BlockManager(mockkeep) as blockmanager: bufferblock = blockmanager.alloc_bufferblock() - bufferblock.owner = mock.MagicMock() + bufferblock.owner = mock.MagicMock(spec=arvados.arvfile.ArvadosFile) def flush(sync=None): blockmanager.commit_bufferblock(bufferblock, sync) bufferblock.owner.flush.side_effect = flush