X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/ffb304afad93f50fd6ee43ecda6584dd0ac000c4..093ec98e4a065acfc537ea22c08c337c115fe273:/sdk/python/tests/test_arvfile.py diff --git a/sdk/python/tests/test_arvfile.py b/sdk/python/tests/test_arvfile.py index a800913055..b45a592ecd 100644 --- a/sdk/python/tests/test_arvfile.py +++ b/sdk/python/tests/test_arvfile.py @@ -1,12 +1,13 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + from __future__ import absolute_import from builtins import hex from builtins import str from builtins import range from builtins import object -import bz2 import datetime -import gzip -import io import mock import os import unittest @@ -15,7 +16,7 @@ import time import arvados from arvados._ranges import Range from arvados.keep import KeepLocator -from arvados.collection import Collection, CollectionReader +from arvados.collection import Collection from arvados.arvfile import ArvadosFile, ArvadosFileReader from . import arvados_testutil as tutil @@ -26,13 +27,13 @@ class ArvadosFileWriterTestCase(unittest.TestCase): def __init__(self, blocks): self.blocks = blocks self.requests = [] - def get(self, locator, num_retries=0): + def get(self, locator, num_retries=0, prefetch=False): self.requests.append(locator) return self.blocks.get(locator) def get_from_cache(self, locator): self.requests.append(locator) return self.blocks.get(locator) - def put(self, data, num_retries=None, copies=None): + def put(self, data, num_retries=None, copies=None, classes=[]): pdh = tutil.str_keep_locator(data) self.blocks[pdh] = bytes(data) return pdh @@ -169,8 +170,6 @@ class ArvadosFileWriterTestCase(unittest.TestCase): self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator()) self.assertFalse(c.modified()) - - def test_write_to_end(self): keep = ArvadosFileWriterTestCase.MockKeep({ "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789", @@ -213,26 +212,41 @@ class ArvadosFileWriterTestCase(unittest.TestCase): keep = ArvadosFileWriterTestCase.MockKeep({ "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789", }) - c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', keep_client=keep) - writer = c.open("count.txt", "ab+") - self.assertEqual(writer.read(20), b"0123456789") - - writer.seek(0, os.SEEK_SET) - writer.write("hello") - self.assertEqual(writer.read(), b"") - writer.seek(-5, os.SEEK_CUR) - self.assertEqual(writer.read(3), b"hel") - self.assertEqual(writer.read(), b"lo") - writer.seek(0, os.SEEK_SET) - self.assertEqual(writer.read(), b"0123456789hello") - - writer.seek(0) - writer.write("world") - self.assertEqual(writer.read(), b"") - writer.seek(0) - self.assertEqual(writer.read(), b"0123456789helloworld") - - self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 fc5e038d38a57032085441e7fe7010b0+10 0:20:count.txt\n", c.portable_manifest_text()) + for (mode, convert) in ( + ('a+', lambda data: data.decode(encoding='utf-8')), + ('at+', lambda data: data.decode(encoding='utf-8')), + ('ab+', lambda data: data)): + c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', keep_client=keep) + writer = c.open("count.txt", mode) + self.assertEqual(writer.read(20), convert(b"0123456789")) + + writer.seek(0, os.SEEK_SET) + writer.write(convert(b"hello")) + self.assertEqual(writer.read(), convert(b"")) + if 'b' in mode: + writer.seek(-5, os.SEEK_CUR) + self.assertEqual(writer.read(3), convert(b"hel")) + self.assertEqual(writer.read(), convert(b"lo")) + else: + with self.assertRaises(IOError): + writer.seek(-5, os.SEEK_CUR) + with self.assertRaises(IOError): + writer.seek(-3, os.SEEK_END) + writer.seek(0, os.SEEK_SET) + writer.read(7) + self.assertEqual(7, writer.tell()) + self.assertEqual(7, writer.seek(7, os.SEEK_SET)) + + writer.seek(0, os.SEEK_SET) + self.assertEqual(writer.read(), convert(b"0123456789hello")) + + writer.seek(0) + writer.write(convert(b"world")) + self.assertEqual(writer.read(), convert(b"")) + writer.seek(0) + self.assertEqual(writer.read(), convert(b"0123456789helloworld")) + + self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 fc5e038d38a57032085441e7fe7010b0+10 0:20:count.txt\n", c.portable_manifest_text()) def test_write_at_beginning(self): keep = ArvadosFileWriterTestCase.MockKeep({ @@ -347,7 +361,6 @@ class ArvadosFileWriterTestCase(unittest.TestCase): self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator()) self.assertFalse(c.modified()) - def test_large_write(self): keep = ArvadosFileWriterTestCase.MockKeep({}) api = ArvadosFileWriterTestCase.MockApi({}, {}) @@ -381,7 +394,6 @@ class ArvadosFileWriterTestCase(unittest.TestCase): self.assertEqual(c.manifest_text(), ". 7f614da9329cd3aebf59b91aadc30bf0+67108864 781e5e245d69b566979b86e28d23f2c7+10 0:2:count.txt 67108864:10:count.txt\n") - def test_sparse_write2(self): keep = ArvadosFileWriterTestCase.MockKeep({}) api = ArvadosFileWriterTestCase.MockApi({}, {}) @@ -398,7 +410,6 @@ class ArvadosFileWriterTestCase(unittest.TestCase): self.assertEqual(c.manifest_text(), ". 7f614da9329cd3aebf59b91aadc30bf0+67108864 781e5e245d69b566979b86e28d23f2c7+10 0:67108864:count.txt 0:67108864:count.txt 0:2:count.txt 67108864:10:count.txt\n") - def test_sparse_write3(self): keep = ArvadosFileWriterTestCase.MockKeep({}) api = ArvadosFileWriterTestCase.MockApi({}, {}) @@ -429,7 +440,6 @@ class ArvadosFileWriterTestCase(unittest.TestCase): writer.seek(0) self.assertEqual(writer.read(), b"000000000011111111112222222222\x00\x00\x00\x00\x00\x00\x00\x00\x00\x004444444444") - def test_rewrite_on_empty_file(self): keep = ArvadosFileWriterTestCase.MockKeep({}) with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt', @@ -617,6 +627,7 @@ class ArvadosFileReaderTestCase(StreamFileReaderTestCase): def __init__(self, blocks, nocache): self.blocks = blocks self.nocache = nocache + self.num_get_threads = 1 def block_prefetch(self, loc): pass @@ -832,7 +843,7 @@ class BlockManagerTest(unittest.TestCase): mockkeep = mock.MagicMock() with arvados.arvfile._BlockManager(mockkeep) as blockmanager: bufferblock = blockmanager.alloc_bufferblock() - bufferblock.owner = mock.MagicMock() + bufferblock.owner = mock.MagicMock(spec=arvados.arvfile.ArvadosFile) def flush(sync=None): blockmanager.commit_bufferblock(bufferblock, sync) bufferblock.owner.flush.side_effect = flush @@ -846,7 +857,7 @@ class BlockManagerTest(unittest.TestCase): def test_bufferblock_commit_pending(self): # Test for bug #7225 mockkeep = mock.MagicMock() - mockkeep.put.side_effect = lambda x: time.sleep(1) + mockkeep.put.side_effect = lambda *args, **kwargs: time.sleep(1) with arvados.arvfile._BlockManager(mockkeep) as blockmanager: bufferblock = blockmanager.alloc_bufferblock() bufferblock.append("foo") @@ -857,13 +868,12 @@ class BlockManagerTest(unittest.TestCase): blockmanager.commit_bufferblock(bufferblock, True) self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.COMMITTED) - def test_bufferblock_commit_with_error(self): mockkeep = mock.MagicMock() mockkeep.put.side_effect = arvados.errors.KeepWriteError("fail") with arvados.arvfile._BlockManager(mockkeep) as blockmanager: bufferblock = blockmanager.alloc_bufferblock() - bufferblock.owner = mock.MagicMock() + bufferblock.owner = mock.MagicMock(spec=arvados.arvfile.ArvadosFile) def flush(sync=None): blockmanager.commit_bufferblock(bufferblock, sync) bufferblock.owner.flush.side_effect = flush