X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/2d56bd2e4722e9c664f7605b3a46b505a30782db..HEAD:/sdk/python/tests/test_stream.py diff --git a/sdk/python/tests/test_stream.py b/sdk/python/tests/test_stream.py index acb9929cae..7e6e1f55a3 100644 --- a/sdk/python/tests/test_stream.py +++ b/sdk/python/tests/test_stream.py @@ -1,51 +1,48 @@ -#!/usr/bin/env python +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 import bz2 import gzip import io -import mock import os import unittest import hashlib +from unittest import mock + import arvados -from arvados import StreamReader, StreamFileReader from arvados._ranges import Range -import arvados_testutil as tutil -import run_test_server - -class StreamFileReaderTestCase(unittest.TestCase): - def make_count_reader(self): - stream = tutil.MockStreamReader('.', '01234', '34567', '67890') - return StreamFileReader(stream, [Range(1, 0, 3), Range(6, 3, 3), Range(11, 6, 3)], - 'count.txt') +from . import arvados_testutil as tutil +from . import run_test_server - def test_read_returns_first_block(self): +class StreamFileReaderTestMixin(object): + def test_read_block_crossing_behavior(self): # read() calls will be aligned on block boundaries - see #3663. sfile = self.make_count_reader() - self.assertEqual('123', sfile.read(10)) + self.assertEqual(b'123', sfile.read(10)) def test_small_read(self): sfile = self.make_count_reader() - self.assertEqual('12', sfile.read(2)) + self.assertEqual(b'12', sfile.read(2)) def test_successive_reads(self): sfile = self.make_count_reader() - for expect in ['123', '456', '789', '']: - self.assertEqual(expect, sfile.read(10)) + for expect in [b'1234', b'5678', b'9', b'']: + self.assertEqual(expect, sfile.read(4)) def test_readfrom_spans_blocks(self): sfile = self.make_count_reader() - self.assertEqual('6789', sfile.readfrom(5, 12)) + self.assertEqual(b'6789', sfile.readfrom(5, 12)) def test_small_readfrom_spanning_blocks(self): sfile = self.make_count_reader() - self.assertEqual('2345', sfile.readfrom(1, 4)) + self.assertEqual(b'2345', sfile.readfrom(1, 4)) def test_readall(self): sfile = self.make_count_reader() - self.assertEqual('123456789', ''.join(sfile.readall())) + self.assertEqual(b'123456789', b''.join(sfile.readall())) def test_one_arg_seek(self): self.test_absolute_seek([]) @@ -53,39 +50,37 @@ class StreamFileReaderTestCase(unittest.TestCase): def test_absolute_seek(self, args=[os.SEEK_SET]): sfile = self.make_count_reader() sfile.seek(6, *args) - self.assertEqual('78', sfile.read(2)) + self.assertEqual(b'78', sfile.read(2)) sfile.seek(4, *args) - self.assertEqual('56', sfile.read(2)) + self.assertEqual(b'56', sfile.read(2)) def test_relative_seek(self, args=[os.SEEK_CUR]): sfile = self.make_count_reader() - self.assertEqual('12', sfile.read(2)) + self.assertEqual(b'12', sfile.read(2)) sfile.seek(2, *args) - self.assertEqual('56', sfile.read(2)) + self.assertEqual(b'56', sfile.read(2)) def test_end_seek(self): sfile = self.make_count_reader() sfile.seek(-6, os.SEEK_END) - self.assertEqual('45', sfile.read(2)) + self.assertEqual(b'45', sfile.read(2)) def test_seek_min_zero(self): sfile = self.make_count_reader() - sfile.seek(-2, os.SEEK_SET) + self.assertEqual(0, sfile.tell()) + with self.assertRaises(IOError): + sfile.seek(-2, os.SEEK_SET) self.assertEqual(0, sfile.tell()) def test_seek_max_size(self): sfile = self.make_count_reader() sfile.seek(2, os.SEEK_END) - self.assertEqual(9, sfile.tell()) + # POSIX permits seeking past end of file. + self.assertEqual(11, sfile.tell()) def test_size(self): self.assertEqual(9, self.make_count_reader().size()) - def test_tell_after_block_read(self): - sfile = self.make_count_reader() - sfile.read(5) - self.assertEqual(3, sfile.tell()) - def test_tell_after_small_read(self): sfile = self.make_count_reader() sfile.read(1) @@ -99,13 +94,9 @@ class StreamFileReaderTestCase(unittest.TestCase): def test_context(self): with self.make_count_reader() as sfile: self.assertFalse(sfile.closed, "reader is closed inside context") - self.assertEqual('12', sfile.read(2)) + self.assertEqual(b'12', sfile.read(2)) self.assertTrue(sfile.closed, "reader is open after context") - def make_newlines_reader(self): - stream = tutil.MockStreamReader('.', 'one\ntwo\n\nth', 'ree\nfour\n\n') - return StreamFileReader(stream, [Range(0, 0, 11), Range(11, 11, 10)], 'count.txt') - def check_lines(self, actual): self.assertEqual(['one\n', 'two\n', '\n', 'three\n', 'four\n', '\n'], actual) @@ -136,19 +127,14 @@ class StreamFileReaderTestCase(unittest.TestCase): def test_readlines_sizehint(self): result = self.make_newlines_reader().readlines(8) - self.assertEqual(['one\n', 'two\n'], result[:2]) - self.assertNotIn('three\n', result) + self.assertEqual(['one\n', 'two\n', '\n', 'three\n', 'four\n', '\n'], result) def test_name_attribute(self): - # Test both .name and .name() (for backward compatibility) - stream = tutil.MockStreamReader() - sfile = StreamFileReader(stream, [Range(0, 0, 0)], 'nametest') + sfile = self.make_file_reader(name='nametest') self.assertEqual('nametest', sfile.name) - self.assertEqual('nametest', sfile.name()) def check_decompressed_name(self, filename, expect): - stream = tutil.MockStreamReader('.', '') - reader = StreamFileReader(stream, [Range(0, 0, 0)], filename) + reader = self.make_file_reader(name=filename) self.assertEqual(expect, reader.decompressed_name()) def test_decompressed_name_uncompressed_file(self): @@ -161,12 +147,10 @@ class StreamFileReaderTestCase(unittest.TestCase): self.check_decompressed_name('test.log.bz2', 'test.log') def check_decompression(self, compress_ext, compress_func): - test_text = 'decompression\ntest\n' + test_text = b'decompression\ntest\n' test_data = compress_func(test_text) - stream = tutil.MockStreamReader('.', test_data) - reader = StreamFileReader(stream, [Range(0, 0, len(test_data))], - 'test.' + compress_ext) - self.assertEqual(test_text, ''.join(reader.readall_decompressed())) + reader = self.make_file_reader(name='test.'+compress_ext, data=test_data) + self.assertEqual(test_text, b''.join(reader.readall_decompressed())) @staticmethod def gzip_compress(data): @@ -184,6 +168,19 @@ class StreamFileReaderTestCase(unittest.TestCase): def test_bz2_decompression(self): self.check_decompression('bz2', bz2.compress) + def test_readline_then_readlines(self): + reader = self.make_newlines_reader() + data = reader.readline() + self.assertEqual('one\n', data) + data = reader.readlines() + self.assertEqual(['two\n', '\n', 'three\n', 'four\n', '\n'], data) + + def test_readline_then_readall(self): + reader = self.make_newlines_reader() + data = reader.readline() + self.assertEqual('one\n', data) + self.assertEqual(b''.join([b'two\n', b'\n', b'three\n', b'four\n', b'\n']), b''.join(reader.readall())) + class StreamRetryTestMixin(object): # Define reader_for(coll_name, **kwargs) @@ -199,94 +196,44 @@ class StreamRetryTestMixin(object): @tutil.skip_sleep def test_success_without_retries(self): - with tutil.mock_get_responses('bar', 200): + with tutil.mock_keep_responses('bar', 200): reader = self.reader_for('bar_file') - self.assertEqual('bar', self.read_for_test(reader, 3)) - - @tutil.skip_sleep - def test_read_no_default_retry(self): - with tutil.mock_get_responses('', 500): - reader = self.reader_for('user_agreement') - with self.assertRaises(arvados.errors.KeepReadError): - self.read_for_test(reader, 10) + self.assertEqual(b'bar', self.read_for_test(reader, 3)) @tutil.skip_sleep def test_read_with_instance_retries(self): - with tutil.mock_get_responses('foo', 500, 200): + with tutil.mock_keep_responses('foo', 500, 200): reader = self.reader_for('foo_file', num_retries=3) - self.assertEqual('foo', self.read_for_test(reader, 3)) + self.assertEqual(b'foo', self.read_for_test(reader, 3)) @tutil.skip_sleep def test_read_with_method_retries(self): - with tutil.mock_get_responses('foo', 500, 200): + with tutil.mock_keep_responses('foo', 500, 200): reader = self.reader_for('foo_file') - self.assertEqual('foo', + self.assertEqual(b'foo', self.read_for_test(reader, 3, num_retries=3)) @tutil.skip_sleep def test_read_instance_retries_exhausted(self): - with tutil.mock_get_responses('bar', 500, 500, 500, 500, 200): + with tutil.mock_keep_responses('bar', 500, 500, 500, 500, 200): reader = self.reader_for('bar_file', num_retries=3) with self.assertRaises(arvados.errors.KeepReadError): self.read_for_test(reader, 3) @tutil.skip_sleep def test_read_method_retries_exhausted(self): - with tutil.mock_get_responses('bar', 500, 500, 500, 500, 200): + with tutil.mock_keep_responses('bar', 500, 500, 500, 500, 200): reader = self.reader_for('bar_file') with self.assertRaises(arvados.errors.KeepReadError): self.read_for_test(reader, 3, num_retries=3) @tutil.skip_sleep def test_method_retries_take_precedence(self): - with tutil.mock_get_responses('', 500, 500, 500, 200): + with tutil.mock_keep_responses('', 500, 500, 500, 200): reader = self.reader_for('user_agreement', num_retries=10) with self.assertRaises(arvados.errors.KeepReadError): self.read_for_test(reader, 10, num_retries=1) -class StreamReaderTestCase(unittest.TestCase, StreamRetryTestMixin): - def reader_for(self, coll_name, **kwargs): - return StreamReader(self.manifest_for(coll_name).split(), - self.keep_client(), **kwargs) - - def read_for_test(self, reader, byte_count, **kwargs): - return reader.readfrom(0, byte_count, **kwargs) - - def test_manifest_text_without_keep_client(self): - mtext = self.manifest_for('multilevel_collection_1') - for line in mtext.rstrip('\n').split('\n'): - reader = StreamReader(line.split()) - self.assertEqual(line + '\n', reader.manifest_text()) - - -class StreamFileReadTestCase(unittest.TestCase, StreamRetryTestMixin): - def reader_for(self, coll_name, **kwargs): - return StreamReader(self.manifest_for(coll_name).split(), - self.keep_client(), **kwargs).all_files()[0] - - def read_for_test(self, reader, byte_count, **kwargs): - return reader.read(byte_count, **kwargs) - - -class StreamFileReadFromTestCase(StreamFileReadTestCase): - def read_for_test(self, reader, byte_count, **kwargs): - return reader.readfrom(0, byte_count, **kwargs) - - -class StreamFileReadAllTestCase(StreamFileReadTestCase): - def read_for_test(self, reader, byte_count, **kwargs): - return ''.join(reader.readall(**kwargs)) - - -class StreamFileReadAllDecompressedTestCase(StreamFileReadTestCase): - def read_for_test(self, reader, byte_count, **kwargs): - return ''.join(reader.readall_decompressed(**kwargs)) - - -class StreamFileReadlinesTestCase(StreamFileReadTestCase): - def read_for_test(self, reader, byte_count, **kwargs): - return ''.join(reader.readlines(**kwargs)) - if __name__ == '__main__': unittest.main()