merge 21447: closes #21447

[arvados.git] / sdk / python / tests / test_stream.py

diff --git a/sdk/python/tests/test_stream.py b/sdk/python/tests/test_stream.py
index d4b22070879901cbe55d604c1fc050f06d07a096..7e6e1f55a3c587bf1b1d4ea012918cdc0d1ddddd 100644 (file)
--- a/sdk/python/tests/test_stream.py
+++ b/sdk/python/tests/test_stream.py
@@ -1,51 +1,48 @@
-#!/usr/bin/env python
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0

 
 import bz2
 import gzip
 import io
 
 import bz2
 import gzip
 import io

-import mock

 import os
 import unittest
 import hashlib
 
 import os
 import unittest
 import hashlib
 

+from unittest import mock
+

 import arvados
 import arvados

-from arvados import StreamReader, StreamFileReader

 from arvados._ranges import Range
 
 from arvados._ranges import Range
 

-import arvados_testutil as tutil
-import run_test_server
-
-class StreamFileReaderTestCase(unittest.TestCase):
-    def make_count_reader(self):
-        stream = tutil.MockStreamReader('.', '01234', '34567', '67890')
-        return StreamFileReader(stream, [Range(1, 0, 3), Range(6, 3, 3), Range(11, 6, 3)],
-                                'count.txt')
+from . import arvados_testutil as tutil
+from . import run_test_server

 
 

+class StreamFileReaderTestMixin(object):

     def test_read_block_crossing_behavior(self):
         # read() calls will be aligned on block boundaries - see #3663.
         sfile = self.make_count_reader()
     def test_read_block_crossing_behavior(self):
         # read() calls will be aligned on block boundaries - see #3663.
         sfile = self.make_count_reader()

-        self.assertEqual('123', sfile.read(10))
+        self.assertEqual(b'123', sfile.read(10))

 
     def test_small_read(self):
         sfile = self.make_count_reader()
 
     def test_small_read(self):
         sfile = self.make_count_reader()

-        self.assertEqual('12', sfile.read(2))
+        self.assertEqual(b'12', sfile.read(2))

 
     def test_successive_reads(self):
         sfile = self.make_count_reader()
 
     def test_successive_reads(self):
         sfile = self.make_count_reader()

-        for expect in ['123', '456', '789', '']:
-            self.assertEqual(expect, sfile.read(10))
+        for expect in [b'1234', b'5678', b'9', b'']:
+            self.assertEqual(expect, sfile.read(4))

 
     def test_readfrom_spans_blocks(self):
         sfile = self.make_count_reader()
 
     def test_readfrom_spans_blocks(self):
         sfile = self.make_count_reader()

-        self.assertEqual('6789', sfile.readfrom(5, 12))
+        self.assertEqual(b'6789', sfile.readfrom(5, 12))

 
     def test_small_readfrom_spanning_blocks(self):
         sfile = self.make_count_reader()
 
     def test_small_readfrom_spanning_blocks(self):
         sfile = self.make_count_reader()

-        self.assertEqual('2345', sfile.readfrom(1, 4))
+        self.assertEqual(b'2345', sfile.readfrom(1, 4))

 
     def test_readall(self):
         sfile = self.make_count_reader()
 
     def test_readall(self):
         sfile = self.make_count_reader()

-        self.assertEqual('123456789', ''.join(sfile.readall()))
+        self.assertEqual(b'123456789', b''.join(sfile.readall()))

 
     def test_one_arg_seek(self):
         self.test_absolute_seek([])
 
     def test_one_arg_seek(self):
         self.test_absolute_seek([])


@@ -53,24 +50,26 @@ class StreamFileReaderTestCase(unittest.TestCase):
     def test_absolute_seek(self, args=[os.SEEK_SET]):
         sfile = self.make_count_reader()
         sfile.seek(6, *args)
     def test_absolute_seek(self, args=[os.SEEK_SET]):
         sfile = self.make_count_reader()
         sfile.seek(6, *args)

-        self.assertEqual('78', sfile.read(2))
+        self.assertEqual(b'78', sfile.read(2))

         sfile.seek(4, *args)
         sfile.seek(4, *args)

-        self.assertEqual('56', sfile.read(2))
+        self.assertEqual(b'56', sfile.read(2))

 
     def test_relative_seek(self, args=[os.SEEK_CUR]):
         sfile = self.make_count_reader()
 
     def test_relative_seek(self, args=[os.SEEK_CUR]):
         sfile = self.make_count_reader()

-        self.assertEqual('12', sfile.read(2))
+        self.assertEqual(b'12', sfile.read(2))

         sfile.seek(2, *args)
         sfile.seek(2, *args)

-        self.assertEqual('56', sfile.read(2))
+        self.assertEqual(b'56', sfile.read(2))

 
     def test_end_seek(self):
         sfile = self.make_count_reader()
         sfile.seek(-6, os.SEEK_END)
 
     def test_end_seek(self):
         sfile = self.make_count_reader()
         sfile.seek(-6, os.SEEK_END)

-        self.assertEqual('45', sfile.read(2))
+        self.assertEqual(b'45', sfile.read(2))

 
     def test_seek_min_zero(self):
         sfile = self.make_count_reader()
 
     def test_seek_min_zero(self):
         sfile = self.make_count_reader()

-        sfile.seek(-2, os.SEEK_SET)
+        self.assertEqual(0, sfile.tell())
+        with self.assertRaises(IOError):
+            sfile.seek(-2, os.SEEK_SET)

         self.assertEqual(0, sfile.tell())
 
     def test_seek_max_size(self):
         self.assertEqual(0, sfile.tell())
 
     def test_seek_max_size(self):


@@ -82,11 +81,6 @@ class StreamFileReaderTestCase(unittest.TestCase):
     def test_size(self):
         self.assertEqual(9, self.make_count_reader().size())
 
     def test_size(self):
         self.assertEqual(9, self.make_count_reader().size())
 

-    def test_tell_after_block_read(self):
-        sfile = self.make_count_reader()
-        sfile.read(5)
-        self.assertEqual(3, sfile.tell())
-

     def test_tell_after_small_read(self):
         sfile = self.make_count_reader()
         sfile.read(1)
     def test_tell_after_small_read(self):
         sfile = self.make_count_reader()
         sfile.read(1)


@@ -100,13 +94,9 @@ class StreamFileReaderTestCase(unittest.TestCase):
     def test_context(self):
         with self.make_count_reader() as sfile:
             self.assertFalse(sfile.closed, "reader is closed inside context")
     def test_context(self):
         with self.make_count_reader() as sfile:
             self.assertFalse(sfile.closed, "reader is closed inside context")

-            self.assertEqual('12', sfile.read(2))
+            self.assertEqual(b'12', sfile.read(2))

         self.assertTrue(sfile.closed, "reader is open after context")
 
         self.assertTrue(sfile.closed, "reader is open after context")
 

-    def make_newlines_reader(self):
-        stream = tutil.MockStreamReader('.', 'one\ntwo\n\nth', 'ree\nfour\n\n')
-        return StreamFileReader(stream, [Range(0, 0, 11), Range(11, 11, 10)], 'count.txt')
-

     def check_lines(self, actual):
         self.assertEqual(['one\n', 'two\n', '\n', 'three\n', 'four\n', '\n'],
                          actual)
     def check_lines(self, actual):
         self.assertEqual(['one\n', 'two\n', '\n', 'three\n', 'four\n', '\n'],
                          actual)


@@ -137,19 +127,14 @@ class StreamFileReaderTestCase(unittest.TestCase):
 
     def test_readlines_sizehint(self):
         result = self.make_newlines_reader().readlines(8)
 
     def test_readlines_sizehint(self):
         result = self.make_newlines_reader().readlines(8)

-        self.assertEqual(['one\n', 'two\n'], result[:2])
-        self.assertNotIn('three\n', result)
+        self.assertEqual(['one\n', 'two\n', '\n', 'three\n', 'four\n', '\n'], result)

 
     def test_name_attribute(self):
 
     def test_name_attribute(self):

-        # Test both .name and .name() (for backward compatibility)
-        stream = tutil.MockStreamReader()
-        sfile = StreamFileReader(stream, [Range(0, 0, 0)], 'nametest')
+        sfile = self.make_file_reader(name='nametest')

         self.assertEqual('nametest', sfile.name)
         self.assertEqual('nametest', sfile.name)

-        self.assertEqual('nametest', sfile.name())

 
     def check_decompressed_name(self, filename, expect):
 
     def check_decompressed_name(self, filename, expect):

-        stream = tutil.MockStreamReader('.', '')
-        reader = StreamFileReader(stream, [Range(0, 0, 0)], filename)
+        reader = self.make_file_reader(name=filename)

         self.assertEqual(expect, reader.decompressed_name())
 
     def test_decompressed_name_uncompressed_file(self):
         self.assertEqual(expect, reader.decompressed_name())
 
     def test_decompressed_name_uncompressed_file(self):


@@ -162,12 +147,10 @@ class StreamFileReaderTestCase(unittest.TestCase):
         self.check_decompressed_name('test.log.bz2', 'test.log')
 
     def check_decompression(self, compress_ext, compress_func):
         self.check_decompressed_name('test.log.bz2', 'test.log')
 
     def check_decompression(self, compress_ext, compress_func):

-        test_text = 'decompression\ntest\n'
+        test_text = b'decompression\ntest\n'

         test_data = compress_func(test_text)
         test_data = compress_func(test_text)

-        stream = tutil.MockStreamReader('.', test_data)
-        reader = StreamFileReader(stream, [Range(0, 0, len(test_data))],
-                                  'test.' + compress_ext)
-        self.assertEqual(test_text, ''.join(reader.readall_decompressed()))
+        reader = self.make_file_reader(name='test.'+compress_ext, data=test_data)
+        self.assertEqual(test_text, b''.join(reader.readall_decompressed()))

 
     @staticmethod
     def gzip_compress(data):
 
     @staticmethod
     def gzip_compress(data):


@@ -196,7 +179,7 @@ class StreamFileReaderTestCase(unittest.TestCase):
         reader = self.make_newlines_reader()
         data = reader.readline()
         self.assertEqual('one\n', data)
         reader = self.make_newlines_reader()
         data = reader.readline()
         self.assertEqual('one\n', data)

-        self.assertEqual(''.join(['two\n', '\n', 'three\n', 'four\n', '\n']), ''.join(reader.readall()))
+        self.assertEqual(b''.join([b'two\n', b'\n', b'three\n', b'four\n', b'\n']), b''.join(reader.readall()))

 
 
 class StreamRetryTestMixin(object):
 
 
 class StreamRetryTestMixin(object):


@@ -215,26 +198,19 @@ class StreamRetryTestMixin(object):
     def test_success_without_retries(self):
         with tutil.mock_keep_responses('bar', 200):
             reader = self.reader_for('bar_file')
     def test_success_without_retries(self):
         with tutil.mock_keep_responses('bar', 200):
             reader = self.reader_for('bar_file')

-            self.assertEqual('bar', self.read_for_test(reader, 3))
-
-    @tutil.skip_sleep
-    def test_read_no_default_retry(self):
-        with tutil.mock_keep_responses('', 500):
-            reader = self.reader_for('user_agreement')
-            with self.assertRaises(arvados.errors.KeepReadError):
-                self.read_for_test(reader, 10)
+            self.assertEqual(b'bar', self.read_for_test(reader, 3))

 
     @tutil.skip_sleep
     def test_read_with_instance_retries(self):
         with tutil.mock_keep_responses('foo', 500, 200):
             reader = self.reader_for('foo_file', num_retries=3)
 
     @tutil.skip_sleep
     def test_read_with_instance_retries(self):
         with tutil.mock_keep_responses('foo', 500, 200):
             reader = self.reader_for('foo_file', num_retries=3)

-            self.assertEqual('foo', self.read_for_test(reader, 3))
+            self.assertEqual(b'foo', self.read_for_test(reader, 3))

 
     @tutil.skip_sleep
     def test_read_with_method_retries(self):
         with tutil.mock_keep_responses('foo', 500, 200):
             reader = self.reader_for('foo_file')
 
     @tutil.skip_sleep
     def test_read_with_method_retries(self):
         with tutil.mock_keep_responses('foo', 500, 200):
             reader = self.reader_for('foo_file')

-            self.assertEqual('foo',
+            self.assertEqual(b'foo',

                              self.read_for_test(reader, 3, num_retries=3))
 
     @tutil.skip_sleep
                              self.read_for_test(reader, 3, num_retries=3))
 
     @tutil.skip_sleep


@@ -259,48 +235,5 @@ class StreamRetryTestMixin(object):
                 self.read_for_test(reader, 10, num_retries=1)
 
 
                 self.read_for_test(reader, 10, num_retries=1)
 
 

-class StreamReaderTestCase(unittest.TestCase, StreamRetryTestMixin):
-    def reader_for(self, coll_name, **kwargs):
-        return StreamReader(self.manifest_for(coll_name).split(),
-                            self.keep_client(), **kwargs)
-
-    def read_for_test(self, reader, byte_count, **kwargs):
-        return reader.readfrom(0, byte_count, **kwargs)
-
-    def test_manifest_text_without_keep_client(self):
-        mtext = self.manifest_for('multilevel_collection_1')
-        for line in mtext.rstrip('\n').split('\n'):
-            reader = StreamReader(line.split())
-            self.assertEqual(line + '\n', reader.manifest_text())
-
-
-class StreamFileReadTestCase(unittest.TestCase, StreamRetryTestMixin):
-    def reader_for(self, coll_name, **kwargs):
-        return StreamReader(self.manifest_for(coll_name).split(),
-                            self.keep_client(), **kwargs).all_files()[0]
-
-    def read_for_test(self, reader, byte_count, **kwargs):
-        return reader.read(byte_count, **kwargs)
-
-
-class StreamFileReadFromTestCase(StreamFileReadTestCase):
-    def read_for_test(self, reader, byte_count, **kwargs):
-        return reader.readfrom(0, byte_count, **kwargs)
-
-
-class StreamFileReadAllTestCase(StreamFileReadTestCase):
-    def read_for_test(self, reader, byte_count, **kwargs):
-        return ''.join(reader.readall(**kwargs))
-
-
-class StreamFileReadAllDecompressedTestCase(StreamFileReadTestCase):
-    def read_for_test(self, reader, byte_count, **kwargs):
-        return ''.join(reader.readall_decompressed(**kwargs))
-
-
-class StreamFileReadlinesTestCase(StreamFileReadTestCase):
-    def read_for_test(self, reader, byte_count, **kwargs):
-        return ''.join(reader.readlines(**kwargs))
-

 if __name__ == '__main__':
     unittest.main()
 if __name__ == '__main__':
     unittest.main()