From: Tom Clegg Date: Sun, 2 Apr 2017 05:26:36 +0000 (-0400) Subject: 11308: Fix string/bytes confusion for Python 3 X-Git-Tag: 1.1.0~266^2~40 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/991d037e7b4741450dff745d22b0758170fe4ed9?hp=82f763b628589918d0e421b4f5fb4c00c179a628 11308: Fix string/bytes confusion for Python 3 --- diff --git a/sdk/python/arvados/api.py b/sdk/python/arvados/api.py index 59a73b45e5..a06170d2e4 100644 --- a/sdk/python/arvados/api.py +++ b/sdk/python/arvados/api.py @@ -117,6 +117,7 @@ _cast_orig = apiclient_discovery._cast def _cast_objects_too(value, schema_type): global _cast_orig if (type(value) != type('') and + type(value) != type(b'') and (schema_type == 'object' or schema_type == 'array')): return json.dumps(value) else: diff --git a/sdk/python/arvados/arvfile.py b/sdk/python/arvados/arvfile.py index c6cb1c91cc..9f95ee091a 100644 --- a/sdk/python/arvados/arvfile.py +++ b/sdk/python/arvados/arvfile.py @@ -112,7 +112,7 @@ class ArvadosFileReaderBase(_FileLikeObjectBase): def readall(self, size=2**20, num_retries=None): while True: data = self.read(size, num_retries=num_retries) - if data == '': + if len(data) == 0: break yield data @@ -124,23 +124,23 @@ class ArvadosFileReaderBase(_FileLikeObjectBase): data = [cache_data] self._filepos += len(cache_data) else: - data = [''] + data = [b''] data_size = len(data[-1]) - while (data_size < size) and ('\n' not in data[-1]): + while (data_size < size) and (b'\n' not in data[-1]): next_read = self.read(2 ** 20, num_retries=num_retries) if not next_read: break data.append(next_read) data_size += len(next_read) - data = ''.join(data) + data = b''.join(data) try: - nextline_index = data.index('\n') + 1 + nextline_index = data.index(b'\n') + 1 except ValueError: nextline_index = len(data) nextline_index = min(nextline_index, size) self._filepos -= len(data) - nextline_index self._readline_cache = (self.tell(), data[nextline_index:]) - return data[:nextline_index] + return data[:nextline_index].decode() @_FileLikeObjectBase._before_close @retry_method @@ -175,7 +175,7 @@ class ArvadosFileReaderBase(_FileLikeObjectBase): data_size += len(s) if data_size >= sizehint: break - return ''.join(data).splitlines(True) + return b''.join(data).decode().splitlines(True) def size(self): raise NotImplementedError() @@ -212,9 +212,9 @@ class StreamFileReader(ArvadosFileReaderBase): def read(self, size, num_retries=None): """Read up to 'size' bytes from the stream, starting at the current file position""" if size == 0: - return '' + return b'' - data = '' + data = b'' available_chunks = locators_and_ranges(self.segments, self._filepos, size) if available_chunks: lr = available_chunks[0] @@ -230,13 +230,13 @@ class StreamFileReader(ArvadosFileReaderBase): def readfrom(self, start, size, num_retries=None): """Read up to 'size' bytes from the stream, starting at 'start'""" if size == 0: - return '' + return b'' data = [] for lr in locators_and_ranges(self.segments, start, size): data.append(self._stream.readfrom(lr.locator+lr.segment_offset, lr.segment_size, num_retries=num_retries)) - return ''.join(data) + return b''.join(data) def as_manifest(self): segs = [] @@ -316,6 +316,8 @@ class _BufferBlock(object): """ if self._state == _BufferBlock.WRITABLE: + if not isinstance(data, bytes) and not isinstance(data, memoryview): + data = data.encode() while (self.write_pointer+len(data)) > len(self.buffer_block): new_buffer_block = bytearray(len(self.buffer_block) * 2) new_buffer_block[0:self.write_pointer] = self.buffer_block[0:self.write_pointer] @@ -944,7 +946,7 @@ class ArvadosFile(object): with self.lock: if size == 0 or offset >= self.size(): - return '' + return b'' readsegs = locators_and_ranges(self._segments, offset, size) prefetch = locators_and_ranges(self._segments, offset + size, config.KEEP_BLOCK_SIZE, limit=32) @@ -964,7 +966,7 @@ class ArvadosFile(object): self.parent._my_block_manager().block_prefetch(lr.locator) locs.add(lr.locator) - return ''.join(data) + return b''.join(data) def _repack_writes(self, num_retries): """Test if the buffer block has more data than actual segments. @@ -1001,6 +1003,8 @@ class ArvadosFile(object): necessary. """ + if not isinstance(data, bytes) and not isinstance(data, memoryview): + data = data.encode() if len(data) == 0: return @@ -1157,7 +1161,7 @@ class ArvadosFileReader(ArvadosFileReaderBase): data.append(rd) self._filepos += len(rd) rd = self.arvadosfile.readfrom(self._filepos, config.KEEP_BLOCK_SIZE, num_retries) - return ''.join(data) + return b''.join(data) else: data = self.arvadosfile.readfrom(self._filepos, size, num_retries, exact=True) self._filepos += len(data) diff --git a/sdk/python/arvados/cache.py b/sdk/python/arvados/cache.py index 868b478ac5..f59d92f5d9 100644 --- a/sdk/python/arvados/cache.py +++ b/sdk/python/arvados/cache.py @@ -33,7 +33,7 @@ class SafeHTTPCache(object): return self._dir def _filename(self, url): - return os.path.join(self._dir, hashlib.md5(url).hexdigest()+'.tmp') + return os.path.join(self._dir, hashlib.md5(url.encode('utf-8')).hexdigest()+'.tmp') def get(self, url): filename = self._filename(url) @@ -50,7 +50,7 @@ class SafeHTTPCache(object): return None try: try: - f = os.fdopen(fd, 'w') + f = os.fdopen(fd, 'wb') except: os.close(fd) raise diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py index 0d88084340..1c68c8ed43 100644 --- a/sdk/python/arvados/collection.py +++ b/sdk/python/arvados/collection.py @@ -220,7 +220,11 @@ class CollectionWriter(CollectionBase): self.do_queued_work() def write(self, newdata): - if hasattr(newdata, '__iter__'): + if isinstance(newdata, bytes): + pass + elif isinstance(newdata, str): + newdata = newdata.encode() + elif hasattr(newdata, '__iter__'): for s in newdata: self.write(s) return @@ -260,7 +264,7 @@ class CollectionWriter(CollectionBase): return self._last_open def flush_data(self): - data_buffer = ''.join(self._data_buffer) + data_buffer = b''.join(self._data_buffer) if data_buffer: self._current_stream_locators.append( self._my_keep().put( @@ -350,10 +354,11 @@ class CollectionWriter(CollectionBase): sending manifest_text() to the API server's "create collection" endpoint. """ - return self._my_keep().put(self.manifest_text(), copies=self.replication) + return self._my_keep().put(self.manifest_text().encode(), + copies=self.replication) def portable_data_hash(self): - stripped = self.stripped_manifest() + stripped = self.stripped_manifest().encode() return hashlib.md5(stripped).hexdigest() + '+' + str(len(stripped)) def manifest_text(self): @@ -1078,7 +1083,7 @@ class RichCollectionBase(CollectionBase): # then return API server's PDH response. return self._portable_data_hash else: - stripped = self.portable_manifest_text() + stripped = self.portable_manifest_text().encode() return hashlib.md5(stripped).hexdigest() + '+' + str(len(stripped)) @synchronized @@ -1336,7 +1341,7 @@ class Collection(RichCollectionBase): # mode. Return an exception, or None if successful. try: self._manifest_text = self._my_keep().get( - self._manifest_locator, num_retries=self.num_retries) + self._manifest_locator, num_retries=self.num_retries).decode() except Exception as e: return e diff --git a/sdk/python/arvados/commands/keepdocker.py b/sdk/python/arvados/commands/keepdocker.py index a7ef0bec42..354ae50822 100644 --- a/sdk/python/arvados/commands/keepdocker.py +++ b/sdk/python/arvados/commands/keepdocker.py @@ -99,7 +99,7 @@ def docker_image_format(image_hash): cmd = popen_docker(['inspect', '--format={{.Id}}', image_hash], stdout=subprocess.PIPE) try: - image_id = next(cmd.stdout).strip() + image_id = next(cmd.stdout).decode().strip() if image_id.startswith('sha256:'): return 'v2' elif ':' not in image_id: diff --git a/sdk/python/arvados/commands/put.py b/sdk/python/arvados/commands/put.py index ef86fef679..96b6090734 100644 --- a/sdk/python/arvados/commands/put.py +++ b/sdk/python/arvados/commands/put.py @@ -281,13 +281,13 @@ class ResumeCache(object): @classmethod def make_path(cls, args): md5 = hashlib.md5() - md5.update(arvados.config.get('ARVADOS_API_HOST', '!nohost')) + md5.update(arvados.config.get('ARVADOS_API_HOST', '!nohost').encode()) realpaths = sorted(os.path.realpath(path) for path in args.paths) - md5.update('\0'.join(realpaths)) + md5.update(b'\0'.join([p.encode() for p in realpaths])) if any(os.path.isdir(path) for path in realpaths): - md5.update("-1") + md5.update(b'-1') elif args.filename: - md5.update(args.filename) + md5.update(args.filename.encode()) return os.path.join( arv_cmd.make_home_conf_dir(cls.CACHE_DIR, 0o700, 'raise'), md5.hexdigest()) @@ -667,11 +667,11 @@ class ArvPutUploadJob(object): if self.use_cache: # Set up cache file name from input paths. md5 = hashlib.md5() - md5.update(arvados.config.get('ARVADOS_API_HOST', '!nohost')) + md5.update(arvados.config.get('ARVADOS_API_HOST', '!nohost').encode()) realpaths = sorted(os.path.realpath(path) for path in self.paths) - md5.update('\0'.join(realpaths)) + md5.update(b'\0'.join([p.encode() for p in realpaths])) if self.filename: - md5.update(self.filename) + md5.update(self.filename.encode()) cache_filename = md5.hexdigest() cache_filepath = os.path.join( arv_cmd.make_home_conf_dir(self.CACHE_DIR, 0o700, 'raise'), @@ -754,8 +754,8 @@ class ArvPutUploadJob(object): def portable_data_hash(self): pdh = self._my_collection().portable_data_hash() - m = self._my_collection().stripped_manifest() - local_pdh = hashlib.md5(m).hexdigest() + '+' + str(len(m)) + m = self._my_collection().stripped_manifest().encode() + local_pdh = '{}+{}'.format(hashlib.md5(m).hexdigest(), len(m)) if pdh != local_pdh: logger.warning("\n".join([ "arv-put: API server provided PDH differs from local manifest.", diff --git a/sdk/python/arvados/errors.py b/sdk/python/arvados/errors.py index c9eda2d1c9..feb6660a77 100644 --- a/sdk/python/arvados/errors.py +++ b/sdk/python/arvados/errors.py @@ -8,7 +8,7 @@ from collections import OrderedDict class ApiError(apiclient_errors.HttpError): def _get_reason(self): try: - return '; '.join(json.loads(self.content)['errors']) + return '; '.join(json.loads(self.content.decode('utf-8'))['errors']) except (KeyError, TypeError, ValueError): return super(ApiError, self)._get_reason() diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py index ee91491efa..7a9b5bfbee 100644 --- a/sdk/python/arvados/keep.py +++ b/sdk/python/arvados/keep.py @@ -90,7 +90,7 @@ class KeepLocator(object): return getattr(self, data_name) def setter(self, hex_str): if not arvados.util.is_hex(hex_str, length): - raise ValueError("{} is not a {}-digit hex string: {}". + raise ValueError("{} is not a {}-digit hex string: {!r}". format(name, length, hex_str)) setattr(self, data_name, hex_str) return property(getter, setter) @@ -442,7 +442,7 @@ class KeepClient(object): raise arvados.errors.HttpError(0, str(e)) self._result = { 'status_code': curl.getinfo(pycurl.RESPONSE_CODE), - 'body': response_body.getvalue(), + 'body': response_body.getvalue().decode('utf-8'), 'headers': self._headers, 'error': False, } @@ -851,7 +851,7 @@ class KeepClient(object): The weight is md5(h + u) where u is the last 15 characters of the service endpoint's UUID. """ - return hashlib.md5(data_hash + service_uuid[-15:]).hexdigest() + return hashlib.md5((data_hash + service_uuid[-15:]).encode()).hexdigest() def weighted_service_roots(self, locator, force_rebuild=False, need_writable=False): """Return an array of Keep service endpoints, in the order in @@ -1141,7 +1141,7 @@ class KeepClient(object): """ md5 = hashlib.md5(data).hexdigest() locator = '%s+%d' % (md5, len(data)) - with open(os.path.join(self.local_store, md5 + '.tmp'), 'w') as f: + with open(os.path.join(self.local_store, md5 + '.tmp'), 'wb') as f: f.write(data) os.rename(os.path.join(self.local_store, md5 + '.tmp'), os.path.join(self.local_store, md5)) @@ -1155,8 +1155,8 @@ class KeepClient(object): raise arvados.errors.NotFoundError( "Invalid data locator: '%s'" % loc_s) if locator.md5sum == config.EMPTY_BLOCK_LOCATOR.split('+')[0]: - return '' - with open(os.path.join(self.local_store, locator.md5sum), 'r') as f: + return b'' + with open(os.path.join(self.local_store, locator.md5sum), 'rb') as f: return f.read() def is_cached(self, locator): diff --git a/sdk/python/arvados/stream.py b/sdk/python/arvados/stream.py index 59558162b4..1fe5d35f7e 100644 --- a/sdk/python/arvados/stream.py +++ b/sdk/python/arvados/stream.py @@ -80,13 +80,13 @@ class StreamReader(object): def readfrom(self, start, size, num_retries=None): """Read up to 'size' bytes from the stream, starting at 'start'""" if size == 0: - return '' + return b'' if self._keep is None: self._keep = KeepClient(num_retries=self.num_retries) data = [] for lr in locators_and_ranges(self._data_locators, start, size): data.append(self._keepget(lr.locator, num_retries=num_retries)[lr.segment_offset:lr.segment_offset+lr.segment_size]) - return ''.join(data) + return b''.join(data) def manifest_text(self, strip=False): manifest_text = [self.name().replace(' ', '\\040')] diff --git a/sdk/python/tests/arvados_testutil.py b/sdk/python/tests/arvados_testutil.py index 5992371902..23f00b6415 100644 --- a/sdk/python/tests/arvados_testutil.py +++ b/sdk/python/tests/arvados_testutil.py @@ -21,6 +21,11 @@ import sys import tempfile import unittest +if sys.version_info >= (3, 0): + from io import StringIO +else: + from cStringIO import StringIO + # Use this hostname when you want to make sure the traffic will be # instantly refused. 100::/64 is a dedicated black hole. TEST_HOST = '100::' @@ -47,33 +52,55 @@ def fake_httplib2_response(code, **headers): return httplib2.Response(headers) def mock_responses(body, *codes, **headers): + if not isinstance(body, bytes) and hasattr(body, 'encode'): + body = body.encode() return mock.patch('httplib2.Http.request', side_effect=queue_with(( (fake_httplib2_response(code, **headers), body) for code in codes))) def mock_api_responses(api_client, body, codes, headers={}): + if not isinstance(body, bytes) and hasattr(body, 'encode'): + body = body.encode() return mock.patch.object(api_client._http, 'request', side_effect=queue_with(( (fake_httplib2_response(code, **headers), body) for code in codes))) def str_keep_locator(s): - return '{}+{}'.format(hashlib.md5(s).hexdigest(), len(s)) + return '{}+{}'.format(hashlib.md5(s if isinstance(s, bytes) else s.encode()).hexdigest(), len(s)) @contextlib.contextmanager def redirected_streams(stdout=None, stderr=None): + if stdout == StringIO: + stdout = StringIO() + if stderr == StringIO: + stderr = StringIO() orig_stdout, sys.stdout = sys.stdout, stdout or sys.stdout orig_stderr, sys.stderr = sys.stderr, stderr or sys.stderr try: - yield + yield (stdout, stderr) finally: sys.stdout = orig_stdout sys.stderr = orig_stderr +class VersionChecker(object): + def assertVersionOutput(self, out, err): + if sys.version_info >= (3, 0): + self.assertEqual(err.getvalue(), '') + v = out.getvalue() + else: + # Python 2 writes version info on stderr. + self.assertEqual(out.getvalue(), '') + v = err.getvalue() + self.assertRegexpMatches(v, "[0-9]+\.[0-9]+\.[0-9]+$\n") + + class FakeCurl(object): @classmethod - def make(cls, code, body='', headers={}): + def make(cls, code, body=b'', headers={}): + if not isinstance(body, bytes) and hasattr(body, 'encode'): + body = body.encode() return mock.Mock(spec=cls, wraps=cls(code, body, headers)) - def __init__(self, code=200, body='', headers={}): + def __init__(self, code=200, body=b'', headers={}): self._opt = {} self._got_url = None self._writer = None @@ -146,7 +173,9 @@ def mock_keep_responses(body, *codes, **headers): class MockStreamReader(object): def __init__(self, name='.', *data): self._name = name - self._data = ''.join(data) + self._data = b''.join([ + b if isinstance(b, bytes) else b.encode() + for b in data]) self._data_locators = [str_keep_locator(d) for d in data] self.num_retries = 0 @@ -190,7 +219,7 @@ class ApiClientMock(object): mock_method.return_value = body else: mock_method.side_effect = arvados.errors.ApiError( - fake_httplib2_response(code), "{}") + fake_httplib2_response(code), b"{}") class ArvadosBaseTestCase(unittest.TestCase): @@ -227,7 +256,7 @@ class ArvadosBaseTestCase(unittest.TestCase): tmpfile.write(leaf) return tree_root - def make_test_file(self, text="test"): + def make_test_file(self, text=b"test"): testfile = tempfile.NamedTemporaryFile() testfile.write(text) testfile.flush() diff --git a/sdk/python/tests/keepstub.py b/sdk/python/tests/keepstub.py index 28bd483b40..0bb074cec3 100644 --- a/sdk/python/tests/keepstub.py +++ b/sdk/python/tests/keepstub.py @@ -7,6 +7,7 @@ import hashlib import os import re import socketserver +import sys import time class Server(socketserver.ThreadingMixIn, http.server.HTTPServer, object): @@ -88,7 +89,7 @@ class Handler(http.server.BaseHTTPRequestHandler, object): return self.rfile.read(bytes_to_read) else: BYTES_PER_READ = int(self.server.bandwidth/4) or 32768 - data = '' + data = b'' outage_happened = False bytes_read = 0 target_time = time.time() @@ -139,15 +140,21 @@ class Handler(http.server.BaseHTTPRequestHandler, object): self.end_headers() self.server._do_delay('response_close') - def do_PUT(self): + def handle_expect_100(self): self.server._do_delay('request_body') - # The comments at https://bugs.python.org/issue1491 implies that Python - # 2.7 BaseHTTPRequestHandler was patched to support 100 Continue, but - # reading the actual code that ships in Debian it clearly is not, so we - # need to send the response on the socket directly. - self.wfile_bandwidth_write("%s %d %s\r\n\r\n" % - (self.protocol_version, 100, "Continue")) - data = self.rfile_bandwidth_read(int(self.headers.getheader('content-length'))) + + def do_PUT(self): + if sys.version_info < (3, 0): + # The comments at https://bugs.python.org/issue1491 + # implies that Python 2.7 BaseHTTPRequestHandler was + # patched to support 100 Continue, but reading the actual + # code that ships in Debian it clearly is not, so we need + # to send the response on the socket directly. + self.server._do_delay('request_body') + self.wfile.write("{} {} {}\r\n\r\n".format( + self.protocol_version, 100, "Continue")) + data = self.rfile_bandwidth_read( + int(self.headers.get('content-length'))) datahash = hashlib.md5(data).hexdigest() self.server.store[datahash] = data self.server._do_delay('response') diff --git a/sdk/python/tests/test_api.py b/sdk/python/tests/test_api.py index a2dcaa0b2a..7eefd6205f 100644 --- a/sdk/python/tests/test_api.py +++ b/sdk/python/tests/test_api.py @@ -32,7 +32,7 @@ class ArvadosApiTest(run_test_server.TestCaseWithServers): def api_error_response(self, code, *errors): return (fake_httplib2_response(code, **self.ERROR_HEADERS), json.dumps({'errors': errors, - 'error_token': '1234567890+12345678'})) + 'error_token': '1234567890+12345678'}).encode()) def test_new_api_objects_with_cache(self): clients = [arvados.api('v1', cache=True) for index in [0, 1]] @@ -84,7 +84,7 @@ class ArvadosApiTest(run_test_server.TestCaseWithServers): mock_responses = { 'arvados.humans.delete': ( fake_httplib2_response(500, **self.ERROR_HEADERS), - "") + b"") } req_builder = apiclient_http.RequestMockBuilder(mock_responses) api = arvados.api('v1', requestBuilder=req_builder) @@ -101,9 +101,13 @@ class ArvadosApiTest(run_test_server.TestCaseWithServers): def test_ordered_json_model(self): mock_responses = { - 'arvados.humans.get': (None, json.dumps(collections.OrderedDict( - (c, int(c, 16)) for c in string.hexdigits))), - } + 'arvados.humans.get': ( + None, + json.dumps(collections.OrderedDict( + (c, int(c, 16)) for c in string.hexdigits + )).encode(), + ), + } req_builder = apiclient_http.RequestMockBuilder(mock_responses) api = arvados.api('v1', requestBuilder=req_builder, model=OrderedJsonModel()) diff --git a/sdk/python/tests/test_arv_copy.py b/sdk/python/tests/test_arv_copy.py index c8eb5d14f8..cc207c22ac 100644 --- a/sdk/python/tests/test_arv_copy.py +++ b/sdk/python/tests/test_arv_copy.py @@ -2,7 +2,6 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import -import io import os import sys import tempfile @@ -11,7 +10,7 @@ import unittest import arvados.commands.arv_copy as arv_copy from . import arvados_testutil as tutil -class ArvCopyTestCase(unittest.TestCase): +class ArvCopyTestCase(unittest.TestCase, tutil.VersionChecker): def run_copy(self, args): sys.argv = ['arv-copy'] + args return arv_copy.main() @@ -21,10 +20,8 @@ class ArvCopyTestCase(unittest.TestCase): self.run_copy(['-x=unknown']) def test_version_argument(self): - err = io.BytesIO() - out = io.BytesIO() - with tutil.redirected_streams(stdout=out, stderr=err): + with tutil.redirected_streams( + stdout=tutil.StringIO, stderr=tutil.StringIO) as (out, err): with self.assertRaises(SystemExit): self.run_copy(['--version']) - self.assertEqual(out.getvalue(), '') - self.assertRegexpMatches(err.getvalue(), "[0-9]+\.[0-9]+\.[0-9]+") + self.assertVersionOutput(out, err) diff --git a/sdk/python/tests/test_arv_keepdocker.py b/sdk/python/tests/test_arv_keepdocker.py index 2257227c74..16348031ef 100644 --- a/sdk/python/tests/test_arv_keepdocker.py +++ b/sdk/python/tests/test_arv_keepdocker.py @@ -4,7 +4,6 @@ from __future__ import absolute_import import arvados import hashlib -import io import mock import os import subprocess @@ -22,7 +21,7 @@ class StopTest(Exception): pass -class ArvKeepdockerTestCase(unittest.TestCase): +class ArvKeepdockerTestCase(unittest.TestCase, tutil.VersionChecker): def run_arv_keepdocker(self, args, err): sys.argv = ['arv-keepdocker'] + args log_handler = logging.StreamHandler(err) @@ -37,21 +36,19 @@ class ArvKeepdockerTestCase(unittest.TestCase): self.run_arv_keepdocker(['-x=unknown'], sys.stderr) def test_version_argument(self): - err = io.BytesIO() - out = io.BytesIO() - with tutil.redirected_streams(stdout=out, stderr=err): + with tutil.redirected_streams( + stdout=tutil.StringIO, stderr=tutil.StringIO) as (out, err): with self.assertRaises(SystemExit): self.run_arv_keepdocker(['--version'], sys.stderr) - self.assertEqual(out.getvalue(), '') - self.assertRegexpMatches(err.getvalue(), "[0-9]+\.[0-9]+\.[0-9]+") + self.assertVersionOutput(out, err) @mock.patch('arvados.commands.keepdocker.find_image_hashes', return_value=['abc123']) @mock.patch('arvados.commands.keepdocker.find_one_image_hash', return_value='abc123') def test_image_format_compatibility(self, _1, _2): - old_id = hashlib.sha256('old').hexdigest() - new_id = 'sha256:'+hashlib.sha256('new').hexdigest() + old_id = hashlib.sha256(b'old').hexdigest() + new_id = 'sha256:'+hashlib.sha256(b'new').hexdigest() for supported, img_id, expect_ok in [ (['v1'], old_id, True), (['v1'], new_id, False), @@ -68,8 +65,8 @@ class ArvKeepdockerTestCase(unittest.TestCase): else: fakeDD['dockerImageFormats'] = supported - err = io.BytesIO() - out = io.BytesIO() + err = tutil.StringIO() + out = tutil.StringIO() with tutil.redirected_streams(stdout=out), \ mock.patch('arvados.api') as api, \ @@ -101,8 +98,8 @@ class ArvKeepdockerTestCase(unittest.TestCase): fakeDD = arvados.api('v1')._rootDesc fakeDD['dockerImageFormats'] = ['v1'] - err = io.BytesIO() - out = io.BytesIO() + err = tutil.StringIO() + out = tutil.StringIO() with tutil.redirected_streams(stdout=out), \ mock.patch('arvados.api') as api, \ mock.patch('arvados.commands.keepdocker.popen_docker', diff --git a/sdk/python/tests/test_arv_ls.py b/sdk/python/tests/test_arv_ls.py index ae26ae79ce..6bb089b45b 100644 --- a/sdk/python/tests/test_arv_ls.py +++ b/sdk/python/tests/test_arv_ls.py @@ -4,7 +4,6 @@ from __future__ import absolute_import from builtins import str from builtins import range -import io import os import random import sys @@ -15,9 +14,10 @@ import arvados.errors as arv_error import arvados.commands.ls as arv_ls from . import run_test_server -from .arvados_testutil import str_keep_locator, redirected_streams +from . import arvados_testutil as tutil +from .arvados_testutil import str_keep_locator, redirected_streams, StringIO -class ArvLsTestCase(run_test_server.TestCaseWithServers): +class ArvLsTestCase(run_test_server.TestCaseWithServers, tutil.VersionChecker): FAKE_UUID = 'zzzzz-4zz18-12345abcde12345' def newline_join(self, seq): @@ -39,8 +39,8 @@ class ArvLsTestCase(run_test_server.TestCaseWithServers): return coll_info, api_client def run_ls(self, args, api_client): - self.stdout = io.BytesIO() - self.stderr = io.BytesIO() + self.stdout = StringIO() + self.stderr = StringIO() return arv_ls.main(args, self.stdout, self.stderr, api_client) def test_plain_listing(self): @@ -85,10 +85,7 @@ class ArvLsTestCase(run_test_server.TestCaseWithServers): self.assertNotEqual('', self.stderr.getvalue()) def test_version_argument(self): - err = io.BytesIO() - out = io.BytesIO() - with redirected_streams(stdout=out, stderr=err): + with redirected_streams(stdout=StringIO, stderr=StringIO) as (out, err): with self.assertRaises(SystemExit): self.run_ls(['--version'], None) - self.assertEqual(out.getvalue(), '') - self.assertRegexpMatches(err.getvalue(), "[0-9]+\.[0-9]+\.[0-9]+") + self.assertVersionOutput(out, err) diff --git a/sdk/python/tests/test_arv_normalize.py b/sdk/python/tests/test_arv_normalize.py index 426d41a807..b0272f403d 100644 --- a/sdk/python/tests/test_arv_normalize.py +++ b/sdk/python/tests/test_arv_normalize.py @@ -6,22 +6,28 @@ import sys import tempfile import unittest +from . import arvados_testutil as tutil -class ArvNormalizeTestCase(unittest.TestCase): + +class ArvNormalizeTestCase(unittest.TestCase, tutil.VersionChecker): def run_arv_normalize(self, args=[]): p = subprocess.Popen([sys.executable, 'bin/arv-normalize'] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (stdout, stderr) = p.communicate() - return p.returncode, stdout, stderr + out, err = p.communicate() + sys.stdout.write(out.decode()) + sys.stderr.write(err.decode()) + return p.returncode def test_unsupported_arg(self): - returncode, out, err = self.run_arv_normalize(['-x=unknown']) + with tutil.redirected_streams( + stdout=tutil.StringIO, stderr=tutil.StringIO) as (out, err): + returncode = self.run_arv_normalize(['-x=unknown']) self.assertNotEqual(0, returncode) def test_version_argument(self): - returncode, out, err = self.run_arv_normalize(['--version']) - self.assertEqual(b'', out) - self.assertNotEqual(b'', err) - self.assertRegexpMatches(err.decode(), "^bin/arv-normalize [0-9]+\.[0-9]+\.[0-9]+$") + with tutil.redirected_streams( + stdout=tutil.StringIO, stderr=tutil.StringIO) as (out, err): + returncode = self.run_arv_normalize(['--version']) + self.assertVersionOutput(out, err) self.assertEqual(0, returncode) diff --git a/sdk/python/tests/test_arv_put.py b/sdk/python/tests/test_arv_put.py index 05e5e06b65..a7384030c9 100644 --- a/sdk/python/tests/test_arv_put.py +++ b/sdk/python/tests/test_arv_put.py @@ -8,7 +8,6 @@ standard_library.install_aliases() from builtins import str from builtins import range import apiclient -import io import mock import os import pwd @@ -24,11 +23,6 @@ import threading import hashlib import random -if sys.version_info >= (3, 0): - from io import StringIO -else: - from cStringIO import StringIO - import arvados import arvados.commands.put as arv_put from . import arvados_testutil as tutil @@ -291,7 +285,7 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers, def test_writer_works_with_cache(self): with tempfile.NamedTemporaryFile() as f: - f.write('foo') + f.write(b'foo') f.flush() cwriter = arv_put.ArvPutUploadJob([f.name]) cwriter.start(save_collection=False) @@ -310,7 +304,7 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers, def test_progress_reporting(self): with tempfile.NamedTemporaryFile() as f: - f.write('foo') + f.write(b'foo') f.flush() for expect_count in (None, 8): progression, reporter = self.make_progress_tester() @@ -544,13 +538,15 @@ class ArvadosPutReportTest(ArvadosBaseTestCase): arv_put.human_progress(count, None))) -class ArvadosPutTest(run_test_server.TestCaseWithServers, ArvadosBaseTestCase): +class ArvadosPutTest(run_test_server.TestCaseWithServers, + ArvadosBaseTestCase, + tutil.VersionChecker): MAIN_SERVER = {} Z_UUID = 'zzzzz-zzzzz-zzzzzzzzzzzzzzz' def call_main_with_args(self, args): - self.main_stdout = StringIO() - self.main_stderr = StringIO() + self.main_stdout = tutil.StringIO() + self.main_stderr = tutil.StringIO() return arv_put.main(args, self.main_stdout, self.main_stderr) def call_main_on_test_file(self, args=[]): @@ -575,13 +571,11 @@ class ArvadosPutTest(run_test_server.TestCaseWithServers, ArvadosBaseTestCase): super(ArvadosPutTest, self).tearDown() def test_version_argument(self): - err = io.BytesIO() - out = io.BytesIO() - with tutil.redirected_streams(stdout=out, stderr=err): + with tutil.redirected_streams( + stdout=tutil.StringIO, stderr=tutil.StringIO) as (out, err): with self.assertRaises(SystemExit): self.call_main_with_args(['--version']) - self.assertEqual(out.getvalue(), '') - self.assertRegexpMatches(err.getvalue(), "[0-9]+\.[0-9]+\.[0-9]+") + self.assertVersionOutput(out, err) def test_simple_file_put(self): self.call_main_on_test_file() @@ -651,7 +645,7 @@ class ArvadosPutTest(run_test_server.TestCaseWithServers, ArvadosBaseTestCase): def test_api_error_handling(self): coll_save_mock = mock.Mock(name='arv.collection.Collection().save_new()') coll_save_mock.side_effect = arvados.errors.ApiError( - fake_httplib2_response(403), '{}') + fake_httplib2_response(403), b'{}') with mock.patch('arvados.collection.Collection.save_new', new=coll_save_mock): with self.assertRaises(SystemExit) as exc_test: @@ -719,7 +713,7 @@ class ArvPutIntegrationTest(run_test_server.TestCaseWithServers, result = arv_put.desired_project_uuid(arv_put.api_client, BAD_UUID, 0) except ValueError as error: - self.assertIn(BAD_UUID, error.message) + self.assertIn(BAD_UUID, str(error)) else: self.assertFalse(result, "incorrectly found nonexistent project") @@ -739,7 +733,7 @@ class ArvPutIntegrationTest(run_test_server.TestCaseWithServers, [sys.executable, arv_put.__file__, '--stream'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=self.ENVIRON) - pipe.stdin.write('stdin test\n') + pipe.stdin.write(b'stdin test\n') pipe.stdin.close() deadline = time.time() + 5 while (pipe.poll() is None) and (time.time() < deadline): @@ -751,7 +745,8 @@ class ArvPutIntegrationTest(run_test_server.TestCaseWithServers, elif returncode != 0: sys.stdout.write(pipe.stdout.read()) self.fail("arv-put returned exit code {}".format(returncode)) - self.assertIn('4a9c8b735dce4b5fa3acf221a0b13628+11', pipe.stdout.read()) + self.assertIn('4a9c8b735dce4b5fa3acf221a0b13628+11', + pipe.stdout.read().decode()) def test_ArvPutSignedManifest(self): # ArvPutSignedManifest runs "arv-put foo" and then attempts to get @@ -791,11 +786,12 @@ class ArvPutIntegrationTest(run_test_server.TestCaseWithServers, [sys.executable, arv_put.__file__] + extra_args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.ENVIRON) - stdout, stderr = pipe.communicate(text) + stdout, stderr = pipe.communicate(text.encode()) search_key = ('portable_data_hash' if '--portable-data-hash' in extra_args else 'uuid') collection_list = arvados.api('v1').collections().list( - filters=[[search_key, '=', stdout.strip()]]).execute().get('items', []) + filters=[[search_key, '=', stdout.decode().strip()]] + ).execute().get('items', []) self.assertEqual(1, len(collection_list)) return collection_list[0] diff --git a/sdk/python/tests/test_arv_run.py b/sdk/python/tests/test_arv_run.py index 0f33dfc6c8..88238376c2 100644 --- a/sdk/python/tests/test_arv_run.py +++ b/sdk/python/tests/test_arv_run.py @@ -2,7 +2,6 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import -import io import os import sys import tempfile @@ -11,7 +10,7 @@ import unittest import arvados.commands.run as arv_run from . import arvados_testutil as tutil -class ArvRunTestCase(unittest.TestCase): +class ArvRunTestCase(unittest.TestCase, tutil.VersionChecker): def run_arv_run(self, args): sys.argv = ['arv-run'] + args return arv_run.main() @@ -21,10 +20,8 @@ class ArvRunTestCase(unittest.TestCase): self.run_arv_run(['-x=unknown']) def test_version_argument(self): - err = io.BytesIO() - out = io.BytesIO() - with tutil.redirected_streams(stdout=out, stderr=err): + with tutil.redirected_streams( + stdout=tutil.StringIO, stderr=tutil.StringIO) as (out, err): with self.assertRaises(SystemExit): self.run_arv_run(['--version']) - self.assertEqual(out.getvalue(), '') - self.assertRegexpMatches(err.getvalue(), "[0-9]+\.[0-9]+\.[0-9]+") + self.assertVersionOutput(out, err) diff --git a/sdk/python/tests/test_arv_ws.py b/sdk/python/tests/test_arv_ws.py index 4a0b8238a7..86a21cc009 100644 --- a/sdk/python/tests/test_arv_ws.py +++ b/sdk/python/tests/test_arv_ws.py @@ -1,7 +1,6 @@ #!/usr/bin/env python from __future__ import absolute_import -import io import os import sys import tempfile @@ -11,7 +10,7 @@ import arvados.errors as arv_error import arvados.commands.ws as arv_ws from . import arvados_testutil as tutil -class ArvWsTestCase(unittest.TestCase): +class ArvWsTestCase(unittest.TestCase, tutil.VersionChecker): def run_ws(self, args): return arv_ws.main(args) @@ -20,10 +19,8 @@ class ArvWsTestCase(unittest.TestCase): self.run_ws(['-x=unknown']) def test_version_argument(self): - err = io.BytesIO() - out = io.BytesIO() - with tutil.redirected_streams(stdout=out, stderr=err): + with tutil.redirected_streams( + stdout=tutil.StringIO, stderr=tutil.StringIO) as (out, err): with self.assertRaises(SystemExit): self.run_ws(['--version']) - self.assertEqual(out.getvalue(), '') - self.assertRegexpMatches(err.getvalue(), "[0-9]+\.[0-9]+\.[0-9]+") + self.assertVersionOutput(out, err) diff --git a/sdk/python/tests/test_arvfile.py b/sdk/python/tests/test_arvfile.py index 0b2b7412a5..e3ba0903d0 100644 --- a/sdk/python/tests/test_arvfile.py +++ b/sdk/python/tests/test_arvfile.py @@ -68,27 +68,32 @@ class ArvadosFileWriterTestCase(unittest.TestCase): def test_truncate(self): - keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"}) - api = ArvadosFileWriterTestCase.MockApi({"name":"test_truncate", - "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n", - "replication_desired":None}, - {"uuid":"zzzzz-4zz18-mockcollection0", - "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n", - "portable_data_hash":"7fcd0eaac3aad4c31a6a0e756475da92+52"}) + keep = ArvadosFileWriterTestCase.MockKeep({ + "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789", + }) + api = ArvadosFileWriterTestCase.MockApi({ + "name": "test_truncate", + "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n", + "replication_desired": None, + }, { + "uuid": "zzzzz-4zz18-mockcollection0", + "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n", + "portable_data_hash":"7fcd0eaac3aad4c31a6a0e756475da92+52", + }) with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', api_client=api, keep_client=keep) as c: writer = c.open("count.txt", "r+") self.assertEqual(writer.size(), 10) - self.assertEqual("0123456789", writer.read(12)) + self.assertEqual(b"0123456789", writer.read(12)) writer.truncate(8) # Make sure reading off the end doesn't break - self.assertEqual("", writer.read(12)) + self.assertEqual(b"", writer.read(12)) self.assertEqual(writer.size(), 8) writer.seek(0, os.SEEK_SET) - self.assertEqual("01234567", writer.read(12)) + self.assertEqual(b"01234567", writer.read(12)) self.assertIsNone(c.manifest_locator()) self.assertTrue(c.modified()) @@ -97,27 +102,32 @@ class ArvadosFileWriterTestCase(unittest.TestCase): self.assertFalse(c.modified()) def test_write_to_end(self): - keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"}) - api = ArvadosFileWriterTestCase.MockApi({"name":"test_append", - "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n", - "replication_desired":None}, - {"uuid":"zzzzz-4zz18-mockcollection0", - "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n", - "portable_data_hash":"c5c3af76565c8efb6a806546bcf073f3+88"}) + keep = ArvadosFileWriterTestCase.MockKeep({ + "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789", + }) + api = ArvadosFileWriterTestCase.MockApi({ + "name": "test_append", + "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n", + "replication_desired": None, + }, { + "uuid": "zzzzz-4zz18-mockcollection0", + "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n", + "portable_data_hash": "c5c3af76565c8efb6a806546bcf073f3+88", + }) with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', api_client=api, keep_client=keep) as c: writer = c.open("count.txt", "r+") self.assertEqual(writer.size(), 10) writer.seek(5, os.SEEK_SET) - self.assertEqual("56789", writer.read(8)) + self.assertEqual(b"56789", writer.read(8)) writer.seek(10, os.SEEK_SET) writer.write("foo") self.assertEqual(writer.size(), 13) writer.seek(5, os.SEEK_SET) - self.assertEqual("56789foo", writer.read(8)) + self.assertEqual(b"56789foo", writer.read(8)) self.assertIsNone(c.manifest_locator()) self.assertTrue(c.modified()) @@ -126,35 +136,39 @@ class ArvadosFileWriterTestCase(unittest.TestCase): c.save_new("test_append") self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator()) self.assertFalse(c.modified()) - self.assertEqual("foo", keep.get("acbd18db4cc2f85cedef654fccc4a4d8+3")) + self.assertEqual(b"foo", keep.get("acbd18db4cc2f85cedef654fccc4a4d8+3")) def test_append(self): - keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"}) + keep = ArvadosFileWriterTestCase.MockKeep({ + "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789", + }) c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', keep_client=keep) writer = c.open("count.txt", "a+") - self.assertEqual(writer.read(20), "0123456789") + self.assertEqual(writer.read(20), b"0123456789") writer.seek(0, os.SEEK_SET) writer.write("hello") - self.assertEqual(writer.read(20), "0123456789hello") + self.assertEqual(writer.read(20), b"0123456789hello") writer.seek(0, os.SEEK_SET) writer.write("world") - self.assertEqual(writer.read(20), "0123456789helloworld") + self.assertEqual(writer.read(20), b"0123456789helloworld") self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 fc5e038d38a57032085441e7fe7010b0+10 0:20:count.txt\n", c.portable_manifest_text()) def test_write_at_beginning(self): - keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"}) + keep = ArvadosFileWriterTestCase.MockKeep({ + "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789", + }) with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', keep_client=keep) as c: writer = c.open("count.txt", "r+") - self.assertEqual("0123456789", writer.readfrom(0, 13)) + self.assertEqual(b"0123456789", writer.readfrom(0, 13)) writer.seek(0, os.SEEK_SET) writer.write("foo") self.assertEqual(writer.size(), 10) - self.assertEqual("foo3456789", writer.readfrom(0, 13)) + self.assertEqual(b"foo3456789", writer.readfrom(0, 13)) self.assertEqual(". acbd18db4cc2f85cedef654fccc4a4d8+3 781e5e245d69b566979b86e28d23f2c7+10 0:3:count.txt 6:7:count.txt\n", c.portable_manifest_text()) def test_write_empty(self): @@ -168,7 +182,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase): keep = ArvadosFileWriterTestCase.MockKeep({}) with Collection(keep_client=keep) as c: writer = c.open("count.txt", "w") - writer.write("0123456789") + writer.write(b"0123456789") self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.portable_manifest_text()) self.assertNotIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks) @@ -187,51 +201,51 @@ class ArvadosFileWriterTestCase(unittest.TestCase): def test_write_in_middle(self): - keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"}) + keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": b"0123456789"}) with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', keep_client=keep) as c: writer = c.open("count.txt", "r+") - self.assertEqual("0123456789", writer.readfrom(0, 13)) + self.assertEqual(b"0123456789", writer.readfrom(0, 13)) writer.seek(3, os.SEEK_SET) writer.write("foo") self.assertEqual(writer.size(), 10) - self.assertEqual("012foo6789", writer.readfrom(0, 13)) + self.assertEqual(b"012foo6789", writer.readfrom(0, 13)) self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:count.txt 10:3:count.txt 6:4:count.txt\n", c.portable_manifest_text()) def test_write_at_end(self): - keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"}) + keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": b"0123456789"}) with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', keep_client=keep) as c: writer = c.open("count.txt", "r+") - self.assertEqual("0123456789", writer.readfrom(0, 13)) + self.assertEqual(b"0123456789", writer.readfrom(0, 13)) writer.seek(7, os.SEEK_SET) writer.write("foo") self.assertEqual(writer.size(), 10) - self.assertEqual("0123456foo", writer.readfrom(0, 13)) + self.assertEqual(b"0123456foo", writer.readfrom(0, 13)) self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:7:count.txt 10:3:count.txt\n", c.portable_manifest_text()) def test_write_across_segment_boundary(self): - keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"}) + keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": b"0123456789"}) with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt 0:10:count.txt\n', keep_client=keep) as c: writer = c.open("count.txt", "r+") - self.assertEqual("012345678901234", writer.readfrom(0, 15)) + self.assertEqual(b"012345678901234", writer.readfrom(0, 15)) writer.seek(7, os.SEEK_SET) writer.write("foobar") self.assertEqual(writer.size(), 20) - self.assertEqual("0123456foobar34", writer.readfrom(0, 15)) + self.assertEqual(b"0123456foobar34", writer.readfrom(0, 15)) self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 3858f62230ac3c915f300c664312c63f+6 0:7:count.txt 10:6:count.txt 3:7:count.txt\n", c.portable_manifest_text()) def test_write_across_several_segments(self): - keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"}) + keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": b"0123456789"}) with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:4:count.txt 0:4:count.txt 0:4:count.txt', keep_client=keep) as c: writer = c.open("count.txt", "r+") - self.assertEqual("012301230123", writer.readfrom(0, 15)) + self.assertEqual(b"012301230123", writer.readfrom(0, 15)) writer.seek(2, os.SEEK_SET) writer.write("abcdefg") self.assertEqual(writer.size(), 12) - self.assertEqual("01abcdefg123", writer.readfrom(0, 15)) + self.assertEqual(b"01abcdefg123", writer.readfrom(0, 15)) self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 7ac66c0f148de9519b8bd264312c4d64+7 0:2:count.txt 10:7:count.txt 1:3:count.txt\n", c.portable_manifest_text()) def test_write_large(self): @@ -283,15 +297,17 @@ class ArvadosFileWriterTestCase(unittest.TestCase): writer.write("0123456789") self.assertEqual(writer.size(), 10) - self.assertEqual("0123456789", writer.readfrom(0, 20)) + self.assertEqual(b"0123456789", writer.readfrom(0, 20)) self.assertEqual(". 7a08b07e84641703e5f2c836aa59a170+100 90:10:count.txt\n", c.portable_manifest_text()) writer.flush() self.assertEqual(writer.size(), 10) - self.assertEqual("0123456789", writer.readfrom(0, 20)) + self.assertEqual(b"0123456789", writer.readfrom(0, 20)) self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n", c.portable_manifest_text()) def test_rewrite_append_existing_file(self): - keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"}) + keep = ArvadosFileWriterTestCase.MockKeep({ + "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789", + }) with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt', keep_client=keep) as c: writer = c.open("count.txt", "r+") @@ -300,16 +316,18 @@ class ArvadosFileWriterTestCase(unittest.TestCase): writer.write("abcdefghij") self.assertEqual(writer.size(), 20) - self.assertEqual("0123456789abcdefghij", writer.readfrom(0, 20)) + self.assertEqual(b"0123456789abcdefghij", writer.readfrom(0, 20)) self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 ae5f43bab79cf0be33f025fa97ae7398+100 0:10:count.txt 100:10:count.txt\n", c.portable_manifest_text()) writer.arvadosfile.flush() self.assertEqual(writer.size(), 20) - self.assertEqual("0123456789abcdefghij", writer.readfrom(0, 20)) + self.assertEqual(b"0123456789abcdefghij", writer.readfrom(0, 20)) self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:20:count.txt\n", c.portable_manifest_text()) def test_rewrite_over_existing_file(self): - keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"}) + keep = ArvadosFileWriterTestCase.MockKeep({ + "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789", + }) with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt', keep_client=keep) as c: writer = c.open("count.txt", "r+") @@ -318,13 +336,13 @@ class ArvadosFileWriterTestCase(unittest.TestCase): writer.write("abcdefghij") self.assertEqual(writer.size(), 15) - self.assertEqual("01234abcdefghij", writer.readfrom(0, 20)) + self.assertEqual(b"01234abcdefghij", writer.readfrom(0, 20)) self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 ae5f43bab79cf0be33f025fa97ae7398+100 0:5:count.txt 100:10:count.txt\n", c.portable_manifest_text()) writer.arvadosfile.flush() self.assertEqual(writer.size(), 15) - self.assertEqual("01234abcdefghij", writer.readfrom(0, 20)) + self.assertEqual(b"01234abcdefghij", writer.readfrom(0, 20)) self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:5:count.txt 10:10:count.txt\n", c.portable_manifest_text()) def test_write_large_rewrite(self): @@ -338,7 +356,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase): with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt', api_client=api, keep_client=keep) as c: writer = c.open("count.txt", "r+") - text = ''.join(["0123456789" for a in range(0, 100)]) + text = b''.join([b"0123456789" for a in range(0, 100)]) for b in range(0, 100000): writer.write(text) writer.seek(0, os.SEEK_SET) @@ -353,12 +371,15 @@ class ArvadosFileWriterTestCase(unittest.TestCase): def test_create(self): keep = ArvadosFileWriterTestCase.MockKeep({}) - api = ArvadosFileWriterTestCase.MockApi({"name":"test_create", - "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n", - "replication_desired":None}, - {"uuid":"zzzzz-4zz18-mockcollection0", - "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n", - "portable_data_hash":"7a461a8c58601798f690f8b368ac4423+51"}) + api = ArvadosFileWriterTestCase.MockApi({ + "name":"test_create", + "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n", + "replication_desired":None, + }, { + "uuid":"zzzzz-4zz18-mockcollection0", + "manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n", + "portable_data_hash":"7a461a8c58601798f690f8b368ac4423+51", + }) with Collection(api_client=api, keep_client=keep) as c: writer = c.open("count.txt", "w+") self.assertEqual(writer.size(), 0) @@ -371,7 +392,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase): c.save_new("test_create") self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator()) self.assertFalse(c.modified()) - self.assertEqual("01234567", keep.get("2e9ec317e197819358fbc43afca7d837+8")) + self.assertEqual(b"01234567", keep.get("2e9ec317e197819358fbc43afca7d837+8")) def test_create_subdir(self): @@ -444,7 +465,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase): c.save_new("test_create_multiple") self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator()) self.assertFalse(c.modified()) - self.assertEqual("01234567", keep.get("2e9ec317e197819358fbc43afca7d837+8")) + self.assertEqual(b"01234567", keep.get("2e9ec317e197819358fbc43afca7d837+8")) class ArvadosFileReaderTestCase(StreamFileReaderTestCase): @@ -478,7 +499,7 @@ class ArvadosFileReaderTestCase(StreamFileReaderTestCase): stream = [] n = 0 blocks = {} - for d in ['01234', '34567', '67890']: + for d in [b'01234', b'34567', b'67890']: loc = tutil.str_keep_locator(d) blocks[loc] = d stream.append(Range(loc, n, len(d))) @@ -490,27 +511,30 @@ class ArvadosFileReaderTestCase(StreamFileReaderTestCase): # read() needs to return all the data requested if possible, even if it # crosses uncached blocks: https://arvados.org/issues/5856 sfile = self.make_count_reader(nocache=True) - self.assertEqual('12345678', sfile.read(8)) + self.assertEqual(b'12345678', sfile.read(8)) def test_successive_reads(self): # Override StreamFileReaderTestCase.test_successive_reads sfile = self.make_count_reader(nocache=True) - self.assertEqual('1234', sfile.read(4)) - self.assertEqual('5678', sfile.read(4)) - self.assertEqual('9', sfile.read(4)) - self.assertEqual('', sfile.read(4)) + self.assertEqual(b'1234', sfile.read(4)) + self.assertEqual(b'5678', sfile.read(4)) + self.assertEqual(b'9', sfile.read(4)) + self.assertEqual(b'', sfile.read(4)) def test_tell_after_block_read(self): # Override StreamFileReaderTestCase.test_tell_after_block_read sfile = self.make_count_reader(nocache=True) - self.assertEqual('12345678', sfile.read(8)) + self.assertEqual(b'12345678', sfile.read(8)) self.assertEqual(8, sfile.tell()) def test_prefetch(self): - keep = ArvadosFileWriterTestCase.MockKeep({"2e9ec317e197819358fbc43afca7d837+8": "01234567", "e8dc4081b13434b45189a720b77b6818+8": "abcdefgh"}) + keep = ArvadosFileWriterTestCase.MockKeep({ + "2e9ec317e197819358fbc43afca7d837+8": b"01234567", + "e8dc4081b13434b45189a720b77b6818+8": b"abcdefgh", + }) with Collection(". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:16:count.txt\n", keep_client=keep) as c: r = c.open("count.txt", "r") - self.assertEqual("0123", r.read(4)) + self.assertEqual(b"0123", r.read(4)) self.assertIn("2e9ec317e197819358fbc43afca7d837+8", keep.requests) self.assertIn("e8dc4081b13434b45189a720b77b6818+8", keep.requests) @@ -572,17 +596,17 @@ class ArvadosFileReadFromTestCase(ArvadosFileReadTestCase): class ArvadosFileReadAllTestCase(ArvadosFileReadTestCase): def read_for_test(self, reader, byte_count, **kwargs): - return ''.join(reader.readall(**kwargs)) + return b''.join(reader.readall(**kwargs)) class ArvadosFileReadAllDecompressedTestCase(ArvadosFileReadTestCase): def read_for_test(self, reader, byte_count, **kwargs): - return ''.join(reader.readall_decompressed(**kwargs)) + return b''.join(reader.readall_decompressed(**kwargs)) class ArvadosFileReadlinesTestCase(ArvadosFileReadTestCase): def read_for_test(self, reader, byte_count, **kwargs): - return ''.join(reader.readlines(**kwargs)) + return ''.join(reader.readlines(**kwargs)).encode() class ArvadosFileTestCase(unittest.TestCase): @@ -612,13 +636,13 @@ class BlockManagerTest(unittest.TestCase): bufferblock.append("foo") self.assertEqual(bufferblock.size(), 3) - self.assertEqual(bufferblock.buffer_view[0:3], "foo") + self.assertEqual(bufferblock.buffer_view[0:3], b"foo") self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3") bufferblock.append("bar") self.assertEqual(bufferblock.size(), 6) - self.assertEqual(bufferblock.buffer_view[0:6], "foobar") + self.assertEqual(bufferblock.buffer_view[0:6], b"foobar") self.assertEqual(bufferblock.locator(), "3858f62230ac3c915f300c664312c63f+6") bufferblock.set_state(arvados.arvfile._BufferBlock.PENDING) @@ -632,7 +656,7 @@ class BlockManagerTest(unittest.TestCase): bufferblock.append("foo") self.assertEqual(bufferblock.size(), 3) - self.assertEqual(bufferblock.buffer_view[0:3], "foo") + self.assertEqual(bufferblock.buffer_view[0:3], b"foo") self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3") bufferblock.set_state(arvados.arvfile._BufferBlock.PENDING) @@ -642,21 +666,23 @@ class BlockManagerTest(unittest.TestCase): bufferblock2.append("bar") self.assertEqual(bufferblock2.size(), 6) - self.assertEqual(bufferblock2.buffer_view[0:6], "foobar") + self.assertEqual(bufferblock2.buffer_view[0:6], b"foobar") self.assertEqual(bufferblock2.locator(), "3858f62230ac3c915f300c664312c63f+6") self.assertEqual(bufferblock.size(), 3) - self.assertEqual(bufferblock.buffer_view[0:3], "foo") + self.assertEqual(bufferblock.buffer_view[0:3], b"foo") self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3") def test_bufferblock_get(self): - keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"}) + keep = ArvadosFileWriterTestCase.MockKeep({ + "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789", + }) with arvados.arvfile._BlockManager(keep) as blockmanager: bufferblock = blockmanager.alloc_bufferblock() bufferblock.append("foo") - self.assertEqual(blockmanager.get_block_contents("781e5e245d69b566979b86e28d23f2c7+10", 1), "0123456789") - self.assertEqual(blockmanager.get_block_contents(bufferblock.blockid, 1), "foo") + self.assertEqual(blockmanager.get_block_contents("781e5e245d69b566979b86e28d23f2c7+10", 1), b"0123456789") + self.assertEqual(blockmanager.get_block_contents(bufferblock.blockid, 1), b"foo") def test_bufferblock_commit(self): mockkeep = mock.MagicMock() diff --git a/sdk/python/tests/test_cache.py b/sdk/python/tests/test_cache.py index 1c00551e5c..6adab7bbc1 100644 --- a/sdk/python/tests/test_cache.py +++ b/sdk/python/tests/test_cache.py @@ -34,15 +34,17 @@ class CacheTestThread(threading.Thread): for x in range(16): try: data_in = _random(128) - data_in = hashlib.md5(data_in).hexdigest() + bytes("\n") + bytes(data_in) + data_in = bytes(hashlib.md5(data_in).hexdigest()) + bytes("\n") + bytes(data_in) c.set(url, data_in) data_out = c.get(url) digest, _, content = data_out.partition("\n") - if digest != hashlib.md5(content).hexdigest(): + if digest != bytes(hashlib.md5(content).hexdigest()): self.ok = False - except Exception as err: - self.ok = False - print("cache failed: {}".format(err), file=sys.stderr) + finally: + pass + #except Exception as err: + # self.ok = False + # print("cache failed: {}: {}".format(type(err), err), file=sys.stderr) class CacheTest(unittest.TestCase): diff --git a/sdk/python/tests/test_collections.py b/sdk/python/tests/test_collections.py index 259c5aa177..3a1afcecbb 100644 --- a/sdk/python/tests/test_collections.py +++ b/sdk/python/tests/test_collections.py @@ -42,13 +42,13 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers, self.assertEqual(cw.current_stream_name(), '.', 'current_stream_name() should be "." now') cw.set_current_file_name('foo.txt') - cw.write('foo') + cw.write(b'foo') self.assertEqual(cw.current_file_name(), 'foo.txt', 'current_file_name() should be foo.txt now') cw.start_new_file('bar.txt') - cw.write('bar') + cw.write(b'bar') cw.start_new_stream('baz') - cw.write('baz') + cw.write(b'baz') cw.set_current_file_name('baz.txt') self.assertEqual(cw.manifest_text(), ". 3858f62230ac3c915f300c664312c63f+6 0:3:foo.txt 3:3:bar.txt\n" + @@ -58,8 +58,8 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers, return cw.portable_data_hash() def test_keep_local_store(self): - self.assertEqual(self.keep_client.put('foo'), 'acbd18db4cc2f85cedef654fccc4a4d8+3', 'wrong md5 hash from Keep.put') - self.assertEqual(self.keep_client.get('acbd18db4cc2f85cedef654fccc4a4d8+3'), 'foo', 'wrong data from Keep.get') + self.assertEqual(self.keep_client.put(b'foo'), 'acbd18db4cc2f85cedef654fccc4a4d8+3', 'wrong md5 hash from Keep.put') + self.assertEqual(self.keep_client.get('acbd18db4cc2f85cedef654fccc4a4d8+3'), b'foo', 'wrong data from Keep.get') def test_local_collection_writer(self): self.assertEqual(self.write_foo_bar_baz(), @@ -74,20 +74,20 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers, for s in cr.all_streams(): for f in s.all_files(): got += [[f.size(), f.stream_name(), f.name(), f.read(2**26)]] - expected = [[3, '.', 'foo.txt', 'foo'], - [3, '.', 'bar.txt', 'bar'], - [3, './baz', 'baz.txt', 'baz']] + expected = [[3, '.', 'foo.txt', b'foo'], + [3, '.', 'bar.txt', b'bar'], + [3, './baz', 'baz.txt', b'baz']] self.assertEqual(got, expected) stream0 = cr.all_streams()[0] self.assertEqual(stream0.readfrom(0, 0), - '', + b'', 'reading zero bytes should have returned empty string') self.assertEqual(stream0.readfrom(0, 2**26), - 'foobar', + b'foobar', 'reading entire stream failed') self.assertEqual(stream0.readfrom(2**26, 0), - '', + b'', 'reading zero bytes should have returned empty string') def _test_subset(self, collection, expected): @@ -104,50 +104,50 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers, def test_collection_manifest_subset(self): foobarbaz = self.write_foo_bar_baz() self._test_subset(foobarbaz, - [[3, '.', 'bar.txt', 'bar'], - [3, '.', 'foo.txt', 'foo'], - [3, './baz', 'baz.txt', 'baz']]) + [[3, '.', 'bar.txt', b'bar'], + [3, '.', 'foo.txt', b'foo'], + [3, './baz', 'baz.txt', b'baz']]) self._test_subset((". %s %s 0:3:foo.txt 3:3:bar.txt\n" % - (self.keep_client.put("foo"), - self.keep_client.put("bar"))), - [[3, '.', 'bar.txt', 'bar'], - [3, '.', 'foo.txt', 'foo']]) + (self.keep_client.put(b"foo"), + self.keep_client.put(b"bar"))), + [[3, '.', 'bar.txt', b'bar'], + [3, '.', 'foo.txt', b'foo']]) self._test_subset((". %s %s 0:2:fo.txt 2:4:obar.txt\n" % - (self.keep_client.put("foo"), - self.keep_client.put("bar"))), - [[2, '.', 'fo.txt', 'fo'], - [4, '.', 'obar.txt', 'obar']]) + (self.keep_client.put(b"foo"), + self.keep_client.put(b"bar"))), + [[2, '.', 'fo.txt', b'fo'], + [4, '.', 'obar.txt', b'obar']]) self._test_subset((". %s %s 0:2:fo.txt 2:0:zero.txt 2:2:ob.txt 4:2:ar.txt\n" % - (self.keep_client.put("foo"), - self.keep_client.put("bar"))), - [[2, '.', 'ar.txt', 'ar'], - [2, '.', 'fo.txt', 'fo'], - [2, '.', 'ob.txt', 'ob'], - [0, '.', 'zero.txt', '']]) + (self.keep_client.put(b"foo"), + self.keep_client.put(b"bar"))), + [[2, '.', 'ar.txt', b'ar'], + [2, '.', 'fo.txt', b'fo'], + [2, '.', 'ob.txt', b'ob'], + [0, '.', 'zero.txt', b'']]) def test_collection_empty_file(self): cw = arvados.CollectionWriter(self.api_client) cw.start_new_file('zero.txt') - cw.write('') + cw.write(b'') self.assertEqual(cw.manifest_text(), ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:zero.txt\n") self.check_manifest_file_sizes(cw.manifest_text(), [0]) cw = arvados.CollectionWriter(self.api_client) cw.start_new_file('zero.txt') - cw.write('') + cw.write(b'') cw.start_new_file('one.txt') - cw.write('1') + cw.write(b'1') cw.start_new_stream('foo') cw.start_new_file('zero.txt') - cw.write('') + cw.write(b'') self.check_manifest_file_sizes(cw.manifest_text(), [0,1,0]) def test_no_implicit_normalize(self): cw = arvados.CollectionWriter(self.api_client) cw.start_new_file('b') - cw.write('b') + cw.write(b'b') cw.start_new_file('a') - cw.write('') + cw.write(b'') self.check_manifest_file_sizes(cw.manifest_text(), [1,0]) self.check_manifest_file_sizes( arvados.CollectionReader( @@ -313,14 +313,16 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers, return self.content[locator] def test_stream_reader(self): - keepblocks = {'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa+10': 'abcdefghij', - 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb+15': 'klmnopqrstuvwxy', - 'cccccccccccccccccccccccccccccccc+5': 'z0123'} + keepblocks = { + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa+10': b'abcdefghij', + 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb+15': b'klmnopqrstuvwxy', + 'cccccccccccccccccccccccccccccccc+5': b'z0123', + } mk = self.MockKeep(keepblocks) sr = arvados.StreamReader([".", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa+10", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb+15", "cccccccccccccccccccccccccccccccc+5", "0:30:foo"], mk) - content = 'abcdefghijklmnopqrstuvwxyz0123456789' + content = b'abcdefghijklmnopqrstuvwxyz0123456789' self.assertEqual(sr.readfrom(0, 30), content[0:30]) self.assertEqual(sr.readfrom(2, 30), content[2:30]) @@ -334,7 +336,7 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers, self.assertEqual(sr.readfrom(15, 5), content[15:20]) self.assertEqual(sr.readfrom(20, 5), content[20:25]) self.assertEqual(sr.readfrom(25, 5), content[25:30]) - self.assertEqual(sr.readfrom(30, 5), '') + self.assertEqual(sr.readfrom(30, 5), b'') def test_extract_file(self): m1 = """. 5348b82a029fd9e971a811ce1f71360b+43 0:43:md5sum.txt @@ -423,7 +425,7 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers, def test_write_multiple_files(self): cwriter = arvados.CollectionWriter(self.api_client) for letter in 'ABC': - with self.make_test_file(letter) as testfile: + with self.make_test_file(letter.encode()) as testfile: cwriter.write_file(testfile.name, letter) self.assertEqual( cwriter.manifest_text(), @@ -466,7 +468,7 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers, with self.make_test_file() as testfile: cwriter.write_file(testfile.name, 'test') orig_mtime = os.fstat(testfile.fileno()).st_mtime - testfile.write('extra') + testfile.write(b'extra') testfile.flush() os.utime(testfile.name, (orig_mtime, orig_mtime)) self.assertRaises(arvados.errors.StaleWriterStateError, @@ -592,8 +594,8 @@ class CollectionReaderTestCase(unittest.TestCase, CollectionTestMixin): reader = arvados.CollectionReader(self.DEFAULT_UUID, api_client=client, num_retries=3) with tutil.mock_keep_responses('foo', 500, 500, 200): - self.assertEqual('foo', - ''.join(f.read(9) for f in reader.all_files())) + self.assertEqual(b'foo', + b''.join(f.read(9) for f in reader.all_files())) def test_read_nonnormalized_manifest_with_collection_reader(self): # client should be able to use CollectionReader on a manifest without normalizing it @@ -680,7 +682,7 @@ class CollectionWriterTestCase(unittest.TestCase, CollectionTestMixin): kwargs.setdefault('api_client', self.api_client_mock()) writer = arvados.CollectionWriter(**kwargs) writer.start_new_file('foo') - writer.write('foo') + writer.write(b'foo') return writer def test_write_whole_collection(self): @@ -739,7 +741,7 @@ class CollectionWriterTestCase(unittest.TestCase, CollectionTestMixin): with writer.open('out') as out_file: self.assertEqual('.', writer.current_stream_name()) self.assertEqual('out', writer.current_file_name()) - out_file.write('test data') + out_file.write(b'test data') data_loc = tutil.str_keep_locator('test data') self.assertTrue(out_file.closed, "writer file not closed after context") self.assertRaises(ValueError, out_file.write, 'extra text') @@ -764,9 +766,9 @@ class CollectionWriterTestCase(unittest.TestCase, CollectionTestMixin): with self.mock_keep((data_loc1, 200), (data_loc2, 200)) as keep_mock: writer = arvados.CollectionWriter(client) with writer.open('flush_test') as out_file: - out_file.write('flush1') + out_file.write(b'flush1') out_file.flush() - out_file.write('flush2') + out_file.write(b'flush2') self.assertEqual(". {} {} 0:12:flush_test\n".format(data_loc1, data_loc2), writer.manifest_text()) @@ -775,9 +777,9 @@ class CollectionWriterTestCase(unittest.TestCase, CollectionTestMixin): client = self.api_client_mock() writer = arvados.CollectionWriter(client) with writer.open('.', '1') as out_file: - out_file.write('1st') + out_file.write(b'1st') with writer.open('.', '2') as out_file: - out_file.write('2nd') + out_file.write(b'2nd') data_loc = tutil.str_keep_locator('1st2nd') with self.mock_keep(data_loc, 200) as keep_mock: self.assertEqual(". {} 0:3:1 3:3:2\n".format(data_loc), @@ -790,9 +792,9 @@ class CollectionWriterTestCase(unittest.TestCase, CollectionTestMixin): with self.mock_keep((data_loc1, 200), (data_loc2, 200)) as keep_mock: writer = arvados.CollectionWriter(client) with writer.open('file') as out_file: - out_file.write('file') + out_file.write(b'file') with writer.open('./dir', 'indir') as out_file: - out_file.write('indir') + out_file.write(b'indir') expected = ". {} 0:4:file\n./dir {} 0:5:indir\n".format( data_loc1, data_loc2) self.assertEqual(expected, writer.manifest_text()) @@ -1030,7 +1032,7 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin): self.assertEqual(d, [('del', './count1.txt', c1["count1.txt"]), ('add', './count2.txt', c2["count2.txt"])]) f = c1.open("count1.txt", "w") - f.write("zzzzz") + f.write(b"zzzzz") # c1 changed, so it should not be deleted. c1.apply(d) @@ -1042,7 +1044,7 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin): d = c1.diff(c2) self.assertEqual(d, [('mod', './count1.txt', c1["count1.txt"], c2["count1.txt"])]) f = c1.open("count1.txt", "w") - f.write("zzzzz") + f.write(b"zzzzz") # c1 changed, so c2 mod will go to a conflict file c1.apply(d) @@ -1055,7 +1057,7 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin): self.assertEqual(d, [('del', './count2.txt', c1["count2.txt"]), ('add', './count1.txt', c2["count1.txt"])]) f = c1.open("count1.txt", "w") - f.write("zzzzz") + f.write(b"zzzzz") # c1 added count1.txt, so c2 add will go to a conflict file c1.apply(d) @@ -1092,9 +1094,9 @@ class NewCollectionTestCaseWithServers(run_test_server.TestCaseWithServers): with c.open("count.txt", "w") as f: # One file committed with c.open("foo.txt", "w") as foo: - foo.write("foo") + foo.write(b"foo") foo.flush() # Force block commit - f.write("0123456789") + f.write(b"0123456789") # Other file not committed. Block not written to keep yet. self.assertEqual( c._get_manifest_text(".", @@ -1115,14 +1117,14 @@ class NewCollectionTestCaseWithServers(run_test_server.TestCaseWithServers): c = Collection() # Write a couple of small files, f = c.open("count.txt", "w") - f.write("0123456789") + f.write(b"0123456789") f.close(flush=False) foo = c.open("foo.txt", "w") - foo.write("foo") + foo.write(b"foo") foo.close(flush=False) # Then, write a big file, it shouldn't be packed with the ones above big = c.open("bigfile.txt", "w") - big.write("x" * 1024 * 1024 * 33) # 33 MB > KEEP_BLOCK_SIZE/2 + big.write(b"x" * 1024 * 1024 * 33) # 33 MB > KEEP_BLOCK_SIZE/2 big.close(flush=False) self.assertEqual( c.manifest_text("."), @@ -1143,7 +1145,7 @@ class CollectionCreateUpdateTest(run_test_server.TestCaseWithServers): self.assertEqual(c.api_response()["portable_data_hash"], "d41d8cd98f00b204e9800998ecf8427e+0" ) with c.open("count.txt", "w") as f: - f.write("0123456789") + f.write(b"0123456789") self.assertEqual(c.portable_manifest_text(), ". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n") @@ -1165,7 +1167,7 @@ class CollectionCreateUpdateTest(run_test_server.TestCaseWithServers): c2 = Collection(c1.manifest_locator()) with c2.open("count.txt", "w") as f: - f.write("abcdefg") + f.write(b"abcdefg") diff = c1.diff(c2) @@ -1193,7 +1195,7 @@ class CollectionCreateUpdateTest(run_test_server.TestCaseWithServers): c2 = arvados.collection.Collection(c1.manifest_locator()) with c2.open("count.txt", "w") as f: - f.write("abcdefg") + f.write(b"abcdefg") c2.save() @@ -1207,11 +1209,11 @@ class CollectionCreateUpdateTest(run_test_server.TestCaseWithServers): c1.save() with c1.open("count.txt", "w") as f: - f.write("XYZ") + f.write(b"XYZ") c2 = arvados.collection.Collection(c1.manifest_locator()) with c2.open("count.txt", "w") as f: - f.write("abcdefg") + f.write(b"abcdefg") c2.save() diff --git a/sdk/python/tests/test_events.py b/sdk/python/tests/test_events.py index 4596b6cdfb..84b586faab 100644 --- a/sdk/python/tests/test_events.py +++ b/sdk/python/tests/test_events.py @@ -5,24 +5,25 @@ from future import standard_library standard_library.install_aliases() from builtins import range from builtins import object -import arvados -import io import logging import mock import queue -from . import run_test_server +import sys import threading import time import unittest -from . import arvados_testutil +import arvados +from . import arvados_testutil as tutil +from . import run_test_server + class WebsocketTest(run_test_server.TestCaseWithServers): MAIN_SERVER = {} TIME_PAST = time.time()-3600 TIME_FUTURE = time.time()+3600 - MOCK_WS_URL = 'wss://[{}]/'.format(arvados_testutil.TEST_HOST) + MOCK_WS_URL = 'wss://[{}]/'.format(tutil.TEST_HOST) TEST_TIMEOUT = 10.0 @@ -96,10 +97,12 @@ class WebsocketTest(run_test_server.TestCaseWithServers): error_mock = mock.MagicMock() error_mock.resp.status = 0 error_mock._get_reason.return_value = "testing" - api_mock.logs().list().execute.side_effect = (arvados.errors.ApiError(error_mock, ""), - {"items": [{"id": 1}], "items_available": 1}, - arvados.errors.ApiError(error_mock, ""), - {"items": [{"id": 1}], "items_available": 1}) + api_mock.logs().list().execute.side_effect = ( + arvados.errors.ApiError(error_mock, b""), + {"items": [{"id": 1}], "items_available": 1}, + arvados.errors.ApiError(error_mock, b""), + {"items": [{"id": 1}], "items_available": 1}, + ) pc = arvados.events.PollClient(api_mock, [], on_ev, 15, None) pc.start() while len(n) < 2: @@ -161,7 +164,7 @@ class WebsocketTest(run_test_server.TestCaseWithServers): run_test_server.authorize_with('active') events = queue.Queue(100) - logstream = io.BytesIO() + logstream = tutil.StringIO() rootLogger = logging.getLogger() streamHandler = logging.StreamHandler(logstream) rootLogger.addHandler(streamHandler) @@ -229,7 +232,7 @@ class WebsocketTest(run_test_server.TestCaseWithServers): def test_websocket_reconnect_retry(self, event_client_connect): event_client_connect.side_effect = [None, Exception('EventClient.connect error'), None] - logstream = io.BytesIO() + logstream = tutil.StringIO() rootLogger = logging.getLogger() streamHandler = logging.StreamHandler(logstream) rootLogger.addHandler(streamHandler) diff --git a/sdk/python/tests/test_keep_client.py b/sdk/python/tests/test_keep_client.py index 1b0b627a65..5ea2cc19f5 100644 --- a/sdk/python/tests/test_keep_client.py +++ b/sdk/python/tests/test_keep_client.py @@ -12,6 +12,7 @@ import pycurl import random import re import socket +import sys import threading import time import unittest @@ -48,12 +49,12 @@ class KeepTestCase(run_test_server.TestCaseWithServers): self.assertEqual(0, self.keep_client.download_counter.get()) self.assertEqual(self.keep_client.get(foo_locator), - 'foo', + b'foo', 'wrong content from Keep.get(md5("foo"))') self.assertEqual(3, self.keep_client.download_counter.get()) def test_KeepBinaryRWTest(self): - blob_str = '\xff\xfe\xf7\x00\x01\x02' + blob_str = b'\xff\xfe\xf7\x00\x01\x02' blob_locator = self.keep_client.put(blob_str) self.assertRegexpMatches( blob_locator, @@ -64,28 +65,28 @@ class KeepTestCase(run_test_server.TestCaseWithServers): 'wrong content from Keep.get(md5())') def test_KeepLongBinaryRWTest(self): - blob_str = '\xff\xfe\xfd\xfc\x00\x01\x02\x03' + blob_data = b'\xff\xfe\xfd\xfc\x00\x01\x02\x03' for i in range(0,23): - blob_str = blob_str + blob_str - blob_locator = self.keep_client.put(blob_str) + blob_data = blob_data + blob_data + blob_locator = self.keep_client.put(blob_data) self.assertRegexpMatches( blob_locator, '^84d90fc0d8175dd5dcfab04b999bc956\+67108864', ('wrong locator from Keep.put(): ' + blob_locator)) self.assertEqual(self.keep_client.get(blob_locator), - blob_str, + blob_data, 'wrong content from Keep.get(md5())') @unittest.skip("unreliable test - please fix and close #8752") def test_KeepSingleCopyRWTest(self): - blob_str = '\xff\xfe\xfd\xfc\x00\x01\x02\x03' - blob_locator = self.keep_client.put(blob_str, copies=1) + blob_data = b'\xff\xfe\xfd\xfc\x00\x01\x02\x03' + blob_locator = self.keep_client.put(blob_data, copies=1) self.assertRegexpMatches( blob_locator, '^c902006bc98a3eb4a3663b65ab4a6fab\+8', ('wrong locator from Keep.put(): ' + blob_locator)) self.assertEqual(self.keep_client.get(blob_locator), - blob_str, + blob_data, 'wrong content from Keep.get(md5())') def test_KeepEmptyCollectionTest(self): @@ -103,9 +104,10 @@ class KeepTestCase(run_test_server.TestCaseWithServers): '^acbd18db4cc2f85cedef654fccc4a4d8\+3', 'wrong md5 hash from Keep.put("foo"): ' + foo_locator) - with self.assertRaises(UnicodeEncodeError): - # Error if it is not ASCII - self.keep_client.put(u'\xe2') + if sys.version_info < (3, 0): + with self.assertRaises(UnicodeEncodeError): + # Error if it is not ASCII + self.keep_client.put(u'\xe2') with self.assertRaises(AttributeError): # Must be bytes or have an encode() method @@ -119,7 +121,7 @@ class KeepTestCase(run_test_server.TestCaseWithServers): 'wrong md5 hash from Keep.put for "test_head": ' + locator) self.assertEqual(True, self.keep_client.head(locator)) self.assertEqual(self.keep_client.get(locator), - 'test_head', + b'test_head', 'wrong content from Keep.get for "test_head"') class KeepPermissionTestCase(run_test_server.TestCaseWithServers): @@ -136,7 +138,7 @@ class KeepPermissionTestCase(run_test_server.TestCaseWithServers): r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$', 'invalid locator from Keep.put("foo"): ' + foo_locator) self.assertEqual(keep_client.get(foo_locator), - 'foo', + b'foo', 'wrong content from Keep.get(md5("foo"))') # GET with an unsigned locator => NotFound @@ -203,13 +205,13 @@ class KeepOptionalPermission(run_test_server.TestCaseWithServers): def test_KeepAuthenticatedSignedTest(self): signed_locator = self._put_foo_and_check() self.assertEqual(self.keep_client.get(signed_locator), - 'foo', + b'foo', 'wrong content from Keep.get(md5("foo"))') def test_KeepAuthenticatedUnsignedTest(self): signed_locator = self._put_foo_and_check() self.assertEqual(self.keep_client.get("acbd18db4cc2f85cedef654fccc4a4d8"), - 'foo', + b'foo', 'wrong content from Keep.get(md5("foo"))') def test_KeepUnauthenticatedSignedTest(self): @@ -218,7 +220,7 @@ class KeepOptionalPermission(run_test_server.TestCaseWithServers): signed_locator = self._put_foo_and_check() self.keep_client.api_token = '' self.assertEqual(self.keep_client.get(signed_locator), - 'foo', + b'foo', 'wrong content from Keep.get(md5("foo"))') def test_KeepUnauthenticatedUnsignedTest(self): @@ -227,7 +229,7 @@ class KeepOptionalPermission(run_test_server.TestCaseWithServers): signed_locator = self._put_foo_and_check() self.keep_client.api_token = '' self.assertEqual(self.keep_client.get("acbd18db4cc2f85cedef654fccc4a4d8"), - 'foo', + b'foo', 'wrong content from Keep.get(md5("foo"))') @@ -257,7 +259,7 @@ class KeepProxyTestCase(run_test_server.TestCaseWithServers): '^73feffa4b7f6bb68e44cf984c85f6e88\+3', 'wrong md5 hash from Keep.put("baz"): ' + baz_locator) self.assertEqual(keep_client.get(baz_locator), - 'baz', + b'baz', 'wrong content from Keep.get(md5("baz"))') self.assertTrue(keep_client.using_proxy) @@ -273,7 +275,7 @@ class KeepProxyTestCase(run_test_server.TestCaseWithServers): '^91f372a266fe2bf2823cb8ec7fda31ce\+4', 'wrong md5 hash from Keep.put("baz2"): ' + baz_locator) self.assertEqual(keep_client.get(baz_locator), - 'baz2', + b'baz2', 'wrong content from Keep.get(md5("baz2"))') self.assertTrue(keep_client.using_proxy) @@ -341,7 +343,7 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock): with tutil.mock_keep_responses(force_timeout, 0) as mock: keep_client = arvados.KeepClient(api_client=api_client) with self.assertRaises(arvados.errors.KeepWriteError): - keep_client.put('foo') + keep_client.put(b'foo') self.assertEqual( mock.responses[0].getopt(pycurl.CONNECTTIMEOUT_MS), int(arvados.KeepClient.DEFAULT_TIMEOUT[0]*1000)) @@ -482,7 +484,7 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock): self.assertEqual(0, len(exc_check.exception.request_errors())) def test_oddball_service_get(self): - body = 'oddball service get' + body = b'oddball service get' api_client = self.mock_keep_services(service_type='fancynewblobstore') with tutil.mock_keep_responses(body, 200): keep_client = arvados.KeepClient(api_client=api_client) @@ -490,7 +492,7 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock): self.assertEqual(body, actual) def test_oddball_service_put(self): - body = 'oddball service put' + body = b'oddball service put' pdh = tutil.str_keep_locator(body) api_client = self.mock_keep_services(service_type='fancynewblobstore') with tutil.mock_keep_responses(pdh, 200): @@ -499,7 +501,7 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock): self.assertEqual(pdh, actual) def test_oddball_service_writer_count(self): - body = 'oddball service writer count' + body = b'oddball service writer count' pdh = tutil.str_keep_locator(body) api_client = self.mock_keep_services(service_type='fancynewblobstore', count=4) @@ -530,7 +532,7 @@ class KeepClientRendezvousTestCase(unittest.TestCase, tutil.ApiClientMock): list('9d81c02e76a3bf54'), ] self.blocks = [ - "{:064x}".format(x) + "{:064x}".format(x).encode() for x in range(len(self.expected_order))] self.hashes = [ hashlib.md5(self.blocks[x]).hexdigest() @@ -567,7 +569,7 @@ class KeepClientRendezvousTestCase(unittest.TestCase, tutil.ApiClientMock): self.assertRaises(arvados.errors.KeepRequestError): op(i) got_order = [ - re.search(r'//\[?keep0x([0-9a-f]+)', resp.getopt(pycurl.URL)).group(1) + re.search(r'//\[?keep0x([0-9a-f]+)', resp.getopt(pycurl.URL).decode()).group(1) for resp in mock.responses] self.assertEqual(self.expected_order[i]*2, got_order) @@ -578,7 +580,7 @@ class KeepClientRendezvousTestCase(unittest.TestCase, tutil.ApiClientMock): self.assertRaises(arvados.errors.KeepWriteError): self.keep_client.put(self.blocks[i], num_retries=2, copies=copies) got_order = [ - re.search(r'//\[?keep0x([0-9a-f]+)', resp.getopt(pycurl.URL)).group(1) + re.search(r'//\[?keep0x([0-9a-f]+)', resp.getopt(pycurl.URL).decode()).group(1) for resp in mock.responses] # With T threads racing to make requests, the position # of a given server in the sequence of HTTP requests @@ -606,7 +608,7 @@ class KeepClientRendezvousTestCase(unittest.TestCase, tutil.ApiClientMock): def test_probe_waste_adding_one_server(self): hashes = [ - hashlib.md5("{:064x}".format(x)).hexdigest() for x in range(100)] + hashlib.md5("{:064x}".format(x).encode()).hexdigest() for x in range(100)] initial_services = 12 self.api_client = self.mock_keep_services(count=initial_services) self.keep_client = arvados.KeepClient(api_client=self.api_client) @@ -644,7 +646,7 @@ class KeepClientRendezvousTestCase(unittest.TestCase, tutil.ApiClientMock): max_penalty)) def check_64_zeros_error_order(self, verb, exc_class): - data = '0' * 64 + data = b'0' * 64 if verb == 'get': data = tutil.str_keep_locator(data) # Arbitrary port number: @@ -653,7 +655,7 @@ class KeepClientRendezvousTestCase(unittest.TestCase, tutil.ApiClientMock): keep_client = arvados.KeepClient(api_client=api_client) with mock.patch('pycurl.Curl') as curl_mock, \ self.assertRaises(exc_class) as err_check: - curl_mock.return_value.side_effect = socket.timeout + curl_mock.return_value = tutil.FakeCurl.make(code=500, body=b'') getattr(keep_client, verb)(data) urls = [urllib.parse.urlparse(url) for url in err_check.exception.request_errors()] @@ -671,7 +673,7 @@ class KeepClientTimeout(unittest.TestCase, tutil.ApiClientMock): # BANDWIDTH_LOW_LIM must be less than len(DATA) so we can transfer # 1s worth of data and then trigger bandwidth errors before running # out of data. - DATA = 'x'*2**11 + DATA = b'x'*2**11 BANDWIDTH_LOW_LIM = 1024 TIMEOUT_TIME = 1.0 @@ -847,9 +849,9 @@ class KeepClientGatewayTestCase(unittest.TestCase, tutil.ApiClientMock): code=200, body='foo', headers={'Content-Length': 3}) self.mock_disks_and_gateways() locator = 'acbd18db4cc2f85cedef654fccc4a4d8+3+K@' + self.gateways[0]['uuid'] - self.assertEqual('foo', self.keepClient.get(locator)) + self.assertEqual(b'foo', self.keepClient.get(locator)) self.assertEqual(self.gateway_roots[0]+locator, - MockCurl.return_value.getopt(pycurl.URL)) + MockCurl.return_value.getopt(pycurl.URL).decode()) self.assertEqual(True, self.keepClient.head(locator)) @mock.patch('pycurl.Curl') @@ -869,11 +871,11 @@ class KeepClientGatewayTestCase(unittest.TestCase, tutil.ApiClientMock): # Gateways are tried first, in the order given. for i, root in enumerate(self.gateway_roots): self.assertEqual(root+locator, - mocks[i].getopt(pycurl.URL)) + mocks[i].getopt(pycurl.URL).decode()) # Disk services are tried next. for i in range(gateways, gateways+disks): self.assertRegexpMatches( - mocks[i].getopt(pycurl.URL), + mocks[i].getopt(pycurl.URL).decode(), r'keep0x') @mock.patch('pycurl.Curl') @@ -881,7 +883,7 @@ class KeepClientGatewayTestCase(unittest.TestCase, tutil.ApiClientMock): gateways = 4 disks = 3 mocks = [ - tutil.FakeCurl.make(code=404, body='') + tutil.FakeCurl.make(code=404, body=b'') for _ in range(gateways+disks) ] MockCurl.side_effect = tutil.queue_with(mocks) @@ -893,32 +895,32 @@ class KeepClientGatewayTestCase(unittest.TestCase, tutil.ApiClientMock): # Gateways are tried first, in the order given. for i, root in enumerate(self.gateway_roots): self.assertEqual(root+locator, - mocks[i].getopt(pycurl.URL)) + mocks[i].getopt(pycurl.URL).decode()) # Disk services are tried next. for i in range(gateways, gateways+disks): self.assertRegexpMatches( - mocks[i].getopt(pycurl.URL), + mocks[i].getopt(pycurl.URL).decode(), r'keep0x') @mock.patch('pycurl.Curl') def test_get_with_remote_proxy_hint(self, MockCurl): MockCurl.return_value = tutil.FakeCurl.make( - code=200, body='foo', headers={'Content-Length': 3}) + code=200, body=b'foo', headers={'Content-Length': 3}) self.mock_disks_and_gateways() locator = 'acbd18db4cc2f85cedef654fccc4a4d8+3+K@xyzzy' - self.assertEqual('foo', self.keepClient.get(locator)) + self.assertEqual(b'foo', self.keepClient.get(locator)) self.assertEqual('https://keep.xyzzy.arvadosapi.com/'+locator, - MockCurl.return_value.getopt(pycurl.URL)) + MockCurl.return_value.getopt(pycurl.URL).decode()) @mock.patch('pycurl.Curl') def test_head_with_remote_proxy_hint(self, MockCurl): MockCurl.return_value = tutil.FakeCurl.make( - code=200, body='foo', headers={'Content-Length': 3}) + code=200, body=b'foo', headers={'Content-Length': 3}) self.mock_disks_and_gateways() locator = 'acbd18db4cc2f85cedef654fccc4a4d8+3+K@xyzzy' self.assertEqual(True, self.keepClient.head(locator)) self.assertEqual('https://keep.xyzzy.arvadosapi.com/'+locator, - MockCurl.return_value.getopt(pycurl.URL)) + MockCurl.return_value.getopt(pycurl.URL).decode()) class KeepClientRetryTestMixin(object): @@ -937,7 +939,7 @@ class KeepClientRetryTestMixin(object): # out appropriate methods in the client. PROXY_ADDR = 'http://[%s]:65535/' % (tutil.TEST_HOST,) - TEST_DATA = 'testdata' + TEST_DATA = b'testdata' TEST_LOCATOR = 'ef654c40ab4f1747fc699915d4f70902+8' def setUp(self): diff --git a/sdk/python/tests/test_stream.py b/sdk/python/tests/test_stream.py index 7277628122..37cdbf2f24 100644 --- a/sdk/python/tests/test_stream.py +++ b/sdk/python/tests/test_stream.py @@ -26,28 +26,28 @@ class StreamFileReaderTestCase(unittest.TestCase): def test_read_block_crossing_behavior(self): # read() calls will be aligned on block boundaries - see #3663. sfile = self.make_count_reader() - self.assertEqual('123', sfile.read(10)) + self.assertEqual(b'123', sfile.read(10)) def test_small_read(self): sfile = self.make_count_reader() - self.assertEqual('12', sfile.read(2)) + self.assertEqual(b'12', sfile.read(2)) def test_successive_reads(self): sfile = self.make_count_reader() - for expect in ['123', '456', '789', '']: + for expect in [b'123', b'456', b'789', b'']: self.assertEqual(expect, sfile.read(10)) def test_readfrom_spans_blocks(self): sfile = self.make_count_reader() - self.assertEqual('6789', sfile.readfrom(5, 12)) + self.assertEqual(b'6789', sfile.readfrom(5, 12)) def test_small_readfrom_spanning_blocks(self): sfile = self.make_count_reader() - self.assertEqual('2345', sfile.readfrom(1, 4)) + self.assertEqual(b'2345', sfile.readfrom(1, 4)) def test_readall(self): sfile = self.make_count_reader() - self.assertEqual('123456789', ''.join(sfile.readall())) + self.assertEqual(b'123456789', b''.join(sfile.readall())) def test_one_arg_seek(self): self.test_absolute_seek([]) @@ -55,20 +55,20 @@ class StreamFileReaderTestCase(unittest.TestCase): def test_absolute_seek(self, args=[os.SEEK_SET]): sfile = self.make_count_reader() sfile.seek(6, *args) - self.assertEqual('78', sfile.read(2)) + self.assertEqual(b'78', sfile.read(2)) sfile.seek(4, *args) - self.assertEqual('56', sfile.read(2)) + self.assertEqual(b'56', sfile.read(2)) def test_relative_seek(self, args=[os.SEEK_CUR]): sfile = self.make_count_reader() - self.assertEqual('12', sfile.read(2)) + self.assertEqual(b'12', sfile.read(2)) sfile.seek(2, *args) - self.assertEqual('56', sfile.read(2)) + self.assertEqual(b'56', sfile.read(2)) def test_end_seek(self): sfile = self.make_count_reader() sfile.seek(-6, os.SEEK_END) - self.assertEqual('45', sfile.read(2)) + self.assertEqual(b'45', sfile.read(2)) def test_seek_min_zero(self): sfile = self.make_count_reader() @@ -101,7 +101,7 @@ class StreamFileReaderTestCase(unittest.TestCase): def test_context(self): with self.make_count_reader() as sfile: self.assertFalse(sfile.closed, "reader is closed inside context") - self.assertEqual('12', sfile.read(2)) + self.assertEqual(b'12', sfile.read(2)) self.assertTrue(sfile.closed, "reader is open after context") def make_newlines_reader(self): @@ -163,12 +163,12 @@ class StreamFileReaderTestCase(unittest.TestCase): self.check_decompressed_name('test.log.bz2', 'test.log') def check_decompression(self, compress_ext, compress_func): - test_text = 'decompression\ntest\n' + test_text = b'decompression\ntest\n' test_data = compress_func(test_text) stream = tutil.MockStreamReader('.', test_data) reader = StreamFileReader(stream, [Range(0, 0, len(test_data))], 'test.' + compress_ext) - self.assertEqual(test_text, ''.join(reader.readall_decompressed())) + self.assertEqual(test_text, b''.join(reader.readall_decompressed())) @staticmethod def gzip_compress(data): @@ -197,7 +197,7 @@ class StreamFileReaderTestCase(unittest.TestCase): reader = self.make_newlines_reader() data = reader.readline() self.assertEqual('one\n', data) - self.assertEqual(''.join(['two\n', '\n', 'three\n', 'four\n', '\n']), ''.join(reader.readall())) + self.assertEqual(b''.join([b'two\n', b'\n', b'three\n', b'four\n', b'\n']), b''.join(reader.readall())) class StreamRetryTestMixin(object): @@ -216,7 +216,7 @@ class StreamRetryTestMixin(object): def test_success_without_retries(self): with tutil.mock_keep_responses('bar', 200): reader = self.reader_for('bar_file') - self.assertEqual('bar', self.read_for_test(reader, 3)) + self.assertEqual(b'bar', self.read_for_test(reader, 3)) @tutil.skip_sleep def test_read_no_default_retry(self): @@ -229,13 +229,13 @@ class StreamRetryTestMixin(object): def test_read_with_instance_retries(self): with tutil.mock_keep_responses('foo', 500, 200): reader = self.reader_for('foo_file', num_retries=3) - self.assertEqual('foo', self.read_for_test(reader, 3)) + self.assertEqual(b'foo', self.read_for_test(reader, 3)) @tutil.skip_sleep def test_read_with_method_retries(self): with tutil.mock_keep_responses('foo', 500, 200): reader = self.reader_for('foo_file') - self.assertEqual('foo', + self.assertEqual(b'foo', self.read_for_test(reader, 3, num_retries=3)) @tutil.skip_sleep @@ -291,17 +291,17 @@ class StreamFileReadFromTestCase(StreamFileReadTestCase): class StreamFileReadAllTestCase(StreamFileReadTestCase): def read_for_test(self, reader, byte_count, **kwargs): - return ''.join(reader.readall(**kwargs)) + return b''.join(reader.readall(**kwargs)) class StreamFileReadAllDecompressedTestCase(StreamFileReadTestCase): def read_for_test(self, reader, byte_count, **kwargs): - return ''.join(reader.readall_decompressed(**kwargs)) + return b''.join(reader.readall_decompressed(**kwargs)) class StreamFileReadlinesTestCase(StreamFileReadTestCase): def read_for_test(self, reader, byte_count, **kwargs): - return ''.join(reader.readlines(**kwargs)) + return ''.join(reader.readlines(**kwargs)).encode() if __name__ == '__main__': unittest.main() diff --git a/sdk/python/tests/test_util.py b/sdk/python/tests/test_util.py index 41739a92bd..bf59d0deda 100644 --- a/sdk/python/tests/test_util.py +++ b/sdk/python/tests/test_util.py @@ -20,7 +20,7 @@ class MkdirDashPTest(unittest.TestCase): def runTest(self): arvados.util.mkdir_dash_p('./tmp/foo') with open('./tmp/bar', 'wb') as f: - f.write('bar') + f.write(b'bar') self.assertRaises(OSError, arvados.util.mkdir_dash_p, './tmp/bar') @@ -28,8 +28,8 @@ class RunCommandTestCase(unittest.TestCase): def test_success(self): stdout, stderr = arvados.util.run_command(['echo', 'test'], stderr=subprocess.PIPE) - self.assertEqual("test\n", stdout) - self.assertEqual("", stderr) + self.assertEqual("test\n".encode(), stdout) + self.assertEqual("".encode(), stderr) def test_failure(self): with self.assertRaises(arvados.errors.CommandFailedError):