import gflags
-import httplib
-import httplib2
import logging
import os
import pprint
import gflags
-import httplib
-import httplib2
import logging
import os
import pprint
import datetime
import ssl
import socket
+import requests
_logger = logging.getLogger('arvados.keep')
global_client_object = None
class KeepService(object):
# Make requests to a single Keep service, and track results.
- HTTP_ERRORS = (httplib2.HttpLib2Error, httplib.HTTPException,
+ HTTP_ERRORS = (requests.exceptions.RequestException,
socket.error, ssl.SSLError)
- def __init__(self, root, **headers):
+ def __init__(self, root, timeout=None, **headers):
self.root = root
self.last_result = None
self.success_flag = None
def last_status(self):
try:
- return int(self.last_result[0].status)
+ return self.last_result.status_code
except (AttributeError, IndexError, ValueError):
return None
- def get(self, http, locator):
- # http is an httplib2.Http object.
+ def get(self, locator, timeout=None):
# locator is a KeepLocator object.
url = self.root + str(locator)
_logger.debug("Request: GET %s", url)
try:
with timer.Timer() as t:
- result = http.request(url.encode('utf-8'), 'GET',
- headers=self.get_headers)
+ result = requests.get(url.encode('utf-8'),
+ headers=self.get_headers,
+ timeout=timeout)
except self.HTTP_ERRORS as e:
_logger.debug("Request fail: GET %s => %s: %s",
url, type(e), str(e))
else:
self.last_result = result
self.success_flag = retry.check_http_response_success(result)
- content = result[1]
+ content = result.content
_logger.info("%s response: %s bytes in %s msec (%.3f MiB/sec)",
self.last_status(), len(content), t.msecs,
(len(content)/(1024.0*1024))/t.secs)
url, resp_md5)
return None
- def put(self, http, hash_s, body):
+ def put(self, hash_s, body, timeout=None):
url = self.root + hash_s
_logger.debug("Request: PUT %s", url)
try:
- result = http.request(url.encode('utf-8'), 'PUT',
- headers=self.put_headers, body=body)
+ result = requests.put(url.encode('utf-8'),
+ data=body,
+ headers=self.put_headers,
+ timeout=timeout)
except self.HTTP_ERRORS as e:
_logger.debug("Request fail: PUT %s => %s: %s",
url, type(e), str(e))
str(threading.current_thread()),
self.args['data_hash'],
self.args['service_root'])
- h = httplib2.Http(timeout=self.args.get('timeout', None))
self._success = bool(self.service.put(
- h, self.args['data_hash'], self.args['data']))
+ self.args['data_hash'],
+ self.args['data'],
+ timeout=self.args.get('timeout', None)))
status = self.service.last_status()
if self._success:
- resp, body = self.service.last_result
+ result = self.service.last_result
_logger.debug("KeepWriterThread %s succeeded %s %s",
str(threading.current_thread()),
self.args['data_hash'],
# we're talking to a proxy or other backend that
# stores to multiple copies for us.
try:
- replicas_stored = int(resp['x-keep-replicas-stored'])
+ replicas_stored = int(result.headers['x-keep-replicas-stored'])
except (KeyError, ValueError):
replicas_stored = 1
- limiter.save_response(body.strip(), replicas_stored)
+ limiter.save_response(result.text.strip(), replicas_stored)
elif status is not None:
_logger.debug("Request fail: PUT %s => %s %s",
self.args['data_hash'], status,
- self.service.last_result[1])
+ self.service.last_result.text)
def __init__(self, api_client=None, proxy=None, timeout=300,
of the ARVADOS_KEEP_PROXY configuration setting. If you want to
ensure KeepClient does not use a proxy, pass in an empty string.
* timeout: The timeout for all HTTP requests, in seconds. Default
- 300.
+ 300. A tuple of two floats is interpreted as (connection_timeout,
+ read_timeout)
* api_token: If you're not using an API client, but only talking
directly to a Keep proxy, this parameter specifies an API token
to authenticate Keep requests. It is an error to specify both
services_to_try = [roots_map[root]
for root in (local_roots + hint_roots)
if roots_map[root].usable()]
- http = httplib2.Http(timeout=self.timeout)
for keep_service in services_to_try:
- blob = keep_service.get(http, locator)
+ blob = keep_service.get(locator, timeout=self.timeout)
if blob is not None:
break
loop.save_result((blob, len(services_to_try)))
def check_http_response_success(result):
- """Convert an httplib2 request result to a loop control flag.
+ """Convert a 'requests' response to a loop control flag.
- Pass this method the 2-tuple returned by httplib2.Http.request. It
- returns True if the response indicates success, None if it indicates
- temporary failure, and False otherwise. You can use this as the
+ Pass this method a requests.Response object. It returns True if
+ the response indicates success, None if it indicates temporary
+ failure, and False otherwise. You can use this as the
success_check for a RetryLoop.
Implementation details:
retry those requests verbatim.
"""
try:
- status = int(result[0].status)
+ status = result.status_code
except Exception:
return None
if status in _HTTP_SUCCESSES:
import gflags
-import httplib
-import httplib2
import os
import pprint
import sys
'python-gflags',
'google-api-python-client',
'httplib2',
+ 'requests',
'urllib3',
'ws4py'
],
import errno
import httplib
import httplib2
+import io
import mock
import os
import shutil
import tempfile
import unittest
+import requests
# Use this hostname when you want to make sure the traffic will be
# instantly refused. 100::/64 is a dedicated black hole.
skip_sleep = mock.patch('time.sleep', lambda n: None) # clown'll eat me
+# fake_httplib2_response and mock_responses
+# mock calls to httplib2.Http.request()
def fake_httplib2_response(code, **headers):
headers.update(status=str(code),
reason=httplib.responses.get(code, "Unknown Response"))
return mock.patch('httplib2.Http.request', side_effect=(
(fake_httplib2_response(code, **headers), body) for code in codes))
+# fake_requests_response, mock_get_responses and mock_put_responses
+# mock calls to requests.get() and requests.put()
+def fake_requests_response(code, body, **headers):
+ r = requests.Response()
+ r.status_code = code
+ r.reason = httplib.responses.get(code, "Unknown Response")
+ r.headers = headers
+ r.raw = io.BytesIO(body)
+ return r
+
+def mock_get_responses(body, *codes, **headers):
+ return mock.patch('requests.get', side_effect=(
+ fake_requests_response(code, body, **headers) for code in codes))
+
+def mock_put_responses(body, *codes, **headers):
+ return mock.patch('requests.put', side_effect=(
+ fake_requests_response(code, body, **headers) for code in codes))
+
+def mock_requestslib_responses(method, body, *codes, **headers):
+ return mock.patch(method, side_effect=(
+ fake_requests_response(code, body, **headers) for code in codes))
+
class ArvadosBaseTestCase(unittest.TestCase):
# This class provides common utility functions for our tests.
mock_method.return_value = body
else:
mock_method.side_effect = arvados.errors.ApiError(
- tutil.fake_httplib2_response(code), "{}")
+ tutil.fake_requests_response(code, None), "{}")
def mock_keep_services(self, api_mock, code, body):
self._mock_api_call(api_mock.keep_services().accessible, code, body)
def test_locator_init(self):
client = self.api_client_mock(200)
# Ensure Keep will not return anything if asked.
- with tutil.mock_responses(None, 404):
+ with tutil.mock_get_responses(None, 404):
reader = arvados.CollectionReader(self.DEFAULT_DATA_HASH,
api_client=client)
self.assertEqual(self.DEFAULT_MANIFEST, reader.manifest_text())
# been written to Keep.
client = self.api_client_mock(200)
self.mock_get_collection(client, 404, None)
- with tutil.mock_responses(self.DEFAULT_MANIFEST, 200):
+ with tutil.mock_get_responses(self.DEFAULT_MANIFEST, 200):
reader = arvados.CollectionReader(self.DEFAULT_DATA_HASH,
api_client=client)
self.assertEqual(self.DEFAULT_MANIFEST, reader.manifest_text())
client = self.api_client_mock(404)
reader = arvados.CollectionReader(self.DEFAULT_UUID,
api_client=client)
- with tutil.mock_responses(self.DEFAULT_MANIFEST, 200):
+ with tutil.mock_get_responses(self.DEFAULT_MANIFEST, 200):
with self.assertRaises(arvados.errors.ApiError):
reader.manifest_text()
# To verify that CollectionReader tries Keep first here, we
# mock API server to return the wrong data.
client = self.api_client_mock(200)
- with tutil.mock_responses(self.ALT_MANIFEST, 200):
+ with tutil.mock_get_responses(self.ALT_MANIFEST, 200):
self.assertEqual(
self.ALT_MANIFEST,
arvados.CollectionReader(
client = self.api_client_mock(200)
reader = arvados.CollectionReader(self.DEFAULT_UUID, api_client=client,
num_retries=3)
- with tutil.mock_responses('foo', 500, 500, 200):
+ with tutil.mock_get_responses('foo', 500, 500, 200):
self.assertEqual('foo',
''.join(f.read(9) for f in reader.all_files()))
class CollectionWriterTestCase(unittest.TestCase, CollectionTestMixin):
def mock_keep(self, body, *codes, **headers):
headers.setdefault('x-keep-replicas-stored', 2)
- return tutil.mock_responses(body, *codes, **headers)
+ return tutil.mock_put_responses(body, *codes, **headers)
def foo_writer(self, **kwargs):
api_client = self.api_client_mock()
self.assertRaises(error_class, self.run_method, *args, **kwargs)
def test_immediate_success(self):
- with tutil.mock_responses(self.DEFAULT_EXPECT, 200):
+ with tutil.mock_requestslib_responses(self.mock_method, self.DEFAULT_EXPECT, 200):
self.check_success()
def test_retry_then_success(self):
- with tutil.mock_responses(self.DEFAULT_EXPECT, 500, 200):
+ with tutil.mock_requestslib_responses(self.mock_method, self.DEFAULT_EXPECT, 500, 200):
self.check_success(num_retries=3)
def test_no_default_retry(self):
- with tutil.mock_responses(self.DEFAULT_EXPECT, 500, 200):
+ with tutil.mock_requestslib_responses(self.mock_method, self.DEFAULT_EXPECT, 500, 200):
self.check_exception()
def test_no_retry_after_permanent_error(self):
- with tutil.mock_responses(self.DEFAULT_EXPECT, 403, 200):
+ with tutil.mock_requestslib_responses(self.mock_method, self.DEFAULT_EXPECT, 403, 200):
self.check_exception(num_retries=3)
def test_error_after_retries_exhausted(self):
- with tutil.mock_responses(self.DEFAULT_EXPECT, 500, 500, 200):
+ with tutil.mock_requestslib_responses(self.mock_method, self.DEFAULT_EXPECT, 500, 500, 200):
self.check_exception(num_retries=1)
def test_num_retries_instance_fallback(self):
self.client_kwargs['num_retries'] = 3
- with tutil.mock_responses(self.DEFAULT_EXPECT, 500, 200):
+ with tutil.mock_requestslib_responses(self.mock_method, self.DEFAULT_EXPECT, 500, 200):
self.check_success()
+class KeepClientRetryGetTestMixin(KeepClientRetryTestMixin):
+ mock_method = 'requests.get'
+
+
+class KeepClientRetryPutTestMixin(KeepClientRetryTestMixin):
+ mock_method = 'requests.put'
+
+
@tutil.skip_sleep
-class KeepClientRetryGetTestCase(KeepClientRetryTestMixin, unittest.TestCase):
+class KeepClientRetryGetTestCase(KeepClientRetryGetTestMixin, unittest.TestCase):
DEFAULT_EXPECT = KeepClientRetryTestMixin.TEST_DATA
DEFAULT_EXCEPTION = arvados.errors.KeepReadError
HINTED_LOCATOR = KeepClientRetryTestMixin.TEST_LOCATOR + '+K@xyzzy'
return self.new_client().get(locator, *args, **kwargs)
def test_specific_exception_when_not_found(self):
- with tutil.mock_responses(self.DEFAULT_EXPECT, 404, 200):
+ with tutil.mock_get_responses(self.DEFAULT_EXPECT, 404, 200):
self.check_exception(arvados.errors.NotFoundError, num_retries=3)
def test_general_exception_with_mixed_errors(self):
# This test rigs up 50/50 disagreement between two servers, and
# checks that it does not become a NotFoundError.
client = self.new_client()
- with tutil.mock_responses(self.DEFAULT_EXPECT, 404, 500):
+ with tutil.mock_get_responses(self.DEFAULT_EXPECT, 404, 500):
with self.assertRaises(arvados.errors.KeepReadError) as exc_check:
client.get(self.HINTED_LOCATOR)
self.assertNotIsInstance(
"mixed errors raised NotFoundError")
def test_hint_server_can_succeed_without_retries(self):
- with tutil.mock_responses(self.DEFAULT_EXPECT, 404, 200, 500):
+ with tutil.mock_get_responses(self.DEFAULT_EXPECT, 404, 200, 500):
self.check_success(locator=self.HINTED_LOCATOR)
def test_try_next_server_after_timeout(self):
side_effects = [
socket.timeout("timed out"),
- (tutil.fake_httplib2_response(200), self.DEFAULT_EXPECT)]
- with mock.patch('httplib2.Http.request',
+ tutil.fake_requests_response(200, self.DEFAULT_EXPECT)]
+ with mock.patch('requests.get',
side_effect=iter(side_effects)):
self.check_success(locator=self.HINTED_LOCATOR)
def test_retry_data_with_wrong_checksum(self):
- side_effects = ((tutil.fake_httplib2_response(200), s)
+ side_effects = (tutil.fake_requests_response(200, s)
for s in ['baddata', self.TEST_DATA])
- with mock.patch('httplib2.Http.request', side_effect=side_effects):
+ with mock.patch('requests.get', side_effect=side_effects):
self.check_success(locator=self.HINTED_LOCATOR)
@tutil.skip_sleep
-class KeepClientRetryPutTestCase(KeepClientRetryTestMixin, unittest.TestCase):
+class KeepClientRetryPutTestCase(KeepClientRetryPutTestMixin, unittest.TestCase):
DEFAULT_EXPECT = KeepClientRetryTestMixin.TEST_LOCATOR
DEFAULT_EXCEPTION = arvados.errors.KeepWriteError
return self.new_client().put(data, copies, *args, **kwargs)
def test_do_not_send_multiple_copies_to_same_server(self):
- with tutil.mock_responses(self.DEFAULT_EXPECT, 200):
+ with tutil.mock_put_responses(self.DEFAULT_EXPECT, 200):
self.check_exception(copies=2, num_retries=3)
import arvados.retry as arv_retry
import mock
-from arvados_testutil import fake_httplib2_response
+from arvados_testutil import fake_requests_response
class RetryLoopTestMixin(object):
@staticmethod
class CheckHTTPResponseSuccessTestCase(unittest.TestCase):
def results_map(self, *codes):
for code in codes:
- response = (fake_httplib2_response(code), None)
+ response = fake_requests_response(code, None)
yield code, arv_retry.check_http_response_success(response)
def check(assert_name):
@tutil.skip_sleep
def test_success_without_retries(self):
reader = self.reader_for('bar_file')
- with tutil.mock_responses('bar', 200):
+ with tutil.mock_get_responses('bar', 200):
self.assertEqual('bar', self.read_for_test(reader, 3))
@tutil.skip_sleep
def test_read_no_default_retry(self):
reader = self.reader_for('user_agreement')
- with tutil.mock_responses('', 500):
+ with tutil.mock_get_responses('', 500):
with self.assertRaises(arvados.errors.KeepReadError):
self.read_for_test(reader, 10)
@tutil.skip_sleep
def test_read_with_instance_retries(self):
reader = self.reader_for('foo_file', num_retries=3)
- with tutil.mock_responses('foo', 500, 200):
+ with tutil.mock_get_responses('foo', 500, 200):
self.assertEqual('foo', self.read_for_test(reader, 3))
@tutil.skip_sleep
def test_read_with_method_retries(self):
reader = self.reader_for('foo_file')
- with tutil.mock_responses('foo', 500, 200):
+ with tutil.mock_get_responses('foo', 500, 200):
self.assertEqual('foo',
self.read_for_test(reader, 3, num_retries=3))
@tutil.skip_sleep
def test_read_instance_retries_exhausted(self):
reader = self.reader_for('bar_file', num_retries=3)
- with tutil.mock_responses('bar', 500, 500, 500, 500, 200):
+ with tutil.mock_get_responses('bar', 500, 500, 500, 500, 200):
with self.assertRaises(arvados.errors.KeepReadError):
self.read_for_test(reader, 3)
@tutil.skip_sleep
def test_read_method_retries_exhausted(self):
reader = self.reader_for('bar_file')
- with tutil.mock_responses('bar', 500, 500, 500, 500, 200):
+ with tutil.mock_get_responses('bar', 500, 500, 500, 500, 200):
with self.assertRaises(arvados.errors.KeepReadError):
self.read_for_test(reader, 3, num_retries=3)
@tutil.skip_sleep
def test_method_retries_take_precedence(self):
reader = self.reader_for('user_agreement', num_retries=10)
- with tutil.mock_responses('', 500, 500, 500, 200):
+ with tutil.mock_get_responses('', 500, 500, 500, 200):
with self.assertRaises(arvados.errors.KeepReadError):
self.read_for_test(reader, 10, num_retries=1)