super(KeepProxyTestCase, self).tearDown()
def test_KeepProxyTest1(self):
- # Will use ARVADOS_KEEP_PROXY environment variable that is set by
- # setUpClass().
+ # Will use ARVADOS_KEEP_SERVICES environment variable that
+ # is set by setUpClass().
keep_client = arvados.KeepClient(api_client=self.api_client,
local_store='')
baz_locator = keep_client.put('baz')
'wrong content from Keep.get(md5("baz2"))')
self.assertTrue(keep_client.using_proxy)
+ def test_KeepProxyTestMultipleURIs(self):
+ # Test using ARVADOS_KEEP_SERVICES env var overriding any
+ # existing proxy setting and setting multiple proxies
+ arvados.config.settings()['ARVADOS_KEEP_SERVICES'] = 'http://10.0.0.1 https://foo.example.org:1234/'
+ keep_client = arvados.KeepClient(api_client=self.api_client,
+ local_store='')
+ uris = [x['_service_root'] for x in keep_client._keep_services]
+ self.assertEqual(uris, ['http://10.0.0.1/',
+ 'https://foo.example.org:1234/'])
+
+ def test_KeepProxyTestInvalidURI(self):
+ arvados.config.settings()['ARVADOS_KEEP_SERVICES'] = 'bad.uri.org'
+ with self.assertRaises(arvados.errors.ArgumentError):
+ keep_client = arvados.KeepClient(api_client=self.api_client,
+ local_store='')
+
class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock):
def get_service_roots(self, api_client):
# Allow 10s to connect, then 1s for response. Nothing should
# work, and everything should take at least 1s to return.
kc = self.keepClient(timeouts=(10, 1))
- with self.assertTakesBetween(1, 1.9):
+ with self.assertTakesBetween(1, 9):
with self.assertRaises(arvados.errors.KeepReadError):
kc.get(loc, num_retries=0)
- with self.assertTakesBetween(1, 1.9):
+ with self.assertTakesBetween(1, 9):
with self.assertRaises(arvados.errors.KeepWriteError):
kc.put(self.DATA, copies=1, num_retries=0)
self.check_exception(copies=2, num_retries=3)
-class KeepClientAvoidClientOverreplicationTestCase(unittest.TestCase, tutil.ApiClientMock):
+class AvoidOverreplication(unittest.TestCase, tutil.ApiClientMock):
+ class FakeKeepService(object):
+ def __init__(self, delay, will_succeed=False, will_raise=None, replicas=1):
+ self.delay = delay
+ self.will_succeed = will_succeed
+ self.will_raise = will_raise
+ self._result = {}
+ self._result['headers'] = {}
+ self._result['headers']['x-keep-replicas-stored'] = str(replicas)
+ self._result['body'] = 'foobar'
- class KeepFakeWriterThread(threading.Thread):
- """
- Just Simulating the real KeepClient.KeepWriterThread, to test the ThreadLimiter.
- """
- def __init__(self, delay, will_succeed, thread_limiter):
- super(KeepClientAvoidClientOverreplicationTestCase.KeepFakeWriterThread, self).__init__()
- self.delay = delay # in seconds
- self.success = will_succeed
- self.limiter = thread_limiter
+ def put(self, data_hash, data, timeout):
+ time.sleep(self.delay)
+ if self.will_raise is not None:
+ raise self.will_raise
+ return self.will_succeed
- def run(self):
- with self.limiter:
- if not self.limiter.shall_i_proceed():
- return
- time.sleep(self.delay)
- if self.success:
- self.limiter.save_response('foo', 1)
- else:
- self.limiter.save_response(None, 0)
+ def last_result(self):
+ if self.will_succeed:
+ return self._result
- def test_only_write_enough_on_success(self):
- copies = 3
- threads = []
- limiter = arvados.KeepClient.ThreadLimiter(want_copies=copies, max_service_replicas=1)
- # Setting up fake writer threads with different delays so that the bug is revealed
- for i in range(copies*2):
- t = self.KeepFakeWriterThread(
- delay=i/10.0,
- will_succeed=True,
- thread_limiter=limiter)
- t.start()
- threads.append(t)
- for t in threads:
- t.join()
- self.assertEqual(limiter.done(), copies)
-
- def test_only_write_enough_on_partial_failure(self):
- copies = 3
- threads = []
- limiter = arvados.KeepClient.ThreadLimiter(want_copies=copies, max_service_replicas=1)
- for i in range(copies):
- t = self.KeepFakeWriterThread(
- delay=i/10.0,
- will_succeed=False,
- thread_limiter=limiter)
- t.start()
- threads.append(t)
- t = self.KeepFakeWriterThread(
- delay=i/10.0,
- will_succeed=True,
- thread_limiter=limiter)
- t.start()
- threads.append(t)
- for t in threads:
- t.join()
- self.assertEqual(limiter.done(), copies)
+ def finished(self):
+ return False
+
+ def setUp(self):
+ self.copies = 3
+ self.pool = arvados.KeepClient.KeepWriterThreadPool(
+ data = 'foo',
+ data_hash = 'acbd18db4cc2f85cedef654fccc4a4d8+3',
+ max_service_replicas = self.copies,
+ copies = self.copies
+ )
+ def test_only_write_enough_on_success(self):
+ for i in range(10):
+ ks = self.FakeKeepService(delay=i/10.0, will_succeed=True)
+ self.pool.add_task(ks, None)
+ self.pool.join()
+ self.assertEqual(self.pool.done(), self.copies)
+
+ def test_only_write_enough_on_partial_success(self):
+ for i in range(5):
+ ks = self.FakeKeepService(delay=i/10.0, will_succeed=False)
+ self.pool.add_task(ks, None)
+ ks = self.FakeKeepService(delay=i/10.0, will_succeed=True)
+ self.pool.add_task(ks, None)
+ self.pool.join()
+ self.assertEqual(self.pool.done(), self.copies)
+
+ def test_only_write_enough_when_some_crash(self):
+ for i in range(5):
+ ks = self.FakeKeepService(delay=i/10.0, will_raise=Exception())
+ self.pool.add_task(ks, None)
+ ks = self.FakeKeepService(delay=i/10.0, will_succeed=True)
+ self.pool.add_task(ks, None)
+ self.pool.join()
+ self.assertEqual(self.pool.done(), self.copies)
+
+ def test_fail_when_too_many_crash(self):
+ for i in range(self.copies+1):
+ ks = self.FakeKeepService(delay=i/10.0, will_raise=Exception())
+ self.pool.add_task(ks, None)
+ for i in range(self.copies-1):
+ ks = self.FakeKeepService(delay=i/10.0, will_succeed=True)
+ self.pool.add_task(ks, None)
+ self.pool.join()
+ self.assertEqual(self.pool.done(), self.copies-1)
+
@tutil.skip_sleep
class RetryNeedsMultipleServices(unittest.TestCase, tutil.ApiClientMock):
'acbd18db4cc2f85cedef654fccc4a4d8+3',
Exception('mock err'), 200, 200) as req_mock:
self.keep_client.put('foo', num_retries=1, copies=2)
- self.assertTrue(3, req_mock.call_count)
+ self.assertEqual(3, req_mock.call_count)
def test_success_after_retryable_error(self):
with tutil.mock_keep_responses(
'acbd18db4cc2f85cedef654fccc4a4d8+3',
500, 200, 200) as req_mock:
self.keep_client.put('foo', num_retries=1, copies=2)
- self.assertTrue(3, req_mock.call_count)
+ self.assertEqual(3, req_mock.call_count)
def test_fail_after_final_error(self):
# First retry loop gets a 200 (can't achieve replication by
200, 400, 200) as req_mock:
with self.assertRaises(arvados.errors.KeepWriteError):
self.keep_client.put('foo', num_retries=1, copies=2)
- self.assertTrue(2, req_mock.call_count)
+ self.assertEqual(2, req_mock.call_count)