X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/c9fe930b422bd1675af3adb324ede3b5aa28888c..4a275f0de83f6d38348a24ea06b73adcfe5c1354:/sdk/python/tests/test_keep_client.py?ds=sidebyside diff --git a/sdk/python/tests/test_keep_client.py b/sdk/python/tests/test_keep_client.py index 27e3cf6330..0eefa586d9 100644 --- a/sdk/python/tests/test_keep_client.py +++ b/sdk/python/tests/test_keep_client.py @@ -70,7 +70,7 @@ class KeepTestCase(run_test_server.TestCaseWithServers): def test_KeepLongBinaryRWTest(self): blob_data = b'\xff\xfe\xfd\xfc\x00\x01\x02\x03' - for i in range(0,23): + for i in range(0, 23): blob_data = blob_data + blob_data blob_locator = self.keep_client.put(blob_data) self.assertRegex( @@ -532,6 +532,93 @@ class KeepClientCacheTestCase(unittest.TestCase, tutil.ApiClientMock): # First reponse was not cached because it was from a HEAD request. self.assertNotEqual(head_resp, get_resp) +@tutil.skip_sleep +class KeepStorageClassesTestCase(unittest.TestCase, tutil.ApiClientMock): + def setUp(self): + self.api_client = self.mock_keep_services(count=2) + self.keep_client = arvados.KeepClient(api_client=self.api_client) + self.data = b'xyzzy' + self.locator = '1271ed5ef305aadabc605b1609e24c52' + + def test_storage_classes_req_header(self): + cases = [ + # requested, expected + [['foo'], 'X-Keep-Storage-Classes: foo'], + [['bar', 'foo'], 'X-Keep-Storage-Classes: bar, foo'], + [[], None], + ] + for req_classes, expected_header in cases: + headers = {'x-keep-replicas-stored': 1} + if len(req_classes) > 0: + confirmed_hdr = ', '.join(["{}=1".format(cls) for cls in req_classes]) + headers.update({'x-keep-storage-classes-confirmed': confirmed_hdr}) + with tutil.mock_keep_responses(self.locator, 200, **headers) as mock: + self.keep_client.put(self.data, copies=1, classes=req_classes) + resp = mock.responses[0] + if expected_header is not None: + self.assertIn(expected_header, resp.getopt(pycurl.HTTPHEADER)) + else: + for hdr in resp.getopt(pycurl.HTTPHEADER): + self.assertNotRegex(hdr, r'^X-Keep-Storage-Classes.*') + + def test_partial_storage_classes_put(self): + headers = { + 'x-keep-replicas-stored': 1, + 'x-keep-storage-classes-confirmed': 'foo=1'} + with tutil.mock_keep_responses(self.locator, 200, 503, **headers) as mock: + with self.assertRaises(arvados.errors.KeepWriteError): + self.keep_client.put(self.data, copies=1, classes=['foo', 'bar']) + # 1st request, both classes pending + req1_headers = mock.responses[0].getopt(pycurl.HTTPHEADER) + self.assertIn('X-Keep-Storage-Classes: bar, foo', req1_headers) + # 2nd try, 'foo' class already satisfied + req2_headers = mock.responses[1].getopt(pycurl.HTTPHEADER) + self.assertIn('X-Keep-Storage-Classes: bar', req2_headers) + + def test_successful_storage_classes_put_requests(self): + cases = [ + # wanted_copies, wanted_classes, confirmed_copies, confirmed_classes, expected_requests + [ 1, ['foo'], 1, 'foo=1', 1], + [ 1, ['foo'], 2, 'foo=2', 1], + [ 2, ['foo'], 2, 'foo=2', 1], + [ 2, ['foo'], 1, 'foo=1', 2], + [ 1, ['foo', 'bar'], 1, 'foo=1, bar=1', 1], + [ 1, ['foo', 'bar'], 2, 'foo=2, bar=2', 1], + [ 2, ['foo', 'bar'], 2, 'foo=2, bar=2', 1], + [ 2, ['foo', 'bar'], 1, 'foo=1, bar=1', 2], + [ 1, ['foo', 'bar'], 1, None, 1], + [ 1, ['foo'], 1, None, 1], + [ 2, ['foo'], 2, None, 1], + [ 2, ['foo'], 1, None, 2], + ] + for w_copies, w_classes, c_copies, c_classes, e_reqs in cases: + headers = {'x-keep-replicas-stored': c_copies} + if c_classes is not None: + headers.update({'x-keep-storage-classes-confirmed': c_classes}) + with tutil.mock_keep_responses(self.locator, 200, 200, **headers) as mock: + case_desc = 'wanted_copies={}, wanted_classes="{}", confirmed_copies={}, confirmed_classes="{}", expected_requests={}'.format(w_copies, ', '.join(w_classes), c_copies, c_classes, e_reqs) + self.assertEqual(self.locator, + self.keep_client.put(self.data, copies=w_copies, classes=w_classes), + case_desc) + self.assertEqual(e_reqs, mock.call_count, case_desc) + + def test_failed_storage_classes_put_requests(self): + cases = [ + # wanted_copies, wanted_classes, confirmed_copies, confirmed_classes, return_code + [ 1, ['foo'], 1, 'bar=1', 200], + [ 1, ['foo'], 1, None, 503], + [ 2, ['foo'], 1, 'bar=1, foo=0', 200], + [ 3, ['foo'], 1, 'bar=1, foo=1', 200], + [ 3, ['foo', 'bar'], 1, 'bar=2, foo=1', 200], + ] + for w_copies, w_classes, c_copies, c_classes, return_code in cases: + headers = {'x-keep-replicas-stored': c_copies} + if c_classes is not None: + headers.update({'x-keep-storage-classes-confirmed': c_classes}) + with tutil.mock_keep_responses(self.locator, return_code, return_code, **headers): + case_desc = 'wanted_copies={}, wanted_classes="{}", confirmed_copies={}, confirmed_classes="{}"'.format(w_copies, ', '.join(w_classes), c_copies, c_classes) + with self.assertRaises(arvados.errors.KeepWriteError, msg=case_desc): + self.keep_client.put(self.data, copies=w_copies, classes=w_classes) @tutil.skip_sleep class KeepXRequestIdTestCase(unittest.TestCase, tutil.ApiClientMock): @@ -1178,9 +1265,10 @@ class AvoidOverreplication(unittest.TestCase, tutil.ApiClientMock): self._result = {} self._result['headers'] = {} self._result['headers']['x-keep-replicas-stored'] = str(replicas) + self._result['headers']['x-keep-storage-classes-confirmed'] = 'default={}'.format(replicas) self._result['body'] = 'foobar' - def put(self, data_hash, data, timeout): + def put(self, data_hash, data, timeout, headers): time.sleep(self.delay) if self.will_raise is not None: raise self.will_raise @@ -1207,7 +1295,7 @@ class AvoidOverreplication(unittest.TestCase, tutil.ApiClientMock): ks = self.FakeKeepService(delay=i/10.0, will_succeed=True) self.pool.add_task(ks, None) self.pool.join() - self.assertEqual(self.pool.done(), self.copies) + self.assertEqual(self.pool.done(), (self.copies, [])) def test_only_write_enough_on_partial_success(self): for i in range(5): @@ -1216,7 +1304,7 @@ class AvoidOverreplication(unittest.TestCase, tutil.ApiClientMock): ks = self.FakeKeepService(delay=i/10.0, will_succeed=True) self.pool.add_task(ks, None) self.pool.join() - self.assertEqual(self.pool.done(), self.copies) + self.assertEqual(self.pool.done(), (self.copies, [])) def test_only_write_enough_when_some_crash(self): for i in range(5): @@ -1225,7 +1313,7 @@ class AvoidOverreplication(unittest.TestCase, tutil.ApiClientMock): ks = self.FakeKeepService(delay=i/10.0, will_succeed=True) self.pool.add_task(ks, None) self.pool.join() - self.assertEqual(self.pool.done(), self.copies) + self.assertEqual(self.pool.done(), (self.copies, [])) def test_fail_when_too_many_crash(self): for i in range(self.copies+1): @@ -1235,7 +1323,7 @@ class AvoidOverreplication(unittest.TestCase, tutil.ApiClientMock): ks = self.FakeKeepService(delay=i/10.0, will_succeed=True) self.pool.add_task(ks, None) self.pool.join() - self.assertEqual(self.pool.done(), self.copies-1) + self.assertEqual(self.pool.done(), (self.copies-1, [])) @tutil.skip_sleep