sdk/python/tests/test_keep_client.py

   1 import mock
   2 import os
   3 import socket
   4 import unittest
   5 import urlparse
   6
   7 import arvados
   8 import arvados.retry
   9 import arvados_testutil as tutil
  10 import run_test_server
  11
  12 class KeepTestCase(run_test_server.TestCaseWithServers):
  13     MAIN_SERVER = {}
  14     KEEP_SERVER = {}
  15
  16     @classmethod
  17     def setUpClass(cls):
  18         super(KeepTestCase, cls).setUpClass()
  19         run_test_server.authorize_with("admin")
  20         cls.api_client = arvados.api('v1')
  21         cls.keep_client = arvados.KeepClient(api_client=cls.api_client,
  22                                              proxy='', local_store='')
  23
  24     def test_KeepBasicRWTest(self):
  25         foo_locator = self.keep_client.put('foo')
  26         self.assertRegexpMatches(
  27             foo_locator,
  28             '^acbd18db4cc2f85cedef654fccc4a4d8\+3',
  29             'wrong md5 hash from Keep.put("foo"): ' + foo_locator)
  30         self.assertEqual(self.keep_client.get(foo_locator),
  31                          'foo',
  32                          'wrong content from Keep.get(md5("foo"))')
  33
  34     def test_KeepBinaryRWTest(self):
  35         blob_str = '\xff\xfe\xf7\x00\x01\x02'
  36         blob_locator = self.keep_client.put(blob_str)
  37         self.assertRegexpMatches(
  38             blob_locator,
  39             '^7fc7c53b45e53926ba52821140fef396\+6',
  40             ('wrong locator from Keep.put(<binarydata>):' + blob_locator))
  41         self.assertEqual(self.keep_client.get(blob_locator),
  42                          blob_str,
  43                          'wrong content from Keep.get(md5(<binarydata>))')
  44
  45     def test_KeepLongBinaryRWTest(self):
  46         blob_str = '\xff\xfe\xfd\xfc\x00\x01\x02\x03'
  47         for i in range(0,23):
  48             blob_str = blob_str + blob_str
  49         blob_locator = self.keep_client.put(blob_str)
  50         self.assertRegexpMatches(
  51             blob_locator,
  52             '^84d90fc0d8175dd5dcfab04b999bc956\+67108864',
  53             ('wrong locator from Keep.put(<binarydata>): ' + blob_locator))
  54         self.assertEqual(self.keep_client.get(blob_locator),
  55                          blob_str,
  56                          'wrong content from Keep.get(md5(<binarydata>))')
  57
  58     def test_KeepSingleCopyRWTest(self):
  59         blob_str = '\xff\xfe\xfd\xfc\x00\x01\x02\x03'
  60         blob_locator = self.keep_client.put(blob_str, copies=1)
  61         self.assertRegexpMatches(
  62             blob_locator,
  63             '^c902006bc98a3eb4a3663b65ab4a6fab\+8',
  64             ('wrong locator from Keep.put(<binarydata>): ' + blob_locator))
  65         self.assertEqual(self.keep_client.get(blob_locator),
  66                          blob_str,
  67                          'wrong content from Keep.get(md5(<binarydata>))')
  68
  69
  70 class KeepPermissionTestCase(run_test_server.TestCaseWithServers):
  71     MAIN_SERVER = {}
  72     KEEP_SERVER = {'blob_signing_key': 'abcdefghijk0123456789',
  73                    'enforce_permissions': True}
  74
  75     def test_KeepBasicRWTest(self):
  76         run_test_server.authorize_with('active')
  77         keep_client = arvados.KeepClient()
  78         foo_locator = keep_client.put('foo')
  79         self.assertRegexpMatches(
  80             foo_locator,
  81             r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
  82             'invalid locator from Keep.put("foo"): ' + foo_locator)
  83         self.assertEqual(keep_client.get(foo_locator),
  84                          'foo',
  85                          'wrong content from Keep.get(md5("foo"))')
  86
  87         # GET with an unsigned locator => NotFound
  88         bar_locator = keep_client.put('bar')
  89         unsigned_bar_locator = "37b51d194a7513e45b56f6524f2d51f2+3"
  90         self.assertRegexpMatches(
  91             bar_locator,
  92             r'^37b51d194a7513e45b56f6524f2d51f2\+3\+A[a-f0-9]+@[a-f0-9]+$',
  93             'invalid locator from Keep.put("bar"): ' + bar_locator)
  94         self.assertRaises(arvados.errors.NotFoundError,
  95                           keep_client.get,
  96                           unsigned_bar_locator)
  97
  98         # GET from a different user => NotFound
  99         run_test_server.authorize_with('spectator')
 100         self.assertRaises(arvados.errors.NotFoundError,
 101                           arvados.Keep.get,
 102                           bar_locator)
 103
 104         # Unauthenticated GET for a signed locator => NotFound
 105         # Unauthenticated GET for an unsigned locator => NotFound
 106         keep_client.api_token = ''
 107         self.assertRaises(arvados.errors.NotFoundError,
 108                           keep_client.get,
 109                           bar_locator)
 110         self.assertRaises(arvados.errors.NotFoundError,
 111                           keep_client.get,
 112                           unsigned_bar_locator)
 113
 114
 115 # KeepOptionalPermission: starts Keep with --permission-key-file
 116 # but not --enforce-permissions (i.e. generate signatures on PUT
 117 # requests, but do not require them for GET requests)
 118 #
 119 # All of these requests should succeed when permissions are optional:
 120 # * authenticated request, signed locator
 121 # * authenticated request, unsigned locator
 122 # * unauthenticated request, signed locator
 123 # * unauthenticated request, unsigned locator
 124 class KeepOptionalPermission(run_test_server.TestCaseWithServers):
 125     MAIN_SERVER = {}
 126     KEEP_SERVER = {'blob_signing_key': 'abcdefghijk0123456789',
 127                    'enforce_permissions': False}
 128
 129     @classmethod
 130     def setUpClass(cls):
 131         super(KeepOptionalPermission, cls).setUpClass()
 132         run_test_server.authorize_with("admin")
 133         cls.api_client = arvados.api('v1')
 134
 135     def setUp(self):
 136         super(KeepOptionalPermission, self).setUp()
 137         self.keep_client = arvados.KeepClient(api_client=self.api_client,
 138                                               proxy='', local_store='')
 139
 140     def _put_foo_and_check(self):
 141         signed_locator = self.keep_client.put('foo')
 142         self.assertRegexpMatches(
 143             signed_locator,
 144             r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
 145             'invalid locator from Keep.put("foo"): ' + signed_locator)
 146         return signed_locator
 147
 148     def test_KeepAuthenticatedSignedTest(self):
 149         signed_locator = self._put_foo_and_check()
 150         self.assertEqual(self.keep_client.get(signed_locator),
 151                          'foo',
 152                          'wrong content from Keep.get(md5("foo"))')
 153
 154     def test_KeepAuthenticatedUnsignedTest(self):
 155         signed_locator = self._put_foo_and_check()
 156         self.assertEqual(self.keep_client.get("acbd18db4cc2f85cedef654fccc4a4d8"),
 157                          'foo',
 158                          'wrong content from Keep.get(md5("foo"))')
 159
 160     def test_KeepUnauthenticatedSignedTest(self):
 161         # Check that signed GET requests work even when permissions
 162         # enforcement is off.
 163         signed_locator = self._put_foo_and_check()
 164         self.keep_client.api_token = ''
 165         self.assertEqual(self.keep_client.get(signed_locator),
 166                          'foo',
 167                          'wrong content from Keep.get(md5("foo"))')
 168
 169     def test_KeepUnauthenticatedUnsignedTest(self):
 170         # Since --enforce-permissions is not in effect, GET requests
 171         # need not be authenticated.
 172         signed_locator = self._put_foo_and_check()
 173         self.keep_client.api_token = ''
 174         self.assertEqual(self.keep_client.get("acbd18db4cc2f85cedef654fccc4a4d8"),
 175                          'foo',
 176                          'wrong content from Keep.get(md5("foo"))')
 177
 178
 179 class KeepProxyTestCase(run_test_server.TestCaseWithServers):
 180     MAIN_SERVER = {}
 181     KEEP_SERVER = {}
 182     KEEP_PROXY_SERVER = {'auth': 'admin'}
 183
 184     @classmethod
 185     def setUpClass(cls):
 186         super(KeepProxyTestCase, cls).setUpClass()
 187         cls.api_client = arvados.api('v1')
 188
 189     def tearDown(self):
 190         arvados.config.settings().pop('ARVADOS_EXTERNAL_CLIENT', None)
 191         super(KeepProxyTestCase, self).tearDown()
 192
 193     def test_KeepProxyTest1(self):
 194         # Will use ARVADOS_KEEP_PROXY environment variable that is set by
 195         # setUpClass().
 196         keep_client = arvados.KeepClient(api_client=self.api_client,
 197                                          local_store='')
 198         baz_locator = keep_client.put('baz')
 199         self.assertRegexpMatches(
 200             baz_locator,
 201             '^73feffa4b7f6bb68e44cf984c85f6e88\+3',
 202             'wrong md5 hash from Keep.put("baz"): ' + baz_locator)
 203         self.assertEqual(keep_client.get(baz_locator),
 204                          'baz',
 205                          'wrong content from Keep.get(md5("baz"))')
 206         self.assertTrue(keep_client.using_proxy)
 207
 208     def test_KeepProxyTest2(self):
 209         # Don't instantiate the proxy directly, but set the X-External-Client
 210         # header.  The API server should direct us to the proxy.
 211         arvados.config.settings()['ARVADOS_EXTERNAL_CLIENT'] = 'true'
 212         keep_client = arvados.KeepClient(api_client=self.api_client,
 213                                          proxy='', local_store='')
 214         baz_locator = keep_client.put('baz2')
 215         self.assertRegexpMatches(
 216             baz_locator,
 217             '^91f372a266fe2bf2823cb8ec7fda31ce\+4',
 218             'wrong md5 hash from Keep.put("baz2"): ' + baz_locator)
 219         self.assertEqual(keep_client.get(baz_locator),
 220                          'baz2',
 221                          'wrong content from Keep.get(md5("baz2"))')
 222         self.assertTrue(keep_client.using_proxy)
 223
 224
 225 class KeepClientServiceTestCase(unittest.TestCase):
 226     def mock_keep_services(self, *services):
 227         api_client = mock.MagicMock(name='api_client')
 228         api_client.keep_services().accessible().execute.return_value = {
 229             'items_available': len(services),
 230             'items': [{
 231                     'uuid': 'zzzzz-bi6l4-mockservice{:04x}'.format(index),
 232                     'owner_uuid': 'zzzzz-tpzed-mockownerabcdef',
 233                     'service_host': host,
 234                     'service_port': port,
 235                     'service_ssl_flag': ssl,
 236                     'service_type': servtype,
 237                     } for index, (host, port, ssl, servtype)
 238                       in enumerate(services)],
 239             }
 240         return api_client
 241
 242     def get_service_roots(self, *services):
 243         api_client = self.mock_keep_services(*services)
 244         keep_client = arvados.KeepClient(api_client=api_client)
 245         services = keep_client.shuffled_service_roots('000000')
 246         return [urlparse.urlparse(url) for url in sorted(services)]
 247
 248     def test_ssl_flag_respected_in_roots(self):
 249         services = self.get_service_roots(('keep', 10, False, 'disk'),
 250                                           ('keep', 20, True, 'disk'))
 251         self.assertEqual(10, services[0].port)
 252         self.assertEqual('http', services[0].scheme)
 253         self.assertEqual(20, services[1].port)
 254         self.assertEqual('https', services[1].scheme)
 255
 256     def test_correct_ports_with_ipv6_addresses(self):
 257         service = self.get_service_roots(('100::1', 10, True, 'proxy'))[0]
 258         self.assertEqual('100::1', service.hostname)
 259         self.assertEqual(10, service.port)
 260
 261
 262 class KeepClientRetryTestMixin(object):
 263     # Testing with a local Keep store won't exercise the retry behavior.
 264     # Instead, our strategy is:
 265     # * Create a client with one proxy specified (pointed at a black
 266     #   hole), so there's no need to instantiate an API client, and
 267     #   all HTTP requests come from one place.
 268     # * Mock httplib's request method to provide simulated responses.
 269     # This lets us test the retry logic extensively without relying on any
 270     # supporting servers, and prevents side effects in case something hiccups.
 271     # To use this mixin, define DEFAULT_EXPECT, DEFAULT_EXCEPTION, and
 272     # run_method().
 273     PROXY_ADDR = 'http://[%s]:65535/' % (tutil.TEST_HOST,)
 274     TEST_DATA = 'testdata'
 275     TEST_LOCATOR = 'ef654c40ab4f1747fc699915d4f70902+8'
 276
 277     def setUp(self):
 278         self.client_kwargs = {'proxy': self.PROXY_ADDR, 'local_store': ''}
 279
 280     def new_client(self, **caller_kwargs):
 281         kwargs = self.client_kwargs.copy()
 282         kwargs.update(caller_kwargs)
 283         return arvados.KeepClient(**kwargs)
 284
 285     def run_method(self, *args, **kwargs):
 286         raise NotImplementedError("test subclasses must define run_method")
 287
 288     def check_success(self, expected=None, *args, **kwargs):
 289         if expected is None:
 290             expected = self.DEFAULT_EXPECT
 291         self.assertEqual(expected, self.run_method(*args, **kwargs))
 292
 293     def check_exception(self, error_class=None, *args, **kwargs):
 294         if error_class is None:
 295             error_class = self.DEFAULT_EXCEPTION
 296         self.assertRaises(error_class, self.run_method, *args, **kwargs)
 297
 298     def test_immediate_success(self):
 299         with tutil.mock_responses(self.DEFAULT_EXPECT, 200):
 300             self.check_success()
 301
 302     def test_retry_then_success(self):
 303         with tutil.mock_responses(self.DEFAULT_EXPECT, 500, 200):
 304             self.check_success(num_retries=3)
 305
 306     def test_no_default_retry(self):
 307         with tutil.mock_responses(self.DEFAULT_EXPECT, 500, 200):
 308             self.check_exception()
 309
 310     def test_no_retry_after_permanent_error(self):
 311         with tutil.mock_responses(self.DEFAULT_EXPECT, 403, 200):
 312             self.check_exception(num_retries=3)
 313
 314     def test_error_after_retries_exhausted(self):
 315         with tutil.mock_responses(self.DEFAULT_EXPECT, 500, 500, 200):
 316             self.check_exception(num_retries=1)
 317
 318     def test_num_retries_instance_fallback(self):
 319         self.client_kwargs['num_retries'] = 3
 320         with tutil.mock_responses(self.DEFAULT_EXPECT, 500, 200):
 321             self.check_success()
 322
 323
 324 @tutil.skip_sleep
 325 class KeepClientRetryGetTestCase(KeepClientRetryTestMixin, unittest.TestCase):
 326     DEFAULT_EXPECT = KeepClientRetryTestMixin.TEST_DATA
 327     DEFAULT_EXCEPTION = arvados.errors.KeepReadError
 328     HINTED_LOCATOR = KeepClientRetryTestMixin.TEST_LOCATOR + '+K@xyzzy'
 329
 330     def run_method(self, locator=KeepClientRetryTestMixin.TEST_LOCATOR,
 331                    *args, **kwargs):
 332         return self.new_client().get(locator, *args, **kwargs)
 333
 334     def test_specific_exception_when_not_found(self):
 335         with tutil.mock_responses(self.DEFAULT_EXPECT, 404, 200):
 336             self.check_exception(arvados.errors.NotFoundError, num_retries=3)
 337
 338     def test_general_exception_with_mixed_errors(self):
 339         # get should raise a NotFoundError if no server returns the block,
 340         # and a high threshold of servers report that it's not found.
 341         # This test rigs up 50/50 disagreement between two servers, and
 342         # checks that it does not become a NotFoundError.
 343         client = self.new_client()
 344         with tutil.mock_responses(self.DEFAULT_EXPECT, 404, 500):
 345             with self.assertRaises(arvados.errors.KeepReadError) as exc_check:
 346                 client.get(self.HINTED_LOCATOR)
 347             self.assertNotIsInstance(
 348                 exc_check.exception, arvados.errors.NotFoundError,
 349                 "mixed errors raised NotFoundError")
 350
 351     def test_hint_server_can_succeed_without_retries(self):
 352         with tutil.mock_responses(self.DEFAULT_EXPECT, 404, 200, 500):
 353             self.check_success(locator=self.HINTED_LOCATOR)
 354
 355     def test_try_next_server_after_timeout(self):
 356         side_effects = [
 357             socket.timeout("timed out"),
 358             (tutil.fake_httplib2_response(200), self.DEFAULT_EXPECT)]
 359         with mock.patch('httplib2.Http.request',
 360                         side_effect=iter(side_effects)):
 361             self.check_success(locator=self.HINTED_LOCATOR)
 362
 363     def test_retry_data_with_wrong_checksum(self):
 364         side_effects = ((tutil.fake_httplib2_response(200), s)
 365                         for s in ['baddata', self.TEST_DATA])
 366         with mock.patch('httplib2.Http.request', side_effect=side_effects):
 367             self.check_success(locator=self.HINTED_LOCATOR)
 368
 369
 370 @tutil.skip_sleep
 371 class KeepClientRetryPutTestCase(KeepClientRetryTestMixin, unittest.TestCase):
 372     DEFAULT_EXPECT = KeepClientRetryTestMixin.TEST_LOCATOR
 373     DEFAULT_EXCEPTION = arvados.errors.KeepWriteError
 374
 375     def run_method(self, data=KeepClientRetryTestMixin.TEST_DATA,
 376                    copies=1, *args, **kwargs):
 377         return self.new_client().put(data, copies, *args, **kwargs)
 378
 379     def test_do_not_send_multiple_copies_to_same_server(self):
 380         with tutil.mock_responses(self.DEFAULT_EXPECT, 200):
 381             self.check_exception(copies=2, num_retries=3)