sdk/python/tests/test_keep_client.py

   1 import mock
   2 import os
   3 import socket
   4 import unittest
   5
   6 import arvados
   7 import arvados.retry
   8 import run_test_server
   9 from arvados_testutil import fake_httplib2_response
  10
  11 class KeepTestCase(run_test_server.TestCaseWithServers):
  12     MAIN_SERVER = {}
  13     KEEP_SERVER = {}
  14
  15     @classmethod
  16     def setUpClass(cls):
  17         super(KeepTestCase, cls).setUpClass()
  18         run_test_server.authorize_with("admin")
  19         cls.api_client = arvados.api('v1')
  20         cls.keep_client = arvados.KeepClient(api_client=cls.api_client,
  21                                              proxy='', local_store='')
  22
  23     def test_KeepBasicRWTest(self):
  24         foo_locator = self.keep_client.put('foo')
  25         self.assertRegexpMatches(
  26             foo_locator,
  27             '^acbd18db4cc2f85cedef654fccc4a4d8\+3',
  28             'wrong md5 hash from Keep.put("foo"): ' + foo_locator)
  29         self.assertEqual(self.keep_client.get(foo_locator),
  30                          'foo',
  31                          'wrong content from Keep.get(md5("foo"))')
  32
  33     def test_KeepBinaryRWTest(self):
  34         blob_str = '\xff\xfe\xf7\x00\x01\x02'
  35         blob_locator = self.keep_client.put(blob_str)
  36         self.assertRegexpMatches(
  37             blob_locator,
  38             '^7fc7c53b45e53926ba52821140fef396\+6',
  39             ('wrong locator from Keep.put(<binarydata>):' + blob_locator))
  40         self.assertEqual(self.keep_client.get(blob_locator),
  41                          blob_str,
  42                          'wrong content from Keep.get(md5(<binarydata>))')
  43
  44     def test_KeepLongBinaryRWTest(self):
  45         blob_str = '\xff\xfe\xfd\xfc\x00\x01\x02\x03'
  46         for i in range(0,23):
  47             blob_str = blob_str + blob_str
  48         blob_locator = self.keep_client.put(blob_str)
  49         self.assertRegexpMatches(
  50             blob_locator,
  51             '^84d90fc0d8175dd5dcfab04b999bc956\+67108864',
  52             ('wrong locator from Keep.put(<binarydata>): ' + blob_locator))
  53         self.assertEqual(self.keep_client.get(blob_locator),
  54                          blob_str,
  55                          'wrong content from Keep.get(md5(<binarydata>))')
  56
  57     def test_KeepSingleCopyRWTest(self):
  58         blob_str = '\xff\xfe\xfd\xfc\x00\x01\x02\x03'
  59         blob_locator = self.keep_client.put(blob_str, copies=1)
  60         self.assertRegexpMatches(
  61             blob_locator,
  62             '^c902006bc98a3eb4a3663b65ab4a6fab\+8',
  63             ('wrong locator from Keep.put(<binarydata>): ' + blob_locator))
  64         self.assertEqual(self.keep_client.get(blob_locator),
  65                          blob_str,
  66                          'wrong content from Keep.get(md5(<binarydata>))')
  67
  68     def test_KeepEmptyCollectionTest(self):
  69         blob_locator = self.keep_client.put('', copies=1)
  70         self.assertRegexpMatches(
  71             blob_locator,
  72             '^d41d8cd98f00b204e9800998ecf8427e\+0',
  73             ('wrong locator from Keep.put(""): ' + blob_locator))
  74
  75
  76 class KeepPermissionTestCase(run_test_server.TestCaseWithServers):
  77     MAIN_SERVER = {}
  78     KEEP_SERVER = {'blob_signing_key': 'abcdefghijk0123456789',
  79                    'enforce_permissions': True}
  80
  81     def test_KeepBasicRWTest(self):
  82         run_test_server.authorize_with('active')
  83         keep_client = arvados.KeepClient()
  84         foo_locator = keep_client.put('foo')
  85         self.assertRegexpMatches(
  86             foo_locator,
  87             r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
  88             'invalid locator from Keep.put("foo"): ' + foo_locator)
  89         self.assertEqual(keep_client.get(foo_locator),
  90                          'foo',
  91                          'wrong content from Keep.get(md5("foo"))')
  92
  93         # GET with an unsigned locator => NotFound
  94         bar_locator = keep_client.put('bar')
  95         unsigned_bar_locator = "37b51d194a7513e45b56f6524f2d51f2+3"
  96         self.assertRegexpMatches(
  97             bar_locator,
  98             r'^37b51d194a7513e45b56f6524f2d51f2\+3\+A[a-f0-9]+@[a-f0-9]+$',
  99             'invalid locator from Keep.put("bar"): ' + bar_locator)
 100         self.assertRaises(arvados.errors.NotFoundError,
 101                           keep_client.get,
 102                           unsigned_bar_locator)
 103
 104         # GET from a different user => NotFound
 105         run_test_server.authorize_with('spectator')
 106         self.assertRaises(arvados.errors.NotFoundError,
 107                           arvados.Keep.get,
 108                           bar_locator)
 109
 110         # Unauthenticated GET for a signed locator => NotFound
 111         # Unauthenticated GET for an unsigned locator => NotFound
 112         keep_client.api_token = ''
 113         self.assertRaises(arvados.errors.NotFoundError,
 114                           keep_client.get,
 115                           bar_locator)
 116         self.assertRaises(arvados.errors.NotFoundError,
 117                           keep_client.get,
 118                           unsigned_bar_locator)
 119
 120
 121 # KeepOptionalPermission: starts Keep with --permission-key-file
 122 # but not --enforce-permissions (i.e. generate signatures on PUT
 123 # requests, but do not require them for GET requests)
 124 #
 125 # All of these requests should succeed when permissions are optional:
 126 # * authenticated request, signed locator
 127 # * authenticated request, unsigned locator
 128 # * unauthenticated request, signed locator
 129 # * unauthenticated request, unsigned locator
 130 class KeepOptionalPermission(run_test_server.TestCaseWithServers):
 131     MAIN_SERVER = {}
 132     KEEP_SERVER = {'blob_signing_key': 'abcdefghijk0123456789',
 133                    'enforce_permissions': False}
 134
 135     @classmethod
 136     def setUpClass(cls):
 137         super(KeepOptionalPermission, cls).setUpClass()
 138         run_test_server.authorize_with("admin")
 139         cls.api_client = arvados.api('v1')
 140
 141     def setUp(self):
 142         super(KeepOptionalPermission, self).setUp()
 143         self.keep_client = arvados.KeepClient(api_client=self.api_client,
 144                                               proxy='', local_store='')
 145
 146     def _put_foo_and_check(self):
 147         signed_locator = self.keep_client.put('foo')
 148         self.assertRegexpMatches(
 149             signed_locator,
 150             r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
 151             'invalid locator from Keep.put("foo"): ' + signed_locator)
 152         return signed_locator
 153
 154     def test_KeepAuthenticatedSignedTest(self):
 155         signed_locator = self._put_foo_and_check()
 156         self.assertEqual(self.keep_client.get(signed_locator),
 157                          'foo',
 158                          'wrong content from Keep.get(md5("foo"))')
 159
 160     def test_KeepAuthenticatedUnsignedTest(self):
 161         signed_locator = self._put_foo_and_check()
 162         self.assertEqual(self.keep_client.get("acbd18db4cc2f85cedef654fccc4a4d8"),
 163                          'foo',
 164                          'wrong content from Keep.get(md5("foo"))')
 165
 166     def test_KeepUnauthenticatedSignedTest(self):
 167         # Check that signed GET requests work even when permissions
 168         # enforcement is off.
 169         signed_locator = self._put_foo_and_check()
 170         self.keep_client.api_token = ''
 171         self.assertEqual(self.keep_client.get(signed_locator),
 172                          'foo',
 173                          'wrong content from Keep.get(md5("foo"))')
 174
 175     def test_KeepUnauthenticatedUnsignedTest(self):
 176         # Since --enforce-permissions is not in effect, GET requests
 177         # need not be authenticated.
 178         signed_locator = self._put_foo_and_check()
 179         self.keep_client.api_token = ''
 180         self.assertEqual(self.keep_client.get("acbd18db4cc2f85cedef654fccc4a4d8"),
 181                          'foo',
 182                          'wrong content from Keep.get(md5("foo"))')
 183
 184
 185 class KeepProxyTestCase(run_test_server.TestCaseWithServers):
 186     MAIN_SERVER = {}
 187     KEEP_SERVER = {}
 188     KEEP_PROXY_SERVER = {'auth': 'admin'}
 189
 190     @classmethod
 191     def setUpClass(cls):
 192         super(KeepProxyTestCase, cls).setUpClass()
 193         cls.api_client = arvados.api('v1')
 194
 195     def tearDown(self):
 196         arvados.config.settings().pop('ARVADOS_EXTERNAL_CLIENT', None)
 197         super(KeepProxyTestCase, self).tearDown()
 198
 199     def test_KeepProxyTest1(self):
 200         # Will use ARVADOS_KEEP_PROXY environment variable that is set by
 201         # setUpClass().
 202         keep_client = arvados.KeepClient(api_client=self.api_client,
 203                                          local_store='')
 204         baz_locator = keep_client.put('baz')
 205         self.assertRegexpMatches(
 206             baz_locator,
 207             '^73feffa4b7f6bb68e44cf984c85f6e88\+3',
 208             'wrong md5 hash from Keep.put("baz"): ' + baz_locator)
 209         self.assertEqual(keep_client.get(baz_locator),
 210                          'baz',
 211                          'wrong content from Keep.get(md5("baz"))')
 212         self.assertTrue(keep_client.using_proxy)
 213
 214     def test_KeepProxyTest2(self):
 215         # Don't instantiate the proxy directly, but set the X-External-Client
 216         # header.  The API server should direct us to the proxy.
 217         arvados.config.settings()['ARVADOS_EXTERNAL_CLIENT'] = 'true'
 218         keep_client = arvados.KeepClient(api_client=self.api_client,
 219                                          proxy='', local_store='')
 220         baz_locator = keep_client.put('baz2')
 221         self.assertRegexpMatches(
 222             baz_locator,
 223             '^91f372a266fe2bf2823cb8ec7fda31ce\+4',
 224             'wrong md5 hash from Keep.put("baz2"): ' + baz_locator)
 225         self.assertEqual(keep_client.get(baz_locator),
 226                          'baz2',
 227                          'wrong content from Keep.get(md5("baz2"))')
 228         self.assertTrue(keep_client.using_proxy)
 229
 230
 231 class KeepClientRetryTestMixin(object):
 232     # Testing with a local Keep store won't exercise the retry behavior.
 233     # Instead, our strategy is:
 234     # * Create a client with one proxy specified (pointed at a black
 235     #   hole), so there's no need to instantiate an API client, and
 236     #   all HTTP requests come from one place.
 237     # * Mock httplib's request method to provide simulated responses.
 238     # This lets us test the retry logic extensively without relying on any
 239     # supporting servers, and prevents side effects in case something hiccups.
 240     # To use this mixin, define DEFAULT_EXPECT, DEFAULT_EXCEPTION, and
 241     # run_method().
 242     PROXY_ADDR = 'http://[100::]/'
 243     TEST_DATA = 'testdata'
 244     TEST_LOCATOR = 'ef654c40ab4f1747fc699915d4f70902+8'
 245
 246     @staticmethod
 247     def mock_responses(body, *codes):
 248         return mock.patch('httplib2.Http.request', side_effect=(
 249                 (fake_httplib2_response(code), body) for code in codes))
 250
 251     def new_client(self):
 252         return arvados.KeepClient(proxy=self.PROXY_ADDR, local_store='')
 253
 254     def run_method(self, *args, **kwargs):
 255         raise NotImplementedError("test subclasses must define run_method")
 256
 257     def check_success(self, expected=None, *args, **kwargs):
 258         if expected is None:
 259             expected = self.DEFAULT_EXPECT
 260         self.assertEqual(expected, self.run_method(*args, **kwargs))
 261
 262     def check_exception(self, error_class=None, *args, **kwargs):
 263         if error_class is None:
 264             error_class = self.DEFAULT_EXCEPTION
 265         self.assertRaises(error_class, self.run_method, *args, **kwargs)
 266
 267     def test_immediate_success(self):
 268         with self.mock_responses(self.DEFAULT_EXPECT, 200):
 269             self.check_success()
 270
 271     def test_retry_then_success(self):
 272         with self.mock_responses(self.DEFAULT_EXPECT, 500, 200):
 273             self.check_success(num_retries=3)
 274
 275     def test_no_default_retry(self):
 276         with self.mock_responses(self.DEFAULT_EXPECT, 500, 200):
 277             self.check_exception()
 278
 279     def test_no_retry_after_permanent_error(self):
 280         with self.mock_responses(self.DEFAULT_EXPECT, 403, 200):
 281             self.check_exception(num_retries=3)
 282
 283     def test_error_after_retries_exhausted(self):
 284         with self.mock_responses(self.DEFAULT_EXPECT, 500, 500, 200):
 285             self.check_exception(num_retries=1)
 286
 287
 288 # Don't delay from HTTPRetryLoop's exponential backoff.
 289 no_backoff = mock.patch('time.sleep', lambda n: None)
 290 @no_backoff
 291 class KeepClientRetryGetTestCase(unittest.TestCase, KeepClientRetryTestMixin):
 292     DEFAULT_EXPECT = KeepClientRetryTestMixin.TEST_DATA
 293     DEFAULT_EXCEPTION = arvados.errors.KeepReadError
 294     HINTED_LOCATOR = KeepClientRetryTestMixin.TEST_LOCATOR + '+K@xyzzy'
 295
 296     def run_method(self, locator=KeepClientRetryTestMixin.TEST_LOCATOR,
 297                    *args, **kwargs):
 298         return self.new_client().get(locator, *args, **kwargs)
 299
 300     def test_specific_exception_when_not_found(self):
 301         with self.mock_responses(self.DEFAULT_EXPECT, 404, 200):
 302             self.check_exception(arvados.errors.NotFoundError, num_retries=3)
 303
 304     def test_general_exception_with_mixed_errors(self):
 305         # get should raise a NotFoundError if no server returns the block,
 306         # and a high threshold of servers report that it's not found.
 307         # This test rigs up 50/50 disagreement between two servers, and
 308         # checks that it does not become a NotFoundError.
 309         client = self.new_client()
 310         with self.mock_responses(self.DEFAULT_EXPECT, 404, 500):
 311             with self.assertRaises(arvados.errors.KeepReadError) as exc_check:
 312                 client.get(self.HINTED_LOCATOR)
 313             self.assertNotIsInstance(
 314                 exc_check.exception, arvados.errors.NotFoundError,
 315                 "mixed errors raised NotFoundError")
 316
 317     def test_hint_server_can_succeed_without_retries(self):
 318         with self.mock_responses(self.DEFAULT_EXPECT, 404, 200, 500):
 319             self.check_success(locator=self.HINTED_LOCATOR)
 320
 321     def test_try_next_server_after_timeout(self):
 322         responses = iter([None, (fake_httplib2_response(200),
 323                                  self.DEFAULT_EXPECT)])
 324         def side_effect(*args, **kwargs):
 325             response = next(responses)
 326             if response is None:
 327                 raise socket.timeout("timed out")
 328             else:
 329                 return response
 330         with mock.patch('httplib2.Http.request', side_effect=side_effect):
 331             self.check_success(locator=self.HINTED_LOCATOR)
 332
 333
 334 @no_backoff
 335 class KeepClientRetryPutTestCase(unittest.TestCase, KeepClientRetryTestMixin):
 336     DEFAULT_EXPECT = KeepClientRetryTestMixin.TEST_LOCATOR
 337     DEFAULT_EXCEPTION = arvados.errors.KeepWriteError
 338
 339     def run_method(self, data=KeepClientRetryTestMixin.TEST_DATA,
 340                    copies=1, *args, **kwargs):
 341         return self.new_client().put(data, copies, *args, **kwargs)
 342
 343     def test_do_not_send_multiple_copies_to_same_server(self):
 344         with self.mock_responses(self.DEFAULT_EXPECT, 200):
 345             self.check_exception(copies=2, num_retries=3)