sdk/python/tests/test_keep_client.py

   1 # usage example:
   2 #
   3 # ARVADOS_API_TOKEN=abc ARVADOS_API_HOST=arvados.local python -m unittest discover
   4
   5 import contextlib
   6 import os
   7 import unittest
   8
   9 import arvados
  10 import run_test_server
  11
  12 @contextlib.contextmanager
  13 def unauthenticated_client(keep_client=None):
  14     if keep_client is None:
  15         keep_client = arvados.keep.global_client_object
  16     if not hasattr(keep_client, 'api_token'):
  17         yield keep_client
  18     else:
  19         orig_token = keep_client.api_token
  20         keep_client.api_token = ''
  21         yield keep_client
  22         keep_client.api_token = orig_token
  23
  24 class KeepTestCase(run_test_server.TestCaseWithServers):
  25     MAIN_SERVER = {}
  26     KEEP_SERVER = {}
  27
  28     def setUp(self):
  29         arvados.keep.global_client_object = None
  30         run_test_server.authorize_with("admin")
  31
  32     def test_KeepBasicRWTest(self):
  33         foo_locator = arvados.Keep.put('foo')
  34         self.assertRegexpMatches(
  35             foo_locator,
  36             '^acbd18db4cc2f85cedef654fccc4a4d8\+3',
  37             'wrong md5 hash from Keep.put("foo"): ' + foo_locator)
  38         self.assertEqual(arvados.Keep.get(foo_locator),
  39                          'foo',
  40                          'wrong content from Keep.get(md5("foo"))')
  41
  42     def test_KeepBinaryRWTest(self):
  43         blob_str = '\xff\xfe\xf7\x00\x01\x02'
  44         blob_locator = arvados.Keep.put(blob_str)
  45         self.assertRegexpMatches(
  46             blob_locator,
  47             '^7fc7c53b45e53926ba52821140fef396\+6',
  48             ('wrong locator from Keep.put(<binarydata>):' + blob_locator))
  49         self.assertEqual(arvados.Keep.get(blob_locator),
  50                          blob_str,
  51                          'wrong content from Keep.get(md5(<binarydata>))')
  52
  53     def test_KeepLongBinaryRWTest(self):
  54         blob_str = '\xff\xfe\xfd\xfc\x00\x01\x02\x03'
  55         for i in range(0,23):
  56             blob_str = blob_str + blob_str
  57         blob_locator = arvados.Keep.put(blob_str)
  58         self.assertRegexpMatches(
  59             blob_locator,
  60             '^84d90fc0d8175dd5dcfab04b999bc956\+67108864',
  61             ('wrong locator from Keep.put(<binarydata>): ' + blob_locator))
  62         self.assertEqual(arvados.Keep.get(blob_locator),
  63                          blob_str,
  64                          'wrong content from Keep.get(md5(<binarydata>))')
  65
  66     def test_KeepSingleCopyRWTest(self):
  67         blob_str = '\xff\xfe\xfd\xfc\x00\x01\x02\x03'
  68         blob_locator = arvados.Keep.put(blob_str, copies=1)
  69         self.assertRegexpMatches(
  70             blob_locator,
  71             '^c902006bc98a3eb4a3663b65ab4a6fab\+8',
  72             ('wrong locator from Keep.put(<binarydata>): ' + blob_locator))
  73         self.assertEqual(arvados.Keep.get(blob_locator),
  74                          blob_str,
  75                          'wrong content from Keep.get(md5(<binarydata>))')
  76
  77
  78 class KeepPermissionTestCase(run_test_server.TestCaseWithServers):
  79     MAIN_SERVER = {}
  80     KEEP_SERVER = {'blob_signing_key': 'abcdefghijk0123456789',
  81                    'enforce_permissions': True}
  82
  83     def test_KeepBasicRWTest(self):
  84         run_test_server.authorize_with('active')
  85         foo_locator = arvados.Keep.put('foo')
  86         self.assertRegexpMatches(
  87             foo_locator,
  88             r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
  89             'invalid locator from Keep.put("foo"): ' + foo_locator)
  90         self.assertEqual(arvados.Keep.get(foo_locator),
  91                          'foo',
  92                          'wrong content from Keep.get(md5("foo"))')
  93
  94         # GET with an unsigned locator => NotFound
  95         bar_locator = arvados.Keep.put('bar')
  96         unsigned_bar_locator = "37b51d194a7513e45b56f6524f2d51f2+3"
  97         self.assertRegexpMatches(
  98             bar_locator,
  99             r'^37b51d194a7513e45b56f6524f2d51f2\+3\+A[a-f0-9]+@[a-f0-9]+$',
 100             'invalid locator from Keep.put("bar"): ' + bar_locator)
 101         self.assertRaises(arvados.errors.NotFoundError,
 102                           arvados.Keep.get,
 103                           unsigned_bar_locator)
 104
 105         # GET from a different user => NotFound
 106         run_test_server.authorize_with('spectator')
 107         self.assertRaises(arvados.errors.NotFoundError,
 108                           arvados.Keep.get,
 109                           bar_locator)
 110
 111         # Unauthenticated GET for a signed locator => NotFound
 112         # Unauthenticated GET for an unsigned locator => NotFound
 113         with unauthenticated_client():
 114             self.assertRaises(arvados.errors.NotFoundError,
 115                               arvados.Keep.get,
 116                               bar_locator)
 117             self.assertRaises(arvados.errors.NotFoundError,
 118                               arvados.Keep.get,
 119                               unsigned_bar_locator)
 120
 121
 122 # KeepOptionalPermission: starts Keep with --permission-key-file
 123 # but not --enforce-permissions (i.e. generate signatures on PUT
 124 # requests, but do not require them for GET requests)
 125 #
 126 # All of these requests should succeed when permissions are optional:
 127 # * authenticated request, signed locator
 128 # * authenticated request, unsigned locator
 129 # * unauthenticated request, signed locator
 130 # * unauthenticated request, unsigned locator
 131 class KeepOptionalPermission(run_test_server.TestCaseWithServers):
 132     MAIN_SERVER = {}
 133     KEEP_SERVER = {'blob_signing_key': 'abcdefghijk0123456789',
 134                    'enforce_permissions': False}
 135
 136     def test_KeepAuthenticatedSignedTest(self):
 137         run_test_server.authorize_with('active')
 138         signed_locator = arvados.Keep.put('foo')
 139         self.assertRegexpMatches(
 140             signed_locator,
 141             r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
 142             'invalid locator from Keep.put("foo"): ' + signed_locator)
 143         self.assertEqual(arvados.Keep.get(signed_locator),
 144                          'foo',
 145                          'wrong content from Keep.get(md5("foo"))')
 146
 147     def test_KeepAuthenticatedUnsignedTest(self):
 148         run_test_server.authorize_with('active')
 149         signed_locator = arvados.Keep.put('foo')
 150         self.assertRegexpMatches(
 151             signed_locator,
 152             r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
 153             'invalid locator from Keep.put("foo"): ' + signed_locator)
 154         self.assertEqual(arvados.Keep.get("acbd18db4cc2f85cedef654fccc4a4d8"),
 155                          'foo',
 156                          'wrong content from Keep.get(md5("foo"))')
 157
 158     def test_KeepUnauthenticatedSignedTest(self):
 159         # Since --enforce-permissions is not in effect, GET requests
 160         # need not be authenticated.
 161         run_test_server.authorize_with('active')
 162         signed_locator = arvados.Keep.put('foo')
 163         self.assertRegexpMatches(
 164             signed_locator,
 165             r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
 166             'invalid locator from Keep.put("foo"): ' + signed_locator)
 167
 168         with unauthenticated_client():
 169             self.assertEqual(arvados.Keep.get(signed_locator),
 170                              'foo',
 171                              'wrong content from Keep.get(md5("foo"))')
 172
 173     def test_KeepUnauthenticatedUnsignedTest(self):
 174         # Since --enforce-permissions is not in effect, GET requests
 175         # need not be authenticated.
 176         run_test_server.authorize_with('active')
 177         signed_locator = arvados.Keep.put('foo')
 178         self.assertRegexpMatches(
 179             signed_locator,
 180             r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
 181             'invalid locator from Keep.put("foo"): ' + signed_locator)
 182
 183         with unauthenticated_client():
 184             self.assertEqual(arvados.Keep.get("acbd18db4cc2f85cedef654fccc4a4d8"),
 185                              'foo',
 186                              'wrong content from Keep.get(md5("foo"))')
 187
 188
 189 class KeepProxyTestCase(run_test_server.TestCaseWithServers):
 190     MAIN_SERVER = {}
 191     KEEP_SERVER = {}
 192     KEEP_PROXY_SERVER = {'auth': 'admin'}
 193
 194     @classmethod
 195     def setUpClass(cls):
 196         super(KeepProxyTestCase, cls).setUpClass()
 197         cls.proxy_addr = os.environ['ARVADOS_KEEP_PROXY']
 198
 199     def setUp(self):
 200         arvados.keep.global_client_object = None
 201         os.environ['ARVADOS_KEEP_PROXY'] = self.proxy_addr
 202         os.environ.pop('ARVADOS_EXTERNAL_CLIENT', None)
 203
 204     def test_KeepProxyTest1(self):
 205         # Will use ARVADOS_KEEP_PROXY environment variable that is set by
 206         # setUpClass().
 207         baz_locator = arvados.Keep.put('baz')
 208         self.assertRegexpMatches(
 209             baz_locator,
 210             '^73feffa4b7f6bb68e44cf984c85f6e88\+3',
 211             'wrong md5 hash from Keep.put("baz"): ' + baz_locator)
 212         self.assertEqual(arvados.Keep.get(baz_locator),
 213                          'baz',
 214                          'wrong content from Keep.get(md5("baz"))')
 215
 216         self.assertEqual(True, arvados.Keep.global_client_object().using_proxy)
 217
 218     def test_KeepProxyTest2(self):
 219         # We don't want to use ARVADOS_KEEP_PROXY from run_keep_proxy() in
 220         # setUpClass(), so clear it and set ARVADOS_EXTERNAL_CLIENT which will
 221         # contact the API server.
 222         os.environ["ARVADOS_KEEP_PROXY"] = ""
 223         os.environ["ARVADOS_EXTERNAL_CLIENT"] = "true"
 224
 225         # Will send X-External-Client to server and get back the proxy from
 226         # keep_services/accessible
 227         baz_locator = arvados.Keep.put('baz2')
 228         self.assertRegexpMatches(
 229             baz_locator,
 230             '^91f372a266fe2bf2823cb8ec7fda31ce\+4',
 231             'wrong md5 hash from Keep.put("baz2"): ' + baz_locator)
 232         self.assertEqual(arvados.Keep.get(baz_locator),
 233                          'baz2',
 234                          'wrong content from Keep.get(md5("baz2"))')
 235
 236         self.assertEqual(True, arvados.Keep.global_client_object().using_proxy)