sdk/python/tests/test_keep_client.py

   1 # usage example:
   2 #
   3 # ARVADOS_API_TOKEN=abc ARVADOS_API_HOST=arvados.local python -m unittest discover
   4
   5 import os
   6 import unittest
   7
   8 import arvados
   9 import run_test_server
  10
  11 class KeepTestCase(unittest.TestCase):
  12     @classmethod
  13     def setUpClass(cls):
  14         super(KeepTestCase, cls).setUpClass()
  15         try:
  16             del os.environ['KEEP_LOCAL_STORE']
  17         except KeyError:
  18             pass
  19
  20         # Make sure these are clear, we want to talk to the Keep servers
  21         # directly.
  22         os.environ["ARVADOS_KEEP_PROXY"] = ""
  23         os.environ["ARVADOS_EXTERNAL_CLIENT"] = ""
  24
  25         run_test_server.run()
  26         run_test_server.run_keep()
  27         arvados.keep.global_client_object = None
  28         arvados.config._settings = None
  29         run_test_server.authorize_with("admin")
  30
  31     @classmethod
  32     def tearDownClass(cls):
  33         super(KeepTestCase, cls).tearDownClass()
  34         run_test_server.stop()
  35         run_test_server.stop_keep()
  36
  37     def test_KeepBasicRWTest(self):
  38         foo_locator = arvados.Keep.put('foo')
  39         self.assertRegexpMatches(
  40             foo_locator,
  41             '^acbd18db4cc2f85cedef654fccc4a4d8\+3',
  42             'wrong md5 hash from Keep.put("foo"): ' + foo_locator)
  43         self.assertEqual(arvados.Keep.get(foo_locator),
  44                          'foo',
  45                          'wrong content from Keep.get(md5("foo"))')
  46
  47     def test_KeepBinaryRWTest(self):
  48         blob_str = '\xff\xfe\xf7\x00\x01\x02'
  49         blob_locator = arvados.Keep.put(blob_str)
  50         self.assertRegexpMatches(
  51             blob_locator,
  52             '^7fc7c53b45e53926ba52821140fef396\+6',
  53             ('wrong locator from Keep.put(<binarydata>):' + blob_locator))
  54         self.assertEqual(arvados.Keep.get(blob_locator),
  55                          blob_str,
  56                          'wrong content from Keep.get(md5(<binarydata>))')
  57
  58     def test_KeepLongBinaryRWTest(self):
  59         blob_str = '\xff\xfe\xfd\xfc\x00\x01\x02\x03'
  60         for i in range(0,23):
  61             blob_str = blob_str + blob_str
  62         blob_locator = arvados.Keep.put(blob_str)
  63         self.assertRegexpMatches(
  64             blob_locator,
  65             '^84d90fc0d8175dd5dcfab04b999bc956\+67108864',
  66             ('wrong locator from Keep.put(<binarydata>): ' + blob_locator))
  67         self.assertEqual(arvados.Keep.get(blob_locator),
  68                          blob_str,
  69                          'wrong content from Keep.get(md5(<binarydata>))')
  70
  71     def test_KeepSingleCopyRWTest(self):
  72         blob_str = '\xff\xfe\xfd\xfc\x00\x01\x02\x03'
  73         blob_locator = arvados.Keep.put(blob_str, copies=1)
  74         self.assertRegexpMatches(
  75             blob_locator,
  76             '^c902006bc98a3eb4a3663b65ab4a6fab\+8',
  77             ('wrong locator from Keep.put(<binarydata>): ' + blob_locator))
  78         self.assertEqual(arvados.Keep.get(blob_locator),
  79                          blob_str,
  80                          'wrong content from Keep.get(md5(<binarydata>))')
  81
  82 class KeepPermissionTestCase(unittest.TestCase):
  83     @classmethod
  84     def setUpClass(cls):
  85         try:
  86             del os.environ['KEEP_LOCAL_STORE']
  87         except KeyError:
  88             pass
  89
  90         run_test_server.run()
  91         run_test_server.run_keep(blob_signing_key='abcdefghijk0123456789',
  92                                  enforce_permissions=True)
  93
  94     @classmethod
  95     def tearDownClass(cls):
  96         run_test_server.stop()
  97         run_test_server.stop_keep()
  98
  99     def test_KeepBasicRWTest(self):
 100         run_test_server.authorize_with('active')
 101         foo_locator = arvados.Keep.put('foo')
 102         self.assertRegexpMatches(
 103             foo_locator,
 104             r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
 105             'invalid locator from Keep.put("foo"): ' + foo_locator)
 106         self.assertEqual(arvados.Keep.get(foo_locator),
 107                          'foo',
 108                          'wrong content from Keep.get(md5("foo"))')
 109
 110         # GET with an unsigned locator => NotFound
 111         bar_locator = arvados.Keep.put('bar')
 112         unsigned_bar_locator = "37b51d194a7513e45b56f6524f2d51f2+3"
 113         self.assertRegexpMatches(
 114             bar_locator,
 115             r'^37b51d194a7513e45b56f6524f2d51f2\+3\+A[a-f0-9]+@[a-f0-9]+$',
 116             'invalid locator from Keep.put("bar"): ' + bar_locator)
 117         self.assertRaises(arvados.errors.NotFoundError,
 118                           arvados.Keep.get,
 119                           unsigned_bar_locator)
 120
 121         # GET from a different user => NotFound
 122         run_test_server.authorize_with('spectator')
 123         self.assertRaises(arvados.errors.NotFoundError,
 124                           arvados.Keep.get,
 125                           bar_locator)
 126
 127         # Unauthenticated GET for a signed locator => NotFound
 128         # Unauthenticated GET for an unsigned locator => NotFound
 129         del arvados.config.settings()["ARVADOS_API_TOKEN"]
 130         self.assertRaises(arvados.errors.NotFoundError,
 131                           arvados.Keep.get,
 132                           bar_locator)
 133         self.assertRaises(arvados.errors.NotFoundError,
 134                           arvados.Keep.get,
 135                           unsigned_bar_locator)
 136
 137 # KeepOptionalPermission: starts Keep with --permission-key-file
 138 # but not --enforce-permissions (i.e. generate signatures on PUT
 139 # requests, but do not require them for GET requests)
 140 #
 141 # All of these requests should succeed when permissions are optional:
 142 # * authenticated request, signed locator
 143 # * authenticated request, unsigned locator
 144 # * unauthenticated request, signed locator
 145 # * unauthenticated request, unsigned locator
 146
 147 class KeepOptionalPermission(unittest.TestCase):
 148     @classmethod
 149     def setUpClass(cls):
 150         try:
 151             del os.environ['KEEP_LOCAL_STORE']
 152         except KeyError:
 153             pass
 154         run_test_server.run()
 155         run_test_server.run_keep(blob_signing_key='abcdefghijk0123456789',
 156                                  enforce_permissions=False)
 157
 158     @classmethod
 159     def tearDownClass(cls):
 160         run_test_server.stop()
 161         run_test_server.stop_keep()
 162
 163     def test_KeepAuthenticatedSignedTest(self):
 164         run_test_server.authorize_with('active')
 165         signed_locator = arvados.Keep.put('foo')
 166         self.assertRegexpMatches(
 167             signed_locator,
 168             r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
 169             'invalid locator from Keep.put("foo"): ' + signed_locator)
 170         self.assertEqual(arvados.Keep.get(signed_locator),
 171                          'foo',
 172                          'wrong content from Keep.get(md5("foo"))')
 173
 174     def test_KeepAuthenticatedUnsignedTest(self):
 175         run_test_server.authorize_with('active')
 176         signed_locator = arvados.Keep.put('foo')
 177         self.assertRegexpMatches(
 178             signed_locator,
 179             r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
 180             'invalid locator from Keep.put("foo"): ' + signed_locator)
 181         self.assertEqual(arvados.Keep.get("acbd18db4cc2f85cedef654fccc4a4d8"),
 182                          'foo',
 183                          'wrong content from Keep.get(md5("foo"))')
 184
 185     def test_KeepUnauthenticatedSignedTest(self):
 186         # Since --enforce-permissions is not in effect, GET requests
 187         # need not be authenticated.
 188         run_test_server.authorize_with('active')
 189         signed_locator = arvados.Keep.put('foo')
 190         self.assertRegexpMatches(
 191             signed_locator,
 192             r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
 193             'invalid locator from Keep.put("foo"): ' + signed_locator)
 194
 195         del arvados.config.settings()["ARVADOS_API_TOKEN"]
 196         self.assertEqual(arvados.Keep.get(signed_locator),
 197                          'foo',
 198                          'wrong content from Keep.get(md5("foo"))')
 199
 200     def test_KeepUnauthenticatedUnsignedTest(self):
 201         # Since --enforce-permissions is not in effect, GET requests
 202         # need not be authenticated.
 203         run_test_server.authorize_with('active')
 204         signed_locator = arvados.Keep.put('foo')
 205         self.assertRegexpMatches(
 206             signed_locator,
 207             r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
 208             'invalid locator from Keep.put("foo"): ' + signed_locator)
 209
 210         del arvados.config.settings()["ARVADOS_API_TOKEN"]
 211         self.assertEqual(arvados.Keep.get("acbd18db4cc2f85cedef654fccc4a4d8"),
 212                          'foo',
 213                          'wrong content from Keep.get(md5("foo"))')
 214
 215
 216 class KeepProxyTestCase(unittest.TestCase):
 217     @classmethod
 218     def setUpClass(cls):
 219         super(KeepProxyTestCase, cls).setUpClass()
 220
 221         try:
 222             del os.environ['KEEP_LOCAL_STORE']
 223         except KeyError:
 224             pass
 225
 226         os.environ["ARVADOS_KEEP_PROXY"] = ""
 227         os.environ["ARVADOS_EXTERNAL_CLIENT"] = ""
 228
 229         run_test_server.run()
 230         run_test_server.run_keep()
 231         arvados.keep.global_client_object = None
 232         arvados.config._settings = None
 233         run_test_server.run_keep_proxy("admin")
 234         KeepProxyTestCase.arvados_keep_proxy = arvados.config.get("ARVADOS_KEEP_PROXY")
 235
 236     @classmethod
 237     def tearDownClass(cls):
 238         super(KeepProxyTestCase, cls).tearDownClass()
 239         run_test_server.stop()
 240         run_test_server.stop_keep()
 241         run_test_server.stop_keep_proxy()
 242
 243     def test_KeepProxyTest1(self):
 244         # Will use ARVADOS_KEEP_PROXY environment variable that is set by
 245         # run_keep_proxy() in setUpClass()
 246
 247         os.environ["ARVADOS_KEEP_PROXY"] = KeepProxyTestCase.arvados_keep_proxy
 248         os.environ["ARVADOS_EXTERNAL_CLIENT"] = ""
 249         arvados.keep.global_client_object = None
 250         arvados.config._settings = None
 251
 252         baz_locator = arvados.Keep.put('baz')
 253         self.assertRegexpMatches(
 254             baz_locator,
 255             '^73feffa4b7f6bb68e44cf984c85f6e88\+3',
 256             'wrong md5 hash from Keep.put("baz"): ' + baz_locator)
 257         self.assertEqual(arvados.Keep.get(baz_locator),
 258                          'baz',
 259                          'wrong content from Keep.get(md5("baz"))')
 260
 261         self.assertEqual(True, arvados.Keep.global_client_object().using_proxy)
 262
 263     def test_KeepProxyTest2(self):
 264         # We don't want to use ARVADOS_KEEP_PROXY from run_keep_proxy() in
 265         # setUpClass(), so clear it and set ARVADOS_EXTERNAL_CLIENT which will
 266         # contact the API server.
 267         os.environ["ARVADOS_KEEP_PROXY"] = ""
 268         os.environ["ARVADOS_EXTERNAL_CLIENT"] = "true"
 269         arvados.keep.global_client_object = None
 270         arvados.config._settings = None
 271
 272         # Will send X-External-Client to server and get back the proxy from
 273         # keep_services/accessible
 274
 275         baz_locator = arvados.Keep.put('baz2')
 276         self.assertRegexpMatches(
 277             baz_locator,
 278             '^91f372a266fe2bf2823cb8ec7fda31ce\+4',
 279             'wrong md5 hash from Keep.put("baz2"): ' + baz_locator)
 280         self.assertEqual(arvados.Keep.get(baz_locator),
 281                          'baz2',
 282                          'wrong content from Keep.get(md5("baz2"))')
 283
 284         self.assertEqual(True, arvados.Keep.global_client_object().using_proxy)