sdk/python/tests/test_keep_client.py

   1 # usage example:
   2 #
   3 # ARVADOS_API_TOKEN=abc ARVADOS_API_HOST=arvados.local python -m unittest discover
   4
   5 import unittest
   6 import arvados
   7 import os
   8 import run_test_server
   9
  10 class KeepTestCase(unittest.TestCase):
  11     @classmethod
  12     def setUpClass(cls):
  13         super(KeepTestCase, cls).setUpClass()
  14         try:
  15             del os.environ['KEEP_LOCAL_STORE']
  16         except KeyError:
  17             pass
  18
  19         # Make sure these are clear, we want to talk to the Keep servers
  20         # directly.
  21         os.environ["ARVADOS_KEEP_PROXY"] = ""
  22         os.environ["ARVADOS_EXTERNAL_CLIENT"] = ""
  23
  24         run_test_server.run()
  25         run_test_server.run_keep()
  26         arvados.keep.global_client_object = None
  27         arvados.config._settings = None
  28         run_test_server.authorize_with("admin")
  29
  30     @classmethod
  31     def tearDownClass(cls):
  32         super(KeepTestCase, cls).tearDownClass()
  33         run_test_server.stop()
  34         run_test_server.stop_keep()
  35
  36     def test_KeepBasicRWTest(self):
  37         foo_locator = arvados.Keep.put('foo')
  38         self.assertEqual(foo_locator,
  39                          'acbd18db4cc2f85cedef654fccc4a4d8+3',
  40                          'wrong md5 hash from Keep.put("foo"): ' + foo_locator)
  41         self.assertEqual(arvados.Keep.get(foo_locator),
  42                          'foo',
  43                          'wrong content from Keep.get(md5("foo"))')
  44
  45     def test_KeepBinaryRWTest(self):
  46         blob_str = '\xff\xfe\xf7\x00\x01\x02'
  47         blob_locator = arvados.Keep.put(blob_str)
  48         self.assertEqual(blob_locator,
  49                          '7fc7c53b45e53926ba52821140fef396+6',
  50                          ('wrong locator from Keep.put(<binarydata>):' +
  51                           blob_locator))
  52         self.assertEqual(arvados.Keep.get(blob_locator),
  53                          blob_str,
  54                          'wrong content from Keep.get(md5(<binarydata>))')
  55
  56     def test_KeepLongBinaryRWTest(self):
  57         blob_str = '\xff\xfe\xfd\xfc\x00\x01\x02\x03'
  58         for i in range(0,23):
  59             blob_str = blob_str + blob_str
  60         blob_locator = arvados.Keep.put(blob_str)
  61         self.assertEqual(blob_locator,
  62                          '84d90fc0d8175dd5dcfab04b999bc956+67108864',
  63                          ('wrong locator from Keep.put(<binarydata>): ' +
  64                           blob_locator))
  65         self.assertEqual(arvados.Keep.get(blob_locator),
  66                          blob_str,
  67                          'wrong content from Keep.get(md5(<binarydata>))')
  68
  69     def test_KeepSingleCopyRWTest(self):
  70         blob_str = '\xff\xfe\xfd\xfc\x00\x01\x02\x03'
  71         blob_locator = arvados.Keep.put(blob_str, copies=1)
  72         self.assertEqual(blob_locator,
  73                          'c902006bc98a3eb4a3663b65ab4a6fab+8',
  74                          ('wrong locator from Keep.put(<binarydata>): ' +
  75                           blob_locator))
  76         self.assertEqual(arvados.Keep.get(blob_locator),
  77                          blob_str,
  78                          'wrong content from Keep.get(md5(<binarydata>))')
  79
  80 class KeepPermissionTestCase(unittest.TestCase):
  81     @classmethod
  82     def setUpClass(cls):
  83         try:
  84             del os.environ['KEEP_LOCAL_STORE']
  85         except KeyError:
  86             pass
  87
  88         run_test_server.run()
  89         run_test_server.run_keep(blob_signing_key='abcdefghijk0123456789',
  90                                  enforce_permissions=True)
  91
  92     @classmethod
  93     def tearDownClass(cls):
  94         run_test_server.stop()
  95         run_test_server.stop_keep()
  96
  97     def test_KeepBasicRWTest(self):
  98         run_test_server.authorize_with('active')
  99         foo_locator = arvados.Keep.put('foo')
 100         self.assertRegexpMatches(
 101             foo_locator,
 102             r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
 103             'invalid locator from Keep.put("foo"): ' + foo_locator)
 104         self.assertEqual(arvados.Keep.get(foo_locator),
 105                          'foo',
 106                          'wrong content from Keep.get(md5("foo"))')
 107
 108         # With Keep permissions enabled, a GET request without a signature will fail.
 109         bar_locator = arvados.Keep.put('bar')
 110         self.assertRegexpMatches(
 111             bar_locator,
 112             r'^37b51d194a7513e45b56f6524f2d51f2\+3\+A[a-f0-9]+@[a-f0-9]+$',
 113             'invalid locator from Keep.put("bar"): ' + bar_locator)
 114         self.assertRaises(arvados.errors.NotFoundError,
 115                           arvados.Keep.get,
 116                           "37b51d194a7513e45b56f6524f2d51f2")
 117
 118         # A request without an API token will also fail.
 119         del arvados.config.settings()["ARVADOS_API_TOKEN"]
 120         self.assertRaises(arvados.errors.NotFoundError,
 121                           arvados.Keep.get,
 122                           bar_locator)
 123
 124 # KeepOptionalPermission: starts Keep with --permission-key-file
 125 # but not --enforce-permissions (i.e. generate signatures on PUT
 126 # requests, but do not require them for GET requests)
 127 #
 128 # All of these requests should succeed when permissions are optional:
 129 # * authenticated request, signed locator
 130 # * authenticated request, unsigned locator
 131 # * unauthenticated request, signed locator
 132 # * unauthenticated request, unsigned locator
 133
 134 class KeepOptionalPermission(unittest.TestCase):
 135     @classmethod
 136     def setUpClass(cls):
 137         try:
 138             del os.environ['KEEP_LOCAL_STORE']
 139         except KeyError:
 140             pass
 141         run_test_server.run()
 142         run_test_server.run_keep(blob_signing_key='abcdefghijk0123456789',
 143                                  enforce_permissions=False)
 144
 145     @classmethod
 146     def tearDownClass(cls):
 147         run_test_server.stop()
 148         run_test_server.stop_keep()
 149
 150     def test_KeepAuthenticatedSignedTest(self):
 151         run_test_server.authorize_with('active')
 152         signed_locator = arvados.Keep.put('foo')
 153         self.assertRegexpMatches(
 154             signed_locator,
 155             r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
 156             'invalid locator from Keep.put("foo"): ' + signed_locator)
 157         self.assertEqual(arvados.Keep.get(signed_locator),
 158                          'foo',
 159                          'wrong content from Keep.get(md5("foo"))')
 160
 161     def test_KeepAuthenticatedUnsignedTest(self):
 162         run_test_server.authorize_with('active')
 163         signed_locator = arvados.Keep.put('foo')
 164         self.assertRegexpMatches(
 165             signed_locator,
 166             r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
 167             'invalid locator from Keep.put("foo"): ' + signed_locator)
 168         self.assertEqual(arvados.Keep.get("acbd18db4cc2f85cedef654fccc4a4d8"),
 169                          'foo',
 170                          'wrong content from Keep.get(md5("foo"))')
 171
 172     def test_KeepUnauthenticatedSignedTest(self):
 173         # Since --enforce-permissions is not in effect, GET requests
 174         # need not be authenticated.
 175         run_test_server.authorize_with('active')
 176         signed_locator = arvados.Keep.put('foo')
 177         self.assertRegexpMatches(
 178             signed_locator,
 179             r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
 180             'invalid locator from Keep.put("foo"): ' + signed_locator)
 181
 182         del arvados.config.settings()["ARVADOS_API_TOKEN"]
 183         self.assertEqual(arvados.Keep.get(signed_locator),
 184                          'foo',
 185                          'wrong content from Keep.get(md5("foo"))')
 186
 187     def test_KeepUnauthenticatedUnsignedTest(self):
 188         # Since --enforce-permissions is not in effect, GET requests
 189         # need not be authenticated.
 190         run_test_server.authorize_with('active')
 191         signed_locator = arvados.Keep.put('foo')
 192         self.assertRegexpMatches(
 193             signed_locator,
 194             r'^acbd18db4cc2f85cedef654fccc4a4d8\+3\+A[a-f0-9]+@[a-f0-9]+$',
 195             'invalid locator from Keep.put("foo"): ' + signed_locator)
 196
 197         del arvados.config.settings()["ARVADOS_API_TOKEN"]
 198         self.assertEqual(arvados.Keep.get("acbd18db4cc2f85cedef654fccc4a4d8"),
 199                          'foo',
 200                          'wrong content from Keep.get(md5("foo"))')
 201
 202
 203 class KeepProxyTestCase(unittest.TestCase):
 204     @classmethod
 205     def setUpClass(cls):
 206         super(KeepProxyTestCase, cls).setUpClass()
 207
 208         try:
 209             del os.environ['KEEP_LOCAL_STORE']
 210         except KeyError:
 211             pass
 212
 213         os.environ["ARVADOS_KEEP_PROXY"] = ""
 214         os.environ["ARVADOS_EXTERNAL_CLIENT"] = ""
 215
 216         run_test_server.run()
 217         run_test_server.run_keep()
 218         arvados.keep.global_client_object = None
 219         arvados.config._settings = None
 220         run_test_server.run_keep_proxy("admin")
 221         KeepProxyTestCase.arvados_keep_proxy = arvados.config.get("ARVADOS_KEEP_PROXY")
 222
 223     @classmethod
 224     def tearDownClass(cls):
 225         super(KeepProxyTestCase, cls).tearDownClass()
 226         run_test_server.stop()
 227         run_test_server.stop_keep()
 228         run_test_server.stop_keep_proxy()
 229
 230     def test_KeepProxyTest1(self):
 231         # Will use ARVADOS_KEEP_PROXY environment variable that is set by
 232         # run_keep_proxy() in setUpClass()
 233
 234         os.environ["ARVADOS_KEEP_PROXY"] = KeepProxyTestCase.arvados_keep_proxy
 235         os.environ["ARVADOS_EXTERNAL_CLIENT"] = ""
 236         arvados.keep.global_client_object = None
 237         arvados.config._settings = None
 238
 239         baz_locator = arvados.Keep.put('baz')
 240         self.assertEqual(baz_locator,
 241                          '73feffa4b7f6bb68e44cf984c85f6e88+3',
 242                          'wrong md5 hash from Keep.put("baz"): ' + baz_locator)
 243         self.assertEqual(arvados.Keep.get(baz_locator),
 244                          'baz',
 245                          'wrong content from Keep.get(md5("baz"))')
 246
 247         self.assertEqual(True, arvados.Keep.global_client_object().using_proxy)
 248
 249     def test_KeepProxyTest2(self):
 250         # We don't want to use ARVADOS_KEEP_PROXY from run_keep_proxy() in
 251         # setUpClass(), so clear it and set ARVADOS_EXTERNAL_CLIENT which will
 252         # contact the API server.
 253         os.environ["ARVADOS_KEEP_PROXY"] = ""
 254         os.environ["ARVADOS_EXTERNAL_CLIENT"] = "true"
 255         arvados.keep.global_client_object = None
 256         arvados.config._settings = None
 257
 258         # Will send X-External-Client to server and get back the proxy from
 259         # keep_services/accessible
 260
 261         baz_locator = arvados.Keep.put('baz2')
 262         self.assertEqual(baz_locator,
 263                          '91f372a266fe2bf2823cb8ec7fda31ce+4',
 264                          'wrong md5 hash from Keep.put("baz2"): ' + baz_locator)
 265         self.assertEqual(arvados.Keep.get(baz_locator),
 266                          'baz2',
 267                          'wrong content from Keep.get(md5("baz2"))')
 268
 269         self.assertEqual(True, arvados.Keep.global_client_object().using_proxy)