sdk/python/tests/test_http_cache.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import hashlib
   6 import os
   7 import random
   8 import shutil
   9 import sys
  10 import tempfile
  11 import threading
  12 import unittest
  13
  14 import pytest
  15 from unittest import mock
  16
  17 import arvados
  18 import arvados.api
  19 import arvados.util
  20 from arvados._internal import basedirs
  21
  22 from . import run_test_server
  23
  24 def _random(n):
  25     return bytearray(random.getrandbits(8) for _ in range(n))
  26
  27 class CacheTestThread(threading.Thread):
  28     def __init__(self, dir):
  29         super(CacheTestThread, self).__init__()
  30         self._dir = dir
  31
  32     def run(self):
  33         c = arvados.api.ThreadSafeHTTPCache(self._dir)
  34         url = 'http://example.com/foo'
  35         self.ok = True
  36         for x in range(16):
  37             try:
  38                 data_in = _random(128)
  39                 data_in = hashlib.md5(data_in).hexdigest().encode() + b"\n" + data_in
  40                 c.set(url, data_in)
  41                 data_out = c.get(url)
  42                 digest, _, content = data_out.partition(b"\n")
  43                 if digest != hashlib.md5(content).hexdigest().encode():
  44                     self.ok = False
  45             except Exception as err:
  46                 self.ok = False
  47                 print("cache failed: {}: {}".format(type(err), err), file=sys.stderr)
  48                 raise
  49
  50
  51 class TestAPIHTTPCache:
  52     @pytest.mark.parametrize('data_type', ['discovery', 'keep'])
  53     def test_good_storage(self, tmp_path, monkeypatch, data_type):
  54         def storage_path(self, subdir='.', mode=0o700):
  55             path = tmp_path / subdir
  56             path.mkdir(mode=mode)
  57             return path
  58         monkeypatch.setattr(basedirs.BaseDirectories, 'storage_path', storage_path)
  59         actual = arvados.http_cache(data_type)
  60         assert str(actual) == str(tmp_path / data_type)
  61
  62     @pytest.mark.parametrize('error', [RuntimeError, FileExistsError, PermissionError])
  63     def test_unwritable_storage(self, monkeypatch, error):
  64         def fail(self, subdir='.', mode=0o700):
  65             raise error()
  66         monkeypatch.setattr(basedirs.BaseDirectories, 'storage_path', fail)
  67         actual = arvados.http_cache('unwritable')
  68         assert actual is None
  69
  70
  71 class CacheTest(unittest.TestCase):
  72     def setUp(self):
  73         self._dir = tempfile.mkdtemp()
  74
  75     def tearDown(self):
  76         shutil.rmtree(self._dir)
  77
  78     def test_cache_crud(self):
  79         c = arvados.api.ThreadSafeHTTPCache(self._dir, max_age=0)
  80         url = 'https://example.com/foo?bar=baz'
  81         data1 = _random(256)
  82         data2 = _random(128)
  83         self.assertEqual(None, c.get(url))
  84         c.delete(url)
  85         c.set(url, data1)
  86         self.assertEqual(data1, c.get(url))
  87         c.delete(url)
  88         self.assertEqual(None, c.get(url))
  89         c.set(url, data1)
  90         c.set(url, data2)
  91         self.assertEqual(data2, c.get(url))
  92
  93     def test_cache_threads(self):
  94         threads = []
  95         for _ in range(64):
  96             t = CacheTestThread(dir=self._dir)
  97             t.start()
  98             threads.append(t)
  99         for t in threads:
 100             t.join()
 101             self.assertTrue(t.ok)
 102
 103
 104 class CacheIntegrationTest(run_test_server.TestCaseWithServers):
 105     MAIN_SERVER = {}
 106
 107     def test_cache_used_by_default_client(self):
 108         with mock.patch('arvados.api.ThreadSafeHTTPCache.get') as getter:
 109             arvados.api('v1')._rootDesc.get('foobar')
 110             getter.assert_called()