sdk/python/tests/test_cache.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import hashlib
   6 import os
   7 import random
   8 import shutil
   9 import sys
  10 import tempfile
  11 import threading
  12 import unittest
  13
  14 import pytest
  15 from unittest import mock
  16
  17 import arvados
  18 import arvados.cache
  19 import arvados.util
  20 from . import run_test_server
  21
  22 def _random(n):
  23     return bytearray(random.getrandbits(8) for _ in range(n))
  24
  25
  26 class CacheTestThread(threading.Thread):
  27     def __init__(self, dir):
  28         super(CacheTestThread, self).__init__()
  29         self._dir = dir
  30
  31     def run(self):
  32         c = arvados.cache.SafeHTTPCache(self._dir)
  33         url = 'http://example.com/foo'
  34         self.ok = True
  35         for x in range(16):
  36             try:
  37                 data_in = _random(128)
  38                 data_in = hashlib.md5(data_in).hexdigest().encode() + b"\n" + data_in
  39                 c.set(url, data_in)
  40                 data_out = c.get(url)
  41                 digest, _, content = data_out.partition(b"\n")
  42                 if digest != hashlib.md5(content).hexdigest().encode():
  43                     self.ok = False
  44             except Exception as err:
  45                 self.ok = False
  46                 print("cache failed: {}: {}".format(type(err), err), file=sys.stderr)
  47                 raise
  48
  49
  50 class TestAPIHTTPCache:
  51     @pytest.mark.parametrize('data_type', ['discovery', 'keep'])
  52     def test_good_storage(self, tmp_path, monkeypatch, data_type):
  53         def storage_path(self, subdir='.', mode=0o700):
  54             path = tmp_path / subdir
  55             path.mkdir(mode=mode)
  56             return path
  57         monkeypatch.setattr(arvados.util._BaseDirectories, 'storage_path', storage_path)
  58         actual = arvados.http_cache(data_type)
  59         assert str(actual) == str(tmp_path / data_type)
  60
  61     @pytest.mark.parametrize('error', [RuntimeError, FileExistsError, PermissionError])
  62     def test_unwritable_storage(self, monkeypatch, error):
  63         def fail(self, subdir='.', mode=0o700):
  64             raise error()
  65         monkeypatch.setattr(arvados.util._BaseDirectories, 'storage_path', fail)
  66         actual = arvados.http_cache('unwritable')
  67         assert actual is None
  68
  69
  70 class CacheTest(unittest.TestCase):
  71     def setUp(self):
  72         self._dir = tempfile.mkdtemp()
  73
  74     def tearDown(self):
  75         shutil.rmtree(self._dir)
  76
  77     def test_cache_crud(self):
  78         c = arvados.cache.SafeHTTPCache(self._dir, max_age=0)
  79         url = 'https://example.com/foo?bar=baz'
  80         data1 = _random(256)
  81         data2 = _random(128)
  82         self.assertEqual(None, c.get(url))
  83         c.delete(url)
  84         c.set(url, data1)
  85         self.assertEqual(data1, c.get(url))
  86         c.delete(url)
  87         self.assertEqual(None, c.get(url))
  88         c.set(url, data1)
  89         c.set(url, data2)
  90         self.assertEqual(data2, c.get(url))
  91
  92     def test_cache_threads(self):
  93         threads = []
  94         for _ in range(64):
  95             t = CacheTestThread(dir=self._dir)
  96             t.start()
  97             threads.append(t)
  98         for t in threads:
  99             t.join()
 100             self.assertTrue(t.ok)
 101
 102
 103 class CacheIntegrationTest(run_test_server.TestCaseWithServers):
 104     MAIN_SERVER = {}
 105
 106     def test_cache_used_by_default_client(self):
 107         with mock.patch('arvados.cache.SafeHTTPCache.get') as getter:
 108             arvados.api('v1')._rootDesc.get('foobar')
 109             getter.assert_called()