sdk/cwl/tests/test_fsaccess.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import functools
   6 import sys
   7 import unittest
   8 import json
   9 import logging
  10 import os
  11
  12 from unittest import mock
  13
  14 import arvados
  15 import arvados.keep
  16 import arvados.collection
  17 import arvados_cwl
  18
  19 from cwltool.pathmapper import MapperEnt
  20 from .mock_discovery import get_rootDesc
  21
  22 from arvados_cwl.fsaccess import CollectionCache
  23
  24 class TestFsAccess(unittest.TestCase):
  25     @mock.patch("arvados.collection.CollectionReader")
  26     def test_collection_cache(self, cr):
  27         cache = CollectionCache(mock.MagicMock(), mock.MagicMock(), 4)
  28         c1 = cache.get("99999999999999999999999999999991+99")
  29         c2 = cache.get("99999999999999999999999999999991+99")
  30         self.assertIs(c1, c2)
  31         self.assertEqual(1, cr.call_count)
  32         c3 = cache.get("99999999999999999999999999999992+99")
  33         self.assertEqual(2, cr.call_count)
  34
  35     @mock.patch("arvados.collection.CollectionReader")
  36     def test_collection_cache_limit(self, cr):
  37         cache = CollectionCache(mock.MagicMock(), mock.MagicMock(), 4)
  38         cr().manifest_text.return_value = 'x' * 524289
  39         self.assertEqual(0, cache.total)
  40         c1 = cache.get("99999999999999999999999999999991+524289")
  41         self.assertIn("99999999999999999999999999999991+524289", cache.collections)
  42         self.assertNotIn("99999999999999999999999999999992+524289", cache.collections)
  43         self.assertEqual((524289*128)*1, cache.total)
  44
  45         c2 = cache.get("99999999999999999999999999999992+524289")
  46         self.assertIn("99999999999999999999999999999991+524289", cache.collections)
  47         self.assertIn("99999999999999999999999999999992+524289", cache.collections)
  48         self.assertEqual((524289*128)*2, cache.total)
  49
  50         c1 = cache.get("99999999999999999999999999999991+524289")
  51         self.assertIn("99999999999999999999999999999991+524289", cache.collections)
  52         self.assertIn("99999999999999999999999999999992+524289", cache.collections)
  53         self.assertEqual((524289*128)*2, cache.total)
  54
  55         c3 = cache.get("99999999999999999999999999999993+524289")
  56         self.assertIn("99999999999999999999999999999991+524289", cache.collections)
  57         self.assertIn("99999999999999999999999999999992+524289", cache.collections)
  58         self.assertEqual((524289*128)*3, cache.total)
  59
  60         c4 = cache.get("99999999999999999999999999999994+524289")
  61         self.assertIn("99999999999999999999999999999991+524289", cache.collections)
  62         self.assertNotIn("99999999999999999999999999999992+524289", cache.collections)
  63         self.assertEqual((524289*128)*3, cache.total)
  64
  65         c5 = cache.get("99999999999999999999999999999995+524289")
  66         self.assertNotIn("99999999999999999999999999999991+524289", cache.collections)
  67         self.assertNotIn("99999999999999999999999999999992+524289", cache.collections)
  68         self.assertEqual((524289*128)*3, cache.total)
  69
  70
  71     @mock.patch("arvados.collection.CollectionReader")
  72     def test_collection_cache_limit2(self, cr):
  73         cache = CollectionCache(mock.MagicMock(), mock.MagicMock(), 4)
  74         cr().manifest_text.return_value = 'x' * 524287
  75         self.assertEqual(0, cache.total)
  76         c1 = cache.get("99999999999999999999999999999991+524287")
  77         self.assertIn("99999999999999999999999999999991+524287", cache.collections)
  78         self.assertNotIn("99999999999999999999999999999992+524287", cache.collections)
  79         self.assertEqual((524287*128)*1, cache.total)
  80
  81         c2 = cache.get("99999999999999999999999999999992+524287")
  82         self.assertIn("99999999999999999999999999999991+524287", cache.collections)
  83         self.assertIn("99999999999999999999999999999992+524287", cache.collections)
  84         self.assertEqual((524287*128)*2, cache.total)
  85
  86         c1 = cache.get("99999999999999999999999999999991+524287")
  87         self.assertIn("99999999999999999999999999999991+524287", cache.collections)
  88         self.assertIn("99999999999999999999999999999992+524287", cache.collections)
  89         self.assertEqual((524287*128)*2, cache.total)
  90
  91         c3 = cache.get("99999999999999999999999999999993+524287")
  92         self.assertIn("99999999999999999999999999999991+524287", cache.collections)
  93         self.assertIn("99999999999999999999999999999992+524287", cache.collections)
  94         self.assertEqual((524287*128)*3, cache.total)
  95
  96         c4 = cache.get("99999999999999999999999999999994+524287")
  97         self.assertIn("99999999999999999999999999999991+524287", cache.collections)
  98         self.assertIn("99999999999999999999999999999992+524287", cache.collections)
  99         self.assertEqual((524287*128)*4, cache.total)
 100
 101         c5 = cache.get("99999999999999999999999999999995+524287")
 102         self.assertIn("99999999999999999999999999999991+524287", cache.collections)
 103         self.assertNotIn("99999999999999999999999999999992+524287", cache.collections)
 104         self.assertEqual((524287*128)*4, cache.total)
 105
 106         c6 = cache.get("99999999999999999999999999999996+524287")
 107         self.assertNotIn("99999999999999999999999999999991+524287", cache.collections)
 108         self.assertNotIn("99999999999999999999999999999992+524287", cache.collections)
 109         self.assertEqual((524287*128)*4, cache.total)