X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/744bbbafa5dcbba814391eaedfa9489c3614b644..404f868129e22cd627d350d61f74806f5b31a8ad:/sdk/python/tests/test_collections.py diff --git a/sdk/python/tests/test_collections.py b/sdk/python/tests/test_collections.py index f821ff952f..9e753506b3 100644 --- a/sdk/python/tests/test_collections.py +++ b/sdk/python/tests/test_collections.py @@ -16,11 +16,13 @@ import datetime import ciso8601 import time import unittest +import parameterized from . import run_test_server from arvados._ranges import Range, LocatorAndRange from arvados.collection import Collection, CollectionReader from . import arvados_testutil as tutil +from .arvados_testutil import make_block_cache class TestResumableWriter(arvados.ResumableCollectionWriter): KEEP_BLOCK_SIZE = 1024 # PUT to Keep every 1K. @@ -28,9 +30,10 @@ class TestResumableWriter(arvados.ResumableCollectionWriter): def current_state(self): return self.dump_state(copy.deepcopy) - +@parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}]) class ArvadosCollectionsTest(run_test_server.TestCaseWithServers, tutil.ArvadosBaseTestCase): + disk_cache = False MAIN_SERVER = {} @classmethod @@ -40,7 +43,8 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers, run_test_server.authorize_with('admin') cls.api_client = arvados.api('v1') cls.keep_client = arvados.KeepClient(api_client=cls.api_client, - local_store=cls.local_store) + local_store=cls.local_store, + block_cache=make_block_cache(cls.disk_cache)) def write_foo_bar_baz(self): cw = arvados.CollectionWriter(self.api_client) @@ -319,8 +323,9 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers, class MockKeep(object): def __init__(self, content, num_retries=0): self.content = content + self.num_prefetch_threads = 1 - def get(self, locator, num_retries=0): + def get(self, locator, num_retries=0, prefetch=False): return self.content[locator] def test_stream_reader(self): @@ -534,11 +539,11 @@ class CollectionReaderTestCase(unittest.TestCase, CollectionTestMixin): self.mock_get_collection(client, status, 'foo_file') return client - def test_init_no_default_retries(self): + def test_init_default_retries(self): client = self.api_client_mock(200) reader = arvados.CollectionReader(self.DEFAULT_UUID, api_client=client) reader.manifest_text() - client.collections().get().execute.assert_called_with(num_retries=0) + client.collections().get().execute.assert_called_with(num_retries=10) def test_uuid_init_success(self): client = self.api_client_mock(200) @@ -588,7 +593,7 @@ class CollectionReaderTestCase(unittest.TestCase, CollectionTestMixin): # Ensure stripped_manifest() doesn't mangle our manifest in # any way other than stripping hints. self.assertEqual( - re.sub('\+[^\d\s\+]+', '', nonnormal), + re.sub(r'\+[^\d\s\+]+', '', nonnormal), reader.stripped_manifest()) # Ensure stripped_manifest() didn't mutate our reader. self.assertEqual(nonnormal, reader.manifest_text()) @@ -896,7 +901,7 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin): c1.save_new() loc = c1.manifest_locator() c2 = Collection(loc) - self.assertEqual(c1.manifest_text, c2.manifest_text) + self.assertEqual(c1.manifest_text(strip=True), c2.manifest_text(strip=True)) self.assertEqual(c1.replication_desired, c2.replication_desired) def test_replication_desired_not_loaded_if_provided(self): @@ -905,7 +910,7 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin): c1.save_new() loc = c1.manifest_locator() c2 = Collection(loc, replication_desired=2) - self.assertEqual(c1.manifest_text, c2.manifest_text) + self.assertEqual(c1.manifest_text(strip=True), c2.manifest_text(strip=True)) self.assertNotEqual(c1.replication_desired, c2.replication_desired) def test_storage_classes_desired_kept_on_load(self): @@ -914,7 +919,7 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin): c1.save_new() loc = c1.manifest_locator() c2 = Collection(loc) - self.assertEqual(c1.manifest_text, c2.manifest_text) + self.assertEqual(c1.manifest_text(strip=True), c2.manifest_text(strip=True)) self.assertEqual(c1.storage_classes_desired(), c2.storage_classes_desired()) def test_storage_classes_change_after_save(self): @@ -927,7 +932,7 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin): c2.save(storage_classes=['highIO']) self.assertEqual(['highIO'], c2.storage_classes_desired()) c3 = Collection(loc) - self.assertEqual(c1.manifest_text, c3.manifest_text) + self.assertEqual(c1.manifest_text(strip=True), c3.manifest_text(strip=True)) self.assertEqual(['highIO'], c3.storage_classes_desired()) def test_storage_classes_desired_not_loaded_if_provided(self): @@ -936,7 +941,7 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin): c1.save_new() loc = c1.manifest_locator() c2 = Collection(loc, storage_classes_desired=['default']) - self.assertEqual(c1.manifest_text, c2.manifest_text) + self.assertEqual(c1.manifest_text(strip=True), c2.manifest_text(strip=True)) self.assertNotEqual(c1.storage_classes_desired(), c2.storage_classes_desired()) def test_init_manifest(self): @@ -969,6 +974,20 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin): with self.assertRaises(arvados.errors.ArgumentError): c.remove("") + def test_remove_recursive(self): + c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:a/b/c/d/efg.txt 0:10:xyz.txt\n') + self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:xyz.txt\n./a/b/c/d 781e5e245d69b566979b86e28d23f2c7+10 0:10:efg.txt\n", c.portable_manifest_text()) + self.assertIn("a", c) + self.assertEqual(1, len(c["a"].keys())) + # cannot remove non-empty directory with default recursive=False + with self.assertRaises(OSError): + c.remove("a/b") + with self.assertRaises(OSError): + c.remove("a/b/c/d") + c.remove("a/b", recursive=True) + self.assertEqual(0, len(c["a"].keys())) + self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:xyz.txt\n./a d41d8cd98f00b204e9800998ecf8427e+0 0:0:\\056\n", c.portable_manifest_text()) + def test_find(self): c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n') self.assertIs(c.find("."), c) @@ -1360,6 +1379,25 @@ class NewCollectionTestCaseWithServersAndTokens(run_test_server.TestCaseWithServ class NewCollectionTestCaseWithServers(run_test_server.TestCaseWithServers): + def test_preserve_version_on_save(self): + c = Collection() + c.save_new(preserve_version=True) + coll_record = arvados.api().collections().get(uuid=c.manifest_locator()).execute() + self.assertEqual(coll_record['version'], 1) + self.assertEqual(coll_record['preserve_version'], True) + with c.open("foo.txt", "wb") as foo: + foo.write(b"foo") + c.save(preserve_version=True) + coll_record = arvados.api().collections().get(uuid=c.manifest_locator()).execute() + self.assertEqual(coll_record['version'], 2) + self.assertEqual(coll_record['preserve_version'], True) + with c.open("bar.txt", "wb") as foo: + foo.write(b"bar") + c.save(preserve_version=False) + coll_record = arvados.api().collections().get(uuid=c.manifest_locator()).execute() + self.assertEqual(coll_record['version'], 3) + self.assertEqual(coll_record['preserve_version'], False) + def test_get_manifest_text_only_committed(self): c = Collection() with c.open("count.txt", "wb") as f: