Merge branch '18842-arv-mount-disk-config' refs #18842

[arvados.git] / sdk / python / tests / test_collections.py

diff --git a/sdk/python/tests/test_collections.py b/sdk/python/tests/test_collections.py
index a43e0d40dfe7ed48f5477689d3623afefe952ba3..8986cf225840054bc5cd4161f7edd0b2c3f58b32 100644 (file)
--- a/sdk/python/tests/test_collections.py
+++ b/sdk/python/tests/test_collections.py
@@ -16,11 +16,13 @@ import datetime
 import ciso8601
 import time
 import unittest
+import parameterized
 
 from . import run_test_server
 from arvados._ranges import Range, LocatorAndRange
 from arvados.collection import Collection, CollectionReader
 from . import arvados_testutil as tutil
+from .arvados_testutil import make_block_cache
 
 class TestResumableWriter(arvados.ResumableCollectionWriter):
     KEEP_BLOCK_SIZE = 1024  # PUT to Keep every 1K.
@@ -28,9 +30,10 @@ class TestResumableWriter(arvados.ResumableCollectionWriter):
     def current_state(self):
         return self.dump_state(copy.deepcopy)
 
-
+@parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
 class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
                              tutil.ArvadosBaseTestCase):
+    disk_cache = False
     MAIN_SERVER = {}
 
     @classmethod
@@ -40,7 +43,8 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
         run_test_server.authorize_with('admin')
         cls.api_client = arvados.api('v1')
         cls.keep_client = arvados.KeepClient(api_client=cls.api_client,
-                                             local_store=cls.local_store)
+                                             local_store=cls.local_store,
+                                             block_cache=make_block_cache(cls.disk_cache))
 
     def write_foo_bar_baz(self):
         cw = arvados.CollectionWriter(self.api_client)
@@ -320,7 +324,7 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
         def __init__(self, content, num_retries=0):
             self.content = content
 
-        def get(self, locator, num_retries=0):
+        def get(self, locator, num_retries=0, prefetch=False):
             return self.content[locator]
 
     def test_stream_reader(self):
@@ -896,7 +900,7 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
         c1.save_new()
         loc = c1.manifest_locator()
         c2 = Collection(loc)
-        self.assertEqual(c1.manifest_text, c2.manifest_text)
+        self.assertEqual(c1.manifest_text(strip=True), c2.manifest_text(strip=True))
         self.assertEqual(c1.replication_desired, c2.replication_desired)
 
     def test_replication_desired_not_loaded_if_provided(self):
@@ -905,7 +909,7 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
         c1.save_new()
         loc = c1.manifest_locator()
         c2 = Collection(loc, replication_desired=2)
-        self.assertEqual(c1.manifest_text, c2.manifest_text)
+        self.assertEqual(c1.manifest_text(strip=True), c2.manifest_text(strip=True))
         self.assertNotEqual(c1.replication_desired, c2.replication_desired)
 
     def test_storage_classes_desired_kept_on_load(self):
@@ -914,7 +918,7 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
         c1.save_new()
         loc = c1.manifest_locator()
         c2 = Collection(loc)
-        self.assertEqual(c1.manifest_text, c2.manifest_text)
+        self.assertEqual(c1.manifest_text(strip=True), c2.manifest_text(strip=True))
         self.assertEqual(c1.storage_classes_desired(), c2.storage_classes_desired())
 
     def test_storage_classes_change_after_save(self):
@@ -927,7 +931,7 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
         c2.save(storage_classes=['highIO'])
         self.assertEqual(['highIO'], c2.storage_classes_desired())
         c3 = Collection(loc)
-        self.assertEqual(c1.manifest_text, c3.manifest_text)
+        self.assertEqual(c1.manifest_text(strip=True), c3.manifest_text(strip=True))
         self.assertEqual(['highIO'], c3.storage_classes_desired())
 
     def test_storage_classes_desired_not_loaded_if_provided(self):
@@ -936,7 +940,7 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
         c1.save_new()
         loc = c1.manifest_locator()
         c2 = Collection(loc, storage_classes_desired=['default'])
-        self.assertEqual(c1.manifest_text, c2.manifest_text)
+        self.assertEqual(c1.manifest_text(strip=True), c2.manifest_text(strip=True))
         self.assertNotEqual(c1.storage_classes_desired(), c2.storage_classes_desired())
 
     def test_init_manifest(self):
@@ -969,6 +973,20 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
         with self.assertRaises(arvados.errors.ArgumentError):
             c.remove("")
 
+    def test_remove_recursive(self):
+        c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:a/b/c/d/efg.txt 0:10:xyz.txt\n')
+        self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:xyz.txt\n./a/b/c/d 781e5e245d69b566979b86e28d23f2c7+10 0:10:efg.txt\n", c.portable_manifest_text())
+        self.assertIn("a", c)
+        self.assertEqual(1, len(c["a"].keys()))
+        # cannot remove non-empty directory with default recursive=False
+        with self.assertRaises(OSError):
+            c.remove("a/b")
+        with self.assertRaises(OSError):
+            c.remove("a/b/c/d")
+        c.remove("a/b", recursive=True)
+        self.assertEqual(0, len(c["a"].keys()))
+        self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:xyz.txt\n./a d41d8cd98f00b204e9800998ecf8427e+0 0:0:\\056\n", c.portable_manifest_text())
+
     def test_find(self):
         c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n')
         self.assertIs(c.find("."), c)