21721: Remove tests_require=mock

[arvados.git] / sdk / python / tests / test_collections.py

diff --git a/sdk/python/tests/test_collections.py b/sdk/python/tests/test_collections.py
index 5cf4993b2f3804d22209ae16db41fc7bc505efd8..65b89056bb40837a133b6a793ddbfe2382f3de83 100644 (file)
--- a/sdk/python/tests/test_collections.py
+++ b/sdk/python/tests/test_collections.py
@@ -7,7 +7,6 @@ from __future__ import absolute_import
 from builtins import object
 import arvados
 import copy
-import mock
 import os
 import random
 import re
@@ -16,11 +15,15 @@ import datetime
 import ciso8601
 import time
 import unittest
+import parameterized
+
+from unittest import mock
 
 from . import run_test_server
 from arvados._ranges import Range, LocatorAndRange
 from arvados.collection import Collection, CollectionReader
 from . import arvados_testutil as tutil
+from .arvados_testutil import make_block_cache
 
 class TestResumableWriter(arvados.ResumableCollectionWriter):
     KEEP_BLOCK_SIZE = 1024  # PUT to Keep every 1K.
@@ -28,9 +31,10 @@ class TestResumableWriter(arvados.ResumableCollectionWriter):
     def current_state(self):
         return self.dump_state(copy.deepcopy)
 
-
+@parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
 class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
                              tutil.ArvadosBaseTestCase):
+    disk_cache = False
     MAIN_SERVER = {}
 
     @classmethod
@@ -40,7 +44,8 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
         run_test_server.authorize_with('admin')
         cls.api_client = arvados.api('v1')
         cls.keep_client = arvados.KeepClient(api_client=cls.api_client,
-                                             local_store=cls.local_store)
+                                             local_store=cls.local_store,
+                                             block_cache=make_block_cache(cls.disk_cache))
 
     def write_foo_bar_baz(self):
         cw = arvados.CollectionWriter(self.api_client)
@@ -319,6 +324,7 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
     class MockKeep(object):
         def __init__(self, content, num_retries=0):
             self.content = content
+            self.num_prefetch_threads = 1
 
         def get(self, locator, num_retries=0, prefetch=False):
             return self.content[locator]
@@ -534,11 +540,11 @@ class CollectionReaderTestCase(unittest.TestCase, CollectionTestMixin):
         self.mock_get_collection(client, status, 'foo_file')
         return client
 
-    def test_init_no_default_retries(self):
+    def test_init_default_retries(self):
         client = self.api_client_mock(200)
         reader = arvados.CollectionReader(self.DEFAULT_UUID, api_client=client)
         reader.manifest_text()
-        client.collections().get().execute.assert_called_with(num_retries=0)
+        client.collections().get().execute.assert_called_with(num_retries=10)
 
     def test_uuid_init_success(self):
         client = self.api_client_mock(200)
@@ -588,7 +594,7 @@ class CollectionReaderTestCase(unittest.TestCase, CollectionTestMixin):
         # Ensure stripped_manifest() doesn't mangle our manifest in
         # any way other than stripping hints.
         self.assertEqual(
-            re.sub('\+[^\d\s\+]+', '', nonnormal),
+            re.sub(r'\+[^\d\s\+]+', '', nonnormal),
             reader.stripped_manifest())
         # Ensure stripped_manifest() didn't mutate our reader.
         self.assertEqual(nonnormal, reader.manifest_text())
@@ -896,7 +902,7 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
         c1.save_new()
         loc = c1.manifest_locator()
         c2 = Collection(loc)
-        self.assertEqual(c1.manifest_text, c2.manifest_text)
+        self.assertEqual(c1.manifest_text(strip=True), c2.manifest_text(strip=True))
         self.assertEqual(c1.replication_desired, c2.replication_desired)
 
     def test_replication_desired_not_loaded_if_provided(self):
@@ -905,7 +911,7 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
         c1.save_new()
         loc = c1.manifest_locator()
         c2 = Collection(loc, replication_desired=2)
-        self.assertEqual(c1.manifest_text, c2.manifest_text)
+        self.assertEqual(c1.manifest_text(strip=True), c2.manifest_text(strip=True))
         self.assertNotEqual(c1.replication_desired, c2.replication_desired)
 
     def test_storage_classes_desired_kept_on_load(self):
@@ -914,7 +920,7 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
         c1.save_new()
         loc = c1.manifest_locator()
         c2 = Collection(loc)
-        self.assertEqual(c1.manifest_text, c2.manifest_text)
+        self.assertEqual(c1.manifest_text(strip=True), c2.manifest_text(strip=True))
         self.assertEqual(c1.storage_classes_desired(), c2.storage_classes_desired())
 
     def test_storage_classes_change_after_save(self):
@@ -927,7 +933,7 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
         c2.save(storage_classes=['highIO'])
         self.assertEqual(['highIO'], c2.storage_classes_desired())
         c3 = Collection(loc)
-        self.assertEqual(c1.manifest_text, c3.manifest_text)
+        self.assertEqual(c1.manifest_text(strip=True), c3.manifest_text(strip=True))
         self.assertEqual(['highIO'], c3.storage_classes_desired())
 
     def test_storage_classes_desired_not_loaded_if_provided(self):
@@ -936,7 +942,7 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
         c1.save_new()
         loc = c1.manifest_locator()
         c2 = Collection(loc, storage_classes_desired=['default'])
-        self.assertEqual(c1.manifest_text, c2.manifest_text)
+        self.assertEqual(c1.manifest_text(strip=True), c2.manifest_text(strip=True))
         self.assertNotEqual(c1.storage_classes_desired(), c2.storage_classes_desired())
 
     def test_init_manifest(self):
@@ -969,6 +975,20 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
         with self.assertRaises(arvados.errors.ArgumentError):
             c.remove("")
 
+    def test_remove_recursive(self):
+        c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:a/b/c/d/efg.txt 0:10:xyz.txt\n')
+        self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:xyz.txt\n./a/b/c/d 781e5e245d69b566979b86e28d23f2c7+10 0:10:efg.txt\n", c.portable_manifest_text())
+        self.assertIn("a", c)
+        self.assertEqual(1, len(c["a"].keys()))
+        # cannot remove non-empty directory with default recursive=False
+        with self.assertRaises(OSError):
+            c.remove("a/b")
+        with self.assertRaises(OSError):
+            c.remove("a/b/c/d")
+        c.remove("a/b", recursive=True)
+        self.assertEqual(0, len(c["a"].keys()))
+        self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:xyz.txt\n./a d41d8cd98f00b204e9800998ecf8427e+0 0:0:\\056\n", c.portable_manifest_text())
+
     def test_find(self):
         c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n')
         self.assertIs(c.find("."), c)