Merge branch 'master' of git.curoverse.com:arvados into 11876-r-sdk

[arvados.git] / sdk / python / tests / test_collections.py

diff --git a/sdk/python/tests/test_collections.py b/sdk/python/tests/test_collections.py
index c94118c1e8565140673650ab068deca04f9356f2..49c00191bebe02cc8e267b397212a893a33f246a 100644 (file)
--- a/sdk/python/tests/test_collections.py
+++ b/sdk/python/tests/test_collections.py
@@ -1,3 +1,7 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+

 from __future__ import absolute_import
 
 from builtins import object
 from __future__ import absolute_import
 
 from builtins import object


@@ -32,7 +36,8 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
     @classmethod
     def setUpClass(cls):
         super(ArvadosCollectionsTest, cls).setUpClass()
     @classmethod
     def setUpClass(cls):
         super(ArvadosCollectionsTest, cls).setUpClass()

-        run_test_server.authorize_with('active')
+        # need admin privileges to make collections with unsigned blocks
+        run_test_server.authorize_with('admin')

         cls.api_client = arvados.api('v1')
         cls.keep_client = arvados.KeepClient(api_client=cls.api_client,
                                              local_store=cls.local_store)
         cls.api_client = arvados.api('v1')
         cls.keep_client = arvados.KeepClient(api_client=cls.api_client,
                                              local_store=cls.local_store)


@@ -54,9 +59,13 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
                          ". 3858f62230ac3c915f300c664312c63f+6 0:3:foo.txt 3:3:bar.txt\n" +
                          "./baz 73feffa4b7f6bb68e44cf984c85f6e88+3 0:3:baz.txt\n",
                          "wrong manifest: got {}".format(cw.manifest_text()))
                          ". 3858f62230ac3c915f300c664312c63f+6 0:3:foo.txt 3:3:bar.txt\n" +
                          "./baz 73feffa4b7f6bb68e44cf984c85f6e88+3 0:3:baz.txt\n",
                          "wrong manifest: got {}".format(cw.manifest_text()))

-        cw.finish()
+        cw.save_new()

         return cw.portable_data_hash()
 
         return cw.portable_data_hash()
 

+    def test_pdh_is_native_str(self):
+        pdh = self.write_foo_bar_baz()
+        self.assertEqual(type(''), type(pdh))
+

     def test_keep_local_store(self):
         self.assertEqual(self.keep_client.put(b'foo'), 'acbd18db4cc2f85cedef654fccc4a4d8+3', 'wrong md5 hash from Keep.put')
         self.assertEqual(self.keep_client.get('acbd18db4cc2f85cedef654fccc4a4d8+3'), b'foo', 'wrong data from Keep.get')
     def test_keep_local_store(self):
         self.assertEqual(self.keep_client.put(b'foo'), 'acbd18db4cc2f85cedef654fccc4a4d8+3', 'wrong md5 hash from Keep.put')
         self.assertEqual(self.keep_client.get('acbd18db4cc2f85cedef654fccc4a4d8+3'), b'foo', 'wrong data from Keep.get')


@@ -89,6 +98,8 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
         self.assertEqual(stream0.readfrom(2**26, 0),
                          b'',
                          'reading zero bytes should have returned empty string')
         self.assertEqual(stream0.readfrom(2**26, 0),
                          b'',
                          'reading zero bytes should have returned empty string')

+        self.assertEqual(3, len(cr))
+        self.assertTrue(cr)

 
     def _test_subset(self, collection, expected):
         cr = arvados.CollectionReader(collection, self.api_client)
 
     def _test_subset(self, collection, expected):
         cr = arvados.CollectionReader(collection, self.api_client)


@@ -495,15 +506,6 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
         self.assertRaises(arvados.errors.AssertionError,
                           cwriter.write, "badtext")
 
         self.assertRaises(arvados.errors.AssertionError,
                           cwriter.write, "badtext")
 

-    def test_read_arbitrary_data_with_collection_reader(self):
-        # arv-get relies on this to do "arv-get {keep-locator} -".
-        self.write_foo_bar_baz()
-        self.assertEqual(
-            'foobar',
-            arvados.CollectionReader(
-                '3858f62230ac3c915f300c664312c63f+6'
-                ).manifest_text())
-

 
 class CollectionTestMixin(tutil.ApiClientMock):
     API_COLLECTIONS = run_test_server.fixture('collections')
 
 class CollectionTestMixin(tutil.ApiClientMock):
     API_COLLECTIONS = run_test_server.fixture('collections')


@@ -559,34 +561,13 @@ class CollectionReaderTestCase(unittest.TestCase, CollectionTestMixin):
                                               api_client=client)
             self.assertEqual(self.DEFAULT_MANIFEST, reader.manifest_text())
 
                                               api_client=client)
             self.assertEqual(self.DEFAULT_MANIFEST, reader.manifest_text())
 

-    def test_locator_init_fallback_to_keep(self):
-        # crunch-job needs this to read manifests that have only ever
-        # been written to Keep.
-        client = self.api_client_mock(200)
-        self.mock_get_collection(client, 404, None)
-        with tutil.mock_keep_responses(self.DEFAULT_MANIFEST, 200):
-            reader = arvados.CollectionReader(self.DEFAULT_DATA_HASH,
-                                              api_client=client)
-            self.assertEqual(self.DEFAULT_MANIFEST, reader.manifest_text())
-
-    def test_uuid_init_no_fallback_to_keep(self):
-        # Do not look up a collection UUID in Keep.
-        client = self.api_client_mock(404)
-        with tutil.mock_keep_responses(self.DEFAULT_MANIFEST, 200):
-            with self.assertRaises(arvados.errors.ApiError):
-                reader = arvados.CollectionReader(self.DEFAULT_UUID,
-                                                  api_client=client)
-
-    def test_try_keep_first_if_permission_hint(self):
-        # To verify that CollectionReader tries Keep first here, we
-        # mock API server to return the wrong data.
-        client = self.api_client_mock(200)
-        with tutil.mock_keep_responses(self.ALT_MANIFEST, 200):
-            self.assertEqual(
-                self.ALT_MANIFEST,
-                arvados.CollectionReader(
-                    self.ALT_DATA_HASH + '+Affffffffffffffffffffffffffffffffffffffff@fedcba98',
-                    api_client=client).manifest_text())
+    def test_init_no_fallback_to_keep(self):
+        # Do not look up a collection UUID or PDH in Keep.
+        for key in [self.DEFAULT_UUID, self.DEFAULT_DATA_HASH]:
+            client = self.api_client_mock(404)
+            with tutil.mock_keep_responses(self.DEFAULT_MANIFEST, 200):
+                with self.assertRaises(arvados.errors.ApiError):
+                    reader = arvados.CollectionReader(key, api_client=client)

 
     def test_init_num_retries_propagated(self):
         # More of an integration test...
 
     def test_init_num_retries_propagated(self):
         # More of an integration test...


@@ -624,21 +605,14 @@ class CollectionReaderTestCase(unittest.TestCase, CollectionTestMixin):
         reader = arvados.CollectionReader('d41d8cd98f00b204e9800998ecf8427e+0',
                                           api_client=client)
         self.assertEqual('', reader.manifest_text())
         reader = arvados.CollectionReader('d41d8cd98f00b204e9800998ecf8427e+0',
                                           api_client=client)
         self.assertEqual('', reader.manifest_text())

+        self.assertEqual(0, len(reader))
+        self.assertFalse(reader)

 
     def test_api_response(self):
         client = self.api_client_mock()
         reader = arvados.CollectionReader(self.DEFAULT_UUID, api_client=client)
         self.assertEqual(self.DEFAULT_COLLECTION, reader.api_response())
 
 
     def test_api_response(self):
         client = self.api_client_mock()
         reader = arvados.CollectionReader(self.DEFAULT_UUID, api_client=client)
         self.assertEqual(self.DEFAULT_COLLECTION, reader.api_response())
 

-    def test_api_response_with_collection_from_keep(self):
-        client = self.api_client_mock()
-        self.mock_get_collection(client, 404, 'foo')
-        with tutil.mock_keep_responses(self.DEFAULT_MANIFEST, 200):
-            reader = arvados.CollectionReader(self.DEFAULT_DATA_HASH,
-                                              api_client=client)
-            api_response = reader.api_response()
-        self.assertIsNone(api_response)
-

     def check_open_file(self, coll_file, stream_name, file_name, file_size):
         self.assertFalse(coll_file.closed, "returned file is not open")
         self.assertEqual(stream_name, coll_file.stream_name())
     def check_open_file(self, coll_file, stream_name, file_name, file_size):
         self.assertFalse(coll_file.closed, "returned file is not open")
         self.assertEqual(stream_name, coll_file.stream_name())


@@ -1178,7 +1152,7 @@ class NewCollectionTestCaseWithServersAndTokens(run_test_server.TestCaseWithServ
     def setUp(self):
         self.keep_put = getattr(arvados.keep.KeepClient, 'put')
 
     def setUp(self):
         self.keep_put = getattr(arvados.keep.KeepClient, 'put')
 

-    def test_repacked_block_sumbmission_get_permission_token(self):
+    def test_repacked_block_submission_get_permission_token(self):

         '''
         Make sure that those blocks that are committed after repacking small ones,
         get their permission tokens assigned on the collection manifest.
         '''
         Make sure that those blocks that are committed after repacking small ones,
         get their permission tokens assigned on the collection manifest.


@@ -1299,9 +1273,9 @@ class NewCollectionTestCaseWithServers(run_test_server.TestCaseWithServers):
 
         c["b1"].writeto(0, b"1b", 0)
 
 
         c["b1"].writeto(0, b"1b", 0)
 

-        self.assertEquals(c.manifest_text(), ". ed4f3f67c70b02b29c50ce1ea26666bd+4 0:2:b1 2:2:b2\n")
-        self.assertEquals(c["b1"].manifest_text(), ". ed4f3f67c70b02b29c50ce1ea26666bd+4 0:2:b1\n")
-        self.assertEquals(c["b2"].manifest_text(), ". ed4f3f67c70b02b29c50ce1ea26666bd+4 2:2:b2\n")
+        self.assertEqual(c.manifest_text(), ". ed4f3f67c70b02b29c50ce1ea26666bd+4 0:2:b1 2:2:b2\n")
+        self.assertEqual(c["b1"].manifest_text(), ". ed4f3f67c70b02b29c50ce1ea26666bd+4 0:2:b1\n")
+        self.assertEqual(c["b2"].manifest_text(), ". ed4f3f67c70b02b29c50ce1ea26666bd+4 2:2:b2\n")

 
 
 class CollectionCreateUpdateTest(run_test_server.TestCaseWithServers):
 
 
 class CollectionCreateUpdateTest(run_test_server.TestCaseWithServers):


@@ -1399,6 +1373,11 @@ class CollectionCreateUpdateTest(run_test_server.TestCaseWithServers):
             c1.manifest_text(),
             r"\. e65075d550f9b5bf9992fa1d71a131be\+3\S* 7ac66c0f148de9519b8bd264312c4d64\+7\S* 0:3:count\.txt 3:7:count\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~$")
 
             c1.manifest_text(),
             r"\. e65075d550f9b5bf9992fa1d71a131be\+3\S* 7ac66c0f148de9519b8bd264312c4d64\+7\S* 0:3:count\.txt 3:7:count\.txt~\d\d\d\d\d\d\d\d-\d\d\d\d\d\d~conflict~$")
 

+    def test_pdh_is_native_str(self):
+        c1 = self.create_count_txt()
+        pdh = c1.portable_data_hash()
+        self.assertEqual(type(''), type(pdh))
+

 
 if __name__ == '__main__':
     unittest.main()
 
 if __name__ == '__main__':
     unittest.main()