X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/735143cbf278a87d975f71152471d59795836460..032ecb480134f6138346865fba7732a836b809c6:/sdk/python/tests/test_collections.py diff --git a/sdk/python/tests/test_collections.py b/sdk/python/tests/test_collections.py index f3ad16ecfe..b3df880340 100644 --- a/sdk/python/tests/test_collections.py +++ b/sdk/python/tests/test_collections.py @@ -9,11 +9,9 @@ import arvados import copy import mock import os -import pprint import random import re import sys -import tempfile import datetime import ciso8601 import time @@ -322,7 +320,7 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers, def __init__(self, content, num_retries=0): self.content = content - def get(self, locator, num_retries=0): + def get(self, locator, num_retries=0, prefetch=False): return self.content[locator] def test_stream_reader(self): @@ -898,7 +896,7 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin): c1.save_new() loc = c1.manifest_locator() c2 = Collection(loc) - self.assertEqual(c1.manifest_text, c2.manifest_text) + self.assertEqual(c1.manifest_text(), c2.manifest_text()) self.assertEqual(c1.replication_desired, c2.replication_desired) def test_replication_desired_not_loaded_if_provided(self): @@ -907,9 +905,40 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin): c1.save_new() loc = c1.manifest_locator() c2 = Collection(loc, replication_desired=2) - self.assertEqual(c1.manifest_text, c2.manifest_text) + self.assertEqual(c1.manifest_text(), c2.manifest_text()) self.assertNotEqual(c1.replication_desired, c2.replication_desired) + def test_storage_classes_desired_kept_on_load(self): + m = '. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n' + c1 = Collection(m, storage_classes_desired=['archival']) + c1.save_new() + loc = c1.manifest_locator() + c2 = Collection(loc) + self.assertEqual(c1.manifest_text(), c2.manifest_text()) + self.assertEqual(c1.storage_classes_desired(), c2.storage_classes_desired()) + + def test_storage_classes_change_after_save(self): + m = '. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n' + c1 = Collection(m, storage_classes_desired=['archival']) + c1.save_new() + loc = c1.manifest_locator() + c2 = Collection(loc) + self.assertEqual(['archival'], c2.storage_classes_desired()) + c2.save(storage_classes=['highIO']) + self.assertEqual(['highIO'], c2.storage_classes_desired()) + c3 = Collection(loc) + self.assertEqual(c1.manifest_text(), c3.manifest_text()) + self.assertEqual(['highIO'], c3.storage_classes_desired()) + + def test_storage_classes_desired_not_loaded_if_provided(self): + m = '. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n' + c1 = Collection(m, storage_classes_desired=['archival']) + c1.save_new() + loc = c1.manifest_locator() + c2 = Collection(loc, storage_classes_desired=['default']) + self.assertEqual(c1.manifest_text(), c2.manifest_text()) + self.assertNotEqual(c1.storage_classes_desired(), c2.storage_classes_desired()) + def test_init_manifest(self): m1 = """. 5348b82a029fd9e971a811ce1f71360b+43 0:43:md5sum.txt . 085c37f02916da1cad16f93c54d899b7+41 0:41:md5sum.txt @@ -940,6 +969,20 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin): with self.assertRaises(arvados.errors.ArgumentError): c.remove("") + def test_remove_recursive(self): + c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:a/b/c/d/efg.txt 0:10:xyz.txt\n') + self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:xyz.txt\n./a/b/c/d 781e5e245d69b566979b86e28d23f2c7+10 0:10:efg.txt\n", c.portable_manifest_text()) + self.assertIn("a", c) + self.assertEqual(1, len(c["a"].keys())) + # cannot remove non-empty directory with default recursive=False + with self.assertRaises(OSError): + c.remove("a/b") + with self.assertRaises(OSError): + c.remove("a/b/c/d") + c.remove("a/b", recursive=True) + self.assertEqual(0, len(c["a"].keys())) + self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:xyz.txt\n./a d41d8cd98f00b204e9800998ecf8427e+0 0:0:\\056\n", c.portable_manifest_text()) + def test_find(self): c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n') self.assertIs(c.find("."), c) @@ -960,8 +1003,11 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin): def test_other_special_chars_on_file_token(self): cases = [ + ('\\000', '\0'), ('\\011', '\t'), ('\\012', '\n'), + ('\\072', ':'), + ('\\134400', '\\400'), ] for encoded, decoded in cases: manifest = '. d41d8cd98f00b204e9800998ecf8427e+0 0:0:some%sfile.txt\n' % encoded @@ -1245,6 +1291,16 @@ class NewCollectionTestCaseWithServersAndTokens(run_test_server.TestCaseWithServ def setUp(self): self.keep_put = getattr(arvados.keep.KeepClient, 'put') + @mock.patch('arvados.keep.KeepClient.put', autospec=True) + def test_storage_classes_desired(self, put_mock): + put_mock.side_effect = self.keep_put + c = Collection(storage_classes_desired=['default']) + with c.open("file.txt", 'wb') as f: + f.write('content') + c.save_new() + _, kwargs = put_mock.call_args + self.assertEqual(['default'], kwargs['classes']) + @mock.patch('arvados.keep.KeepClient.put', autospec=True) def test_repacked_block_submission_get_permission_token(self, mocked_put): ''' @@ -1318,6 +1374,25 @@ class NewCollectionTestCaseWithServersAndTokens(run_test_server.TestCaseWithServ class NewCollectionTestCaseWithServers(run_test_server.TestCaseWithServers): + def test_preserve_version_on_save(self): + c = Collection() + c.save_new(preserve_version=True) + coll_record = arvados.api().collections().get(uuid=c.manifest_locator()).execute() + self.assertEqual(coll_record['version'], 1) + self.assertEqual(coll_record['preserve_version'], True) + with c.open("foo.txt", "wb") as foo: + foo.write(b"foo") + c.save(preserve_version=True) + coll_record = arvados.api().collections().get(uuid=c.manifest_locator()).execute() + self.assertEqual(coll_record['version'], 2) + self.assertEqual(coll_record['preserve_version'], True) + with c.open("bar.txt", "wb") as foo: + foo.write(b"bar") + c.save(preserve_version=False) + coll_record = arvados.api().collections().get(uuid=c.manifest_locator()).execute() + self.assertEqual(coll_record['version'], 3) + self.assertEqual(coll_record['preserve_version'], False) + def test_get_manifest_text_only_committed(self): c = Collection() with c.open("count.txt", "wb") as f: