X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/5a80356ddc3798f6530e306901d7ea5e19cfe3f9..b887df487c6cb465c3212e8ef99278350e58ff0a:/sdk/python/tests/test_arv_put.py diff --git a/sdk/python/tests/test_arv_put.py b/sdk/python/tests/test_arv_put.py index 66af47a419..d078268c83 100644 --- a/sdk/python/tests/test_arv_put.py +++ b/sdk/python/tests/test_arv_put.py @@ -43,7 +43,7 @@ class ArvadosPutResumeCacheTest(ArvadosBaseTestCase): def test_cache_names_stable(self): for argset in self.CACHE_ARGSET: - self.assertEquals(self.cache_path_from_arglist(argset), + self.assertEqual(self.cache_path_from_arglist(argset), self.cache_path_from_arglist(argset), "cache name changed for {}".format(argset)) @@ -65,10 +65,10 @@ class ArvadosPutResumeCacheTest(ArvadosBaseTestCase): "path too exotic: {}".format(path)) def test_cache_names_ignore_argument_order(self): - self.assertEquals( + self.assertEqual( self.cache_path_from_arglist(['a', 'b', 'c']), self.cache_path_from_arglist(['c', 'a', 'b'])) - self.assertEquals( + self.assertEqual( self.cache_path_from_arglist(['-', '--filename', 'stdin']), self.cache_path_from_arglist(['--filename', 'stdin', '-'])) @@ -84,32 +84,32 @@ class ArvadosPutResumeCacheTest(ArvadosBaseTestCase): args = arv_put.parse_arguments(['/tmp']) args.filename = 'tmp' path2 = arv_put.ResumeCache.make_path(args) - self.assertEquals(path1, path2, + self.assertEqual(path1, path2, "cache path considered --filename for directory") - self.assertEquals( + self.assertEqual( self.cache_path_from_arglist(['-']), self.cache_path_from_arglist(['-', '--max-manifest-depth', '1']), "cache path considered --max-manifest-depth for file") def test_cache_names_treat_negative_manifest_depths_identically(self): base_args = ['/tmp', '--max-manifest-depth'] - self.assertEquals( + self.assertEqual( self.cache_path_from_arglist(base_args + ['-1']), self.cache_path_from_arglist(base_args + ['-2'])) def test_cache_names_treat_stdin_consistently(self): - self.assertEquals( + self.assertEqual( self.cache_path_from_arglist(['-', '--filename', 'test']), self.cache_path_from_arglist(['/dev/stdin', '--filename', 'test'])) def test_cache_names_identical_for_synonymous_names(self): - self.assertEquals( + self.assertEqual( self.cache_path_from_arglist(['.']), self.cache_path_from_arglist([os.path.realpath('.')])) testdir = self.make_tmpdir() looplink = os.path.join(testdir, 'loop') os.symlink(testdir, looplink) - self.assertEquals( + self.assertEqual( self.cache_path_from_arglist([testdir]), self.cache_path_from_arglist([looplink])) @@ -131,7 +131,7 @@ class ArvadosPutResumeCacheTest(ArvadosBaseTestCase): with tempfile.NamedTemporaryFile() as cachefile: self.last_cache = arv_put.ResumeCache(cachefile.name) self.last_cache.save(thing) - self.assertEquals(thing, self.last_cache.load()) + self.assertEqual(thing, self.last_cache.load()) def test_empty_cache(self): with tempfile.NamedTemporaryFile() as cachefile: @@ -145,7 +145,7 @@ class ArvadosPutResumeCacheTest(ArvadosBaseTestCase): cache.save(thing) cache.close() self.last_cache = arv_put.ResumeCache(path) - self.assertEquals(thing, self.last_cache.load()) + self.assertEqual(thing, self.last_cache.load()) def test_multiple_cache_writes(self): thing = ['short', 'list'] @@ -155,7 +155,7 @@ class ArvadosPutResumeCacheTest(ArvadosBaseTestCase): # sure the cache file gets truncated. self.last_cache.save(['long', 'long', 'list']) self.last_cache.save(thing) - self.assertEquals(thing, self.last_cache.load()) + self.assertEqual(thing, self.last_cache.load()) def test_cache_is_locked(self): with tempfile.NamedTemporaryFile() as cachefile: @@ -216,12 +216,12 @@ class ArvadosPutCollectionWriterTest(run_test_server.TestCaseWithServers, cwriter.write_file('/dev/null') cwriter.cache_state() self.assertTrue(self.cache.load()) - self.assertEquals(". d41d8cd98f00b204e9800998ecf8427e+0 0:0:null\n", cwriter.manifest_text()) + self.assertEqual(". d41d8cd98f00b204e9800998ecf8427e+0 0:0:null\n", cwriter.manifest_text()) def test_writer_works_without_cache(self): cwriter = arv_put.ArvPutCollectionWriter() cwriter.write_file('/dev/null') - self.assertEquals(". d41d8cd98f00b204e9800998ecf8427e+0 0:0:null\n", cwriter.manifest_text()) + self.assertEqual(". d41d8cd98f00b204e9800998ecf8427e+0 0:0:null\n", cwriter.manifest_text()) def test_writer_resumes_from_cache(self): cwriter = arv_put.ArvPutCollectionWriter(self.cache) @@ -230,7 +230,7 @@ class ArvadosPutCollectionWriterTest(run_test_server.TestCaseWithServers, cwriter.cache_state() new_writer = arv_put.ArvPutCollectionWriter.from_cache( self.cache) - self.assertEquals( + self.assertEqual( ". 098f6bcd4621d373cade4e832627b4f6+4 0:4:test\n", new_writer.manifest_text()) @@ -240,12 +240,12 @@ class ArvadosPutCollectionWriterTest(run_test_server.TestCaseWithServers, cwriter.write_file(testfile.name, 'test') new_writer = arv_put.ArvPutCollectionWriter.from_cache(self.cache) new_writer.write_file('/dev/null') - self.assertEquals(". d41d8cd98f00b204e9800998ecf8427e+0 0:0:null\n", new_writer.manifest_text()) + self.assertEqual(". d41d8cd98f00b204e9800998ecf8427e+0 0:0:null\n", new_writer.manifest_text()) def test_new_writer_from_empty_cache(self): cwriter = arv_put.ArvPutCollectionWriter.from_cache(self.cache) cwriter.write_file('/dev/null') - self.assertEquals(". d41d8cd98f00b204e9800998ecf8427e+0 0:0:null\n", cwriter.manifest_text()) + self.assertEqual(". d41d8cd98f00b204e9800998ecf8427e+0 0:0:null\n", cwriter.manifest_text()) def test_writer_resumable_after_arbitrary_bytes(self): cwriter = arv_put.ArvPutCollectionWriter(self.cache) @@ -255,7 +255,7 @@ class ArvadosPutCollectionWriterTest(run_test_server.TestCaseWithServers, cwriter.cache_state() new_writer = arv_put.ArvPutCollectionWriter.from_cache( self.cache) - self.assertEquals(cwriter.manifest_text(), new_writer.manifest_text()) + self.assertEqual(cwriter.manifest_text(), new_writer.manifest_text()) def make_progress_tester(self): progression = [] @@ -288,16 +288,16 @@ class ArvadosExpectedBytesTest(ArvadosBaseTestCase): TEST_SIZE = os.path.getsize(__file__) def test_expected_bytes_for_file(self): - self.assertEquals(self.TEST_SIZE, + self.assertEqual(self.TEST_SIZE, arv_put.expected_bytes_for([__file__])) def test_expected_bytes_for_tree(self): tree = self.make_tmpdir() shutil.copyfile(__file__, os.path.join(tree, 'one')) shutil.copyfile(__file__, os.path.join(tree, 'two')) - self.assertEquals(self.TEST_SIZE * 2, + self.assertEqual(self.TEST_SIZE * 2, arv_put.expected_bytes_for([tree])) - self.assertEquals(self.TEST_SIZE * 3, + self.assertEqual(self.TEST_SIZE * 3, arv_put.expected_bytes_for([tree, __file__])) def test_expected_bytes_for_device(self): @@ -326,82 +326,9 @@ class ArvadosPutReportTest(ArvadosBaseTestCase): arv_put.human_progress(count, None))) -class ArvadosPutProjectLinkTest(ArvadosBaseTestCase): - Z_UUID = 'zzzzz-zzzzz-zzzzzzzzzzzzzzz' - - def setUp(self): - self.stderr = StringIO() - super(ArvadosPutProjectLinkTest, self).setUp() - - def tearDown(self): - self.stderr.close() - super(ArvadosPutProjectLinkTest, self).tearDown() - - def prep_link_from_arguments(self, args, uuid_found=True): - try: - link = arv_put.prep_project_link(arv_put.parse_arguments(args), - self.stderr, - lambda uuid: uuid_found) - finally: - self.stderr.seek(0) - return link - - def check_link(self, link, project_uuid, link_name=None): - self.assertEqual(project_uuid, link.get('tail_uuid')) - self.assertEqual(project_uuid, link.get('owner_uuid')) - self.assertEqual('name', link.get('link_class')) - if link_name is None: - self.assertNotIn('name', link) - else: - self.assertEqual(link_name, link.get('name')) - self.assertNotIn('head_uuid', link) - - def check_stderr_empty(self): - self.assertEqual('', self.stderr.getvalue()) - - def test_project_link_with_name(self): - link = self.prep_link_from_arguments(['--project-uuid', self.Z_UUID, - '--name', 'test link AAA']) - self.check_link(link, self.Z_UUID, 'test link AAA') - self.check_stderr_empty() - - def test_project_link_without_name(self): - username = pwd.getpwuid(os.getuid()).pw_name - link = self.prep_link_from_arguments(['--project-uuid', self.Z_UUID]) - self.assertIsNotNone(link.get('name', None)) - self.assertRegexpMatches( - link['name'], - r'^Saved at .* by {}@'.format(re.escape(username))) - self.check_link(link, self.Z_UUID, link.get('name', None)) - for line in self.stderr: - if "No --name specified" in line: - break - else: - self.fail("no warning emitted about the lack of collection name") - - @unittest.skip("prep_project_link needs an API lookup for this case") - def test_collection_without_project_defaults_to_home(self): - link = self.prep_link_from_arguments(['--name', 'test link BBB']) - self.check_link(link, self.Z_UUID) - self.check_stderr_empty() - - def test_no_link_or_warning_with_no_collection(self): - self.assertIsNone(self.prep_link_from_arguments(['--raw'])) - self.check_stderr_empty() - - def test_error_when_project_not_found(self): - self.assertRaises(ValueError, - self.prep_link_from_arguments, - ['--project-uuid', self.Z_UUID], False) - - def test_link_without_collection_is_error(self): - self.assertRaises(ValueError, - self.prep_link_from_arguments, - ['--project-uuid', self.Z_UUID, '--stream']) - - class ArvadosPutTest(run_test_server.TestCaseWithServers, ArvadosBaseTestCase): MAIN_SERVER = {} + Z_UUID = 'zzzzz-zzzzz-zzzzzzzzzzzzzzz' def call_main_with_args(self, args): self.main_stdout = StringIO() @@ -454,17 +381,31 @@ class ArvadosPutTest(run_test_server.TestCaseWithServers, ArvadosBaseTestCase): arv_put.ResumeCache.CACHE_DIR = orig_cachedir os.chmod(cachedir, 0o700) - def test_link_without_collection_aborts(self): + def test_error_name_without_collection(self): self.assertRaises(SystemExit, self.call_main_with_args, ['--name', 'test without Collection', '--stream', '/dev/null']) + def test_error_when_project_not_found(self): + self.assertRaises(SystemExit, + self.call_main_with_args, + ['--project-uuid', self.Z_UUID]) + + def test_error_bad_project_uuid(self): + self.assertRaises(SystemExit, + self.call_main_with_args, + ['--project-uuid', self.Z_UUID, '--stream']) + class ArvPutIntegrationTest(run_test_server.TestCaseWithServers, ArvadosBaseTestCase): def _getKeepServerConfig(): - for config_file in ['application.yml', 'application.default.yml']: - with open(os.path.join(run_test_server.SERVICES_SRC_DIR, - "api", "config", config_file)) as f: + for config_file, mandatory in [ + ['application.yml', True], ['application.default.yml', False]]: + path = os.path.join(run_test_server.SERVICES_SRC_DIR, + "api", "config", config_file) + if not mandatory and not os.path.exists(path): + continue + with open(path) as f: rails_config = yaml.load(f.read()) for config_section in ['test', 'common']: try: @@ -496,26 +437,34 @@ class ArvPutIntegrationTest(run_test_server.TestCaseWithServers, "ARVADOS_API_HOST_INSECURE", "ARVADOS_API_TOKEN"]: self.ENVIRON[v] = arvados.config.settings()[v] - arv_put.api_client = arvados.api('v1', cache=False) + arv_put.api_client = arvados.api('v1') def current_user(self): return arv_put.api_client.users().current().execute() def test_check_real_project_found(self): self.authorize_with('active') - self.assertTrue(arv_put.check_project_exists(self.PROJECT_UUID), + self.assertTrue(arv_put.desired_project_uuid(arv_put.api_client, self.PROJECT_UUID, 0), "did not correctly find test fixture project") - def test_check_error_finding_nonexistent_project(self): + def test_check_error_finding_nonexistent_uuid(self): BAD_UUID = 'zzzzz-zzzzz-zzzzzzzzzzzzzzz' self.authorize_with('active') try: - result = arv_put.check_project_exists(BAD_UUID) + result = arv_put.desired_project_uuid(arv_put.api_client, BAD_UUID, + 0) except ValueError as error: self.assertIn(BAD_UUID, error.message) else: self.assertFalse(result, "incorrectly found nonexistent project") + def test_check_error_finding_nonexistent_project(self): + BAD_UUID = 'zzzzz-tpzed-zzzzzzzzzzzzzzz' + self.authorize_with('active') + with self.assertRaises(apiclient.errors.HttpError): + result = arv_put.desired_project_uuid(arv_put.api_client, BAD_UUID, + 0) + def test_short_put_from_stdin(self): # Have to run this as an integration test since arv-put can't # read from the tests' stdin. @@ -578,14 +527,28 @@ class ArvPutIntegrationTest(run_test_server.TestCaseWithServers, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.ENVIRON) stdout, stderr = pipe.communicate(text) - collection_list = arvados.api('v1', cache=False).collections().list( - filters=[['portable_data_hash', '=', stdout.strip()]]).execute().get('items', []) + search_key = ('portable_data_hash' + if '--portable-data-hash' in extra_args else 'uuid') + collection_list = arvados.api('v1').collections().list( + filters=[[search_key, '=', stdout.strip()]]).execute().get('items', []) self.assertEqual(1, len(collection_list)) return collection_list[0] + def test_put_collection_with_high_redundancy(self): + # Write empty data: we're not testing CollectionWriter, just + # making sure collections.create tells the API server what our + # desired replication level is. + collection = self.run_and_find_collection("", ['--replication', '4']) + self.assertEqual(4, collection['replication_desired']) + + def test_put_collection_with_default_redundancy(self): + collection = self.run_and_find_collection("") + self.assertEqual(None, collection['replication_desired']) + def test_put_collection_with_unnamed_project_link(self): - link = self.run_and_find_collection("Test unnamed collection", - ['--project-uuid', self.PROJECT_UUID]) + link = self.run_and_find_collection( + "Test unnamed collection", + ['--portable-data-hash', '--project-uuid', self.PROJECT_UUID]) username = pwd.getpwuid(os.getuid()).pw_name self.assertRegexpMatches( link['name'], @@ -593,8 +556,9 @@ class ArvPutIntegrationTest(run_test_server.TestCaseWithServers, def test_put_collection_with_name_and_no_project(self): link_name = 'Test Collection Link in home project' - collection = self.run_and_find_collection("Test named collection in home project", - ['--name', link_name]) + collection = self.run_and_find_collection( + "Test named collection in home project", + ['--portable-data-hash', '--name', link_name]) self.assertEqual(link_name, collection['name']) my_user_uuid = self.current_user()['uuid'] self.assertEqual(my_user_uuid, collection['owner_uuid']) @@ -602,7 +566,8 @@ class ArvPutIntegrationTest(run_test_server.TestCaseWithServers, def test_put_collection_with_named_project_link(self): link_name = 'Test auto Collection Link' collection = self.run_and_find_collection("Test named collection", - ['--name', link_name, + ['--portable-data-hash', + '--name', link_name, '--project-uuid', self.PROJECT_UUID]) self.assertEqual(link_name, collection['name'])