X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/d2d20f664ec207d1c9492edea6d893b1e333b055..3082625157ad8abf2de8b6797d41dccb1a1816ff:/sdk/python/tests/test_arv_put.py diff --git a/sdk/python/tests/test_arv_put.py b/sdk/python/tests/test_arv_put.py index e75d39d879..afdf2238a7 100644 --- a/sdk/python/tests/test_arv_put.py +++ b/sdk/python/tests/test_arv_put.py @@ -14,10 +14,10 @@ from functools import partial import apiclient import ciso8601 import datetime -import hashlib import json import logging import mock +import multiprocessing import os import pwd import random @@ -31,7 +31,6 @@ import tempfile import time import unittest import uuid -import yaml import arvados import arvados.commands.put as arv_put @@ -294,7 +293,29 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers, shutil.rmtree(self.small_files_dir) shutil.rmtree(self.tempdir_with_symlink) + def test_non_regular_files_are_ignored_except_symlinks_to_dirs(self): + def pfunc(x): + with open(x, 'w') as f: + f.write('test') + fifo_filename = 'fifo-file' + fifo_path = os.path.join(self.tempdir_with_symlink, fifo_filename) + self.assertTrue(os.path.islink(os.path.join(self.tempdir_with_symlink, 'linkeddir'))) + os.mkfifo(fifo_path) + producer = multiprocessing.Process(target=pfunc, args=(fifo_path,)) + producer.start() + cwriter = arv_put.ArvPutUploadJob([self.tempdir_with_symlink]) + cwriter.start(save_collection=False) + if producer.exitcode is None: + # If the producer is still running, kill it. This should always be + # before any assertion that may fail. + producer.terminate() + producer.join(1) + self.assertIn('linkeddir', cwriter.manifest_text()) + self.assertNotIn(fifo_filename, cwriter.manifest_text()) + def test_symlinks_are_followed_by_default(self): + self.assertTrue(os.path.islink(os.path.join(self.tempdir_with_symlink, 'linkeddir'))) + self.assertTrue(os.path.islink(os.path.join(self.tempdir_with_symlink, 'linkedfile'))) cwriter = arv_put.ArvPutUploadJob([self.tempdir_with_symlink]) cwriter.start(save_collection=False) self.assertIn('linkeddir', cwriter.manifest_text()) @@ -302,12 +323,29 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers, cwriter.destroy_cache() def test_symlinks_are_not_followed_when_requested(self): + self.assertTrue(os.path.islink(os.path.join(self.tempdir_with_symlink, 'linkeddir'))) + self.assertTrue(os.path.islink(os.path.join(self.tempdir_with_symlink, 'linkedfile'))) cwriter = arv_put.ArvPutUploadJob([self.tempdir_with_symlink], follow_links=False) cwriter.start(save_collection=False) self.assertNotIn('linkeddir', cwriter.manifest_text()) self.assertNotIn('linkedfile', cwriter.manifest_text()) cwriter.destroy_cache() + # Check for bug #17800: passed symlinks should also be ignored. + linked_dir = os.path.join(self.tempdir_with_symlink, 'linkeddir') + cwriter = arv_put.ArvPutUploadJob([linked_dir], follow_links=False) + cwriter.start(save_collection=False) + self.assertNotIn('linkeddir', cwriter.manifest_text()) + cwriter.destroy_cache() + + def test_no_empty_collection_saved(self): + self.assertTrue(os.path.islink(os.path.join(self.tempdir_with_symlink, 'linkeddir'))) + linked_dir = os.path.join(self.tempdir_with_symlink, 'linkeddir') + cwriter = arv_put.ArvPutUploadJob([linked_dir], follow_links=False) + cwriter.start(save_collection=True) + self.assertIsNone(cwriter.manifest_locator()) + self.assertEqual('', cwriter.manifest_text()) + cwriter.destroy_cache() def test_passing_nonexistant_path_raise_exception(self): uuid_str = str(uuid.uuid4()) @@ -775,6 +813,7 @@ class ArvadosPutTest(run_test_server.TestCaseWithServers, def test_put_block_replication(self): self.call_main_on_test_file() + arv_put.api_client = None with mock.patch('arvados.collection.KeepClient.local_store_put') as put_mock: put_mock.return_value = 'acbd18db4cc2f85cedef654fccc4a4d8+3' self.call_main_on_test_file(['--replication', '1']) @@ -813,11 +852,6 @@ class ArvadosPutTest(run_test_server.TestCaseWithServers, self.call_main_with_args, ['--project-uuid', self.Z_UUID, '--stream']) - def test_error_when_multiple_storage_classes_specified(self): - self.assertRaises(SystemExit, - self.call_main_with_args, - ['--storage-classes', 'hot,cold']) - def test_error_when_excluding_absolute_path(self): tmpdir = self.make_tmpdir() self.assertRaises(SystemExit, @@ -1041,43 +1075,53 @@ class ArvPutIntegrationTest(run_test_server.TestCaseWithServers, r'INFO: Cache expired, starting from scratch.*') self.assertEqual(p.returncode, 0) - def test_invalid_signature_invalidates_cache(self): - self.authorize_with('active') - tmpdir = self.make_tmpdir() - with open(os.path.join(tmpdir, 'somefile.txt'), 'w') as f: - f.write('foo') - # Upload a directory and get the cache file name - p = subprocess.Popen([sys.executable, arv_put.__file__, tmpdir], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=self.ENVIRON) - (_, err) = p.communicate() - self.assertRegex(err.decode(), r'INFO: Creating new cache file at ') - self.assertEqual(p.returncode, 0) - cache_filepath = re.search(r'INFO: Creating new cache file at (.*)', - err.decode()).groups()[0] - self.assertTrue(os.path.isfile(cache_filepath)) - # Load the cache file contents and modify the manifest to simulate - # an invalid access token - with open(cache_filepath, 'r') as c: - cache = json.load(c) - self.assertRegex(cache['manifest'], r'\+A\S+\@') - cache['manifest'] = re.sub( - r'\+A.*\@', - "+Aabcdef0123456789abcdef0123456789abcdef01@", - cache['manifest']) - with open(cache_filepath, 'w') as c: - c.write(json.dumps(cache)) - # Re-run the upload and expect to get an invalid cache message - p = subprocess.Popen([sys.executable, arv_put.__file__, tmpdir], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=self.ENVIRON) - (_, err) = p.communicate() - self.assertRegex( - err.decode(), - r'ERROR: arv-put: Resume cache contains invalid signature.*') - self.assertEqual(p.returncode, 1) + def test_invalid_signature_in_cache(self): + for batch_mode in [False, True]: + self.authorize_with('active') + tmpdir = self.make_tmpdir() + with open(os.path.join(tmpdir, 'somefile.txt'), 'w') as f: + f.write('foo') + # Upload a directory and get the cache file name + arv_put_args = [tmpdir] + if batch_mode: + arv_put_args = ['--batch'] + arv_put_args + p = subprocess.Popen([sys.executable, arv_put.__file__] + arv_put_args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=self.ENVIRON) + (_, err) = p.communicate() + self.assertRegex(err.decode(), r'INFO: Creating new cache file at ') + self.assertEqual(p.returncode, 0) + cache_filepath = re.search(r'INFO: Creating new cache file at (.*)', + err.decode()).groups()[0] + self.assertTrue(os.path.isfile(cache_filepath)) + # Load the cache file contents and modify the manifest to simulate + # an invalid access token + with open(cache_filepath, 'r') as c: + cache = json.load(c) + self.assertRegex(cache['manifest'], r'\+A\S+\@') + cache['manifest'] = re.sub( + r'\+A.*\@', + "+Aabcdef0123456789abcdef0123456789abcdef01@", + cache['manifest']) + with open(cache_filepath, 'w') as c: + c.write(json.dumps(cache)) + # Re-run the upload and expect to get an invalid cache message + p = subprocess.Popen([sys.executable, arv_put.__file__] + arv_put_args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=self.ENVIRON) + (_, err) = p.communicate() + if not batch_mode: + self.assertRegex( + err.decode(), + r'ERROR: arv-put: Resume cache contains invalid signature.*') + self.assertEqual(p.returncode, 1) + else: + self.assertRegex( + err.decode(), + r'Invalid signatures on cache file \'.*\' while being run in \'batch mode\' -- continuing anyways.*') + self.assertEqual(p.returncode, 0) def test_single_expired_signature_reuploads_file(self): self.authorize_with('active') @@ -1315,13 +1359,16 @@ class ArvPutIntegrationTest(run_test_server.TestCaseWithServers, def test_put_collection_with_storage_classes_specified(self): collection = self.run_and_find_collection("", ['--storage-classes', 'hot']) - self.assertEqual(len(collection['storage_classes_desired']), 1) self.assertEqual(collection['storage_classes_desired'][0], 'hot') + def test_put_collection_with_multiple_storage_classes_specified(self): + collection = self.run_and_find_collection("", ['--storage-classes', ' foo, bar ,baz']) + self.assertEqual(len(collection['storage_classes_desired']), 3) + self.assertEqual(collection['storage_classes_desired'], ['foo', 'bar', 'baz']) + def test_put_collection_without_storage_classes_specified(self): collection = self.run_and_find_collection("") - self.assertEqual(len(collection['storage_classes_desired']), 1) self.assertEqual(collection['storage_classes_desired'][0], 'default')