X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/142a3c236c911731c337e1f9ee06038e6388e87b..18f3b51a1795922a5c5c595b5b3fb5ce12978c64:/sdk/python/tests/test_arv_put.py diff --git a/sdk/python/tests/test_arv_put.py b/sdk/python/tests/test_arv_put.py index 2067b7ce14..320189104a 100644 --- a/sdk/python/tests/test_arv_put.py +++ b/sdk/python/tests/test_arv_put.py @@ -17,6 +17,7 @@ import yaml import threading import hashlib import random +import uuid from cStringIO import StringIO @@ -268,12 +269,39 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers, with open(os.path.join(self.small_files_dir, str(i)), 'w') as f: f.write(data + str(i)) self.arvfile_write = getattr(arvados.arvfile.ArvadosFileWriter, 'write') + # Temp dir to hold a symlink to other temp dir + self.tempdir_with_symlink = tempfile.mkdtemp() + os.symlink(self.tempdir, os.path.join(self.tempdir_with_symlink, 'linkeddir')) + os.symlink(os.path.join(self.tempdir, '1'), + os.path.join(self.tempdir_with_symlink, 'linkedfile')) def tearDown(self): super(ArvPutUploadJobTest, self).tearDown() shutil.rmtree(self.tempdir) os.unlink(self.large_file_name) shutil.rmtree(self.small_files_dir) + shutil.rmtree(self.tempdir_with_symlink) + + def test_symlinks_are_followed_by_default(self): + cwriter = arv_put.ArvPutUploadJob([self.tempdir_with_symlink]) + cwriter.start(save_collection=False) + self.assertIn('linkeddir', cwriter.manifest_text()) + self.assertIn('linkedfile', cwriter.manifest_text()) + cwriter.destroy_cache() + + def test_symlinks_are_not_followed_when_requested(self): + cwriter = arv_put.ArvPutUploadJob([self.tempdir_with_symlink], + follow_links=False) + cwriter.start(save_collection=False) + self.assertNotIn('linkeddir', cwriter.manifest_text()) + self.assertNotIn('linkedfile', cwriter.manifest_text()) + cwriter.destroy_cache() + + def test_passing_nonexistant_path_raise_exception(self): + uuid_str = str(uuid.uuid4()) + cwriter = arv_put.ArvPutUploadJob(["/this/path/does/not/exist/{}".format(uuid_str)]) + with self.assertRaises(arv_put.PathDoesNotExistError): + cwriter.start(save_collection=False) def test_writer_works_without_cache(self): cwriter = arv_put.ArvPutUploadJob(['/dev/null'], resume=False) @@ -322,6 +350,8 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers, data = args[1] # Exit only on last block if len(data) < arvados.config.KEEP_BLOCK_SIZE: + # Simulate a checkpoint before quitting. Ensure block commit. + self.writer._update(final=True) raise SystemExit("Simulated error") return self.arvfile_write(*args, **kwargs) @@ -330,6 +360,8 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers, mocked_write.side_effect = wrapped_write writer = arv_put.ArvPutUploadJob([self.large_file_name], replication_desired=1) + # We'll be accessing from inside the wrapper + self.writer = writer with self.assertRaises(SystemExit): writer.start(save_collection=False) # Confirm that the file was partially uploaded @@ -343,24 +375,26 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers, self.assertEqual(writer.bytes_written + writer2.bytes_written - writer2.bytes_skipped, os.path.getsize(self.large_file_name)) writer2.destroy_cache() + del(self.writer) # Test for bug #11002 def test_graceful_exit_while_repacking_small_blocks(self): - def wrapped_delete(*args, **kwargs): + def wrapped_commit(*args, **kwargs): raise SystemExit("Simulated error") - with mock.patch('arvados.arvfile._BlockManager._delete_bufferblock', - autospec=True) as mocked_delete: - mocked_delete.side_effect = wrapped_delete - # Upload a little more than 1 block, wrapped_delete will make the first block + with mock.patch('arvados.arvfile._BlockManager.commit_bufferblock', + autospec=True) as mocked_commit: + mocked_commit.side_effect = wrapped_commit + # Upload a little more than 1 block, wrapped_commit will make the first block # commit to fail. # arv-put should not exit with an exception by trying to commit the collection # as it's in an inconsistent state. writer = arv_put.ArvPutUploadJob([self.small_files_dir], replication_desired=1) try: - writer.start(save_collection=False) - except: + with self.assertRaises(SystemExit): + writer.start(save_collection=False) + except arvados.arvfile.UnownedBlockError: self.fail("arv-put command is trying to use a corrupted BlockManager. See https://dev.arvados.org/issues/11002") writer.destroy_cache() @@ -369,6 +403,8 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers, data = args[1] # Exit only on last block if len(data) < arvados.config.KEEP_BLOCK_SIZE: + # Simulate a checkpoint before quitting. + self.writer._update() raise SystemExit("Simulated error") return self.arvfile_write(*args, **kwargs) @@ -377,6 +413,8 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers, mocked_write.side_effect = wrapped_write writer = arv_put.ArvPutUploadJob([self.large_file_name], replication_desired=1) + # We'll be accessing from inside the wrapper + self.writer = writer with self.assertRaises(SystemExit): writer.start(save_collection=False) # Confirm that the file was partially uploaded @@ -392,12 +430,15 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers, self.assertEqual(writer2.bytes_written, os.path.getsize(self.large_file_name)) writer2.destroy_cache() + del(self.writer) def test_no_resume_when_no_cache(self): def wrapped_write(*args, **kwargs): data = args[1] # Exit only on last block if len(data) < arvados.config.KEEP_BLOCK_SIZE: + # Simulate a checkpoint before quitting. + self.writer._update() raise SystemExit("Simulated error") return self.arvfile_write(*args, **kwargs) @@ -406,6 +447,8 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers, mocked_write.side_effect = wrapped_write writer = arv_put.ArvPutUploadJob([self.large_file_name], replication_desired=1) + # We'll be accessing from inside the wrapper + self.writer = writer with self.assertRaises(SystemExit): writer.start(save_collection=False) # Confirm that the file was partially uploaded @@ -422,13 +465,15 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers, self.assertEqual(writer2.bytes_written, os.path.getsize(self.large_file_name)) writer2.destroy_cache() - + del(self.writer) def test_dry_run_feature(self): def wrapped_write(*args, **kwargs): data = args[1] # Exit only on last block if len(data) < arvados.config.KEEP_BLOCK_SIZE: + # Simulate a checkpoint before quitting. + self.writer._update() raise SystemExit("Simulated error") return self.arvfile_write(*args, **kwargs) @@ -437,6 +482,8 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers, mocked_write.side_effect = wrapped_write writer = arv_put.ArvPutUploadJob([self.large_file_name], replication_desired=1) + # We'll be accessing from inside the wrapper + self.writer = writer with self.assertRaises(SystemExit): writer.start(save_collection=False) # Confirm that the file was partially uploaded @@ -472,7 +519,7 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers, replication_desired=1, dry_run=True, resume=False) - + del(self.writer) class ArvadosExpectedBytesTest(ArvadosBaseTestCase): TEST_SIZE = os.path.getsize(__file__)