X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/391a1d8378b4bc6b17b71904b3d6494160b51627..7fde9a0a090e5f092b946f7569f1350778a47574:/sdk/python/tests/test_arv_put.py diff --git a/sdk/python/tests/test_arv_put.py b/sdk/python/tests/test_arv_put.py index 898349c032..667abbe63e 100644 --- a/sdk/python/tests/test_arv_put.py +++ b/sdk/python/tests/test_arv_put.py @@ -1,7 +1,14 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +from __future__ import absolute_import +from __future__ import division +from future import standard_library +standard_library.install_aliases() +from builtins import str +from builtins import range import apiclient +import io import mock import os import pwd @@ -17,23 +24,21 @@ import threading import hashlib import random -from cStringIO import StringIO +from io import StringIO import arvados import arvados.commands.put as arv_put -import arvados_testutil as tutil +from . import arvados_testutil as tutil -from arvados_testutil import ArvadosBaseTestCase, fake_httplib2_response -import run_test_server +from .arvados_testutil import ArvadosBaseTestCase, fake_httplib2_response +from . import run_test_server class ArvadosPutResumeCacheTest(ArvadosBaseTestCase): CACHE_ARGSET = [ [], ['/dev/null'], ['/dev/null', '--filename', 'empty'], - ['/tmp'], - ['/tmp', '--max-manifest-depth', '0'], - ['/tmp', '--max-manifest-depth', '1'] + ['/tmp'] ] def tearDown(self): @@ -258,20 +263,27 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers, _, self.large_file_name = tempfile.mkstemp() fileobj = open(self.large_file_name, 'w') # Make sure to write just a little more than one block - for _ in range((arvados.config.KEEP_BLOCK_SIZE/(1024*1024))+1): - data = random.choice(['x', 'y', 'z']) * 1024 * 1024 # 1 MB + for _ in range((arvados.config.KEEP_BLOCK_SIZE>>20)+1): + data = random.choice(['x', 'y', 'z']) * 1024 * 1024 # 1 MiB fileobj.write(data) fileobj.close() + # Temp dir containing small files to be repacked + self.small_files_dir = tempfile.mkdtemp() + data = 'y' * 1024 * 1024 # 1 MB + for i in range(1, 70): + with open(os.path.join(self.small_files_dir, str(i)), 'w') as f: + f.write(data + str(i)) self.arvfile_write = getattr(arvados.arvfile.ArvadosFileWriter, 'write') def tearDown(self): super(ArvPutUploadJobTest, self).tearDown() shutil.rmtree(self.tempdir) os.unlink(self.large_file_name) + shutil.rmtree(self.small_files_dir) def test_writer_works_without_cache(self): cwriter = arv_put.ArvPutUploadJob(['/dev/null'], resume=False) - cwriter.start() + cwriter.start(save_collection=False) self.assertEqual(". d41d8cd98f00b204e9800998ecf8427e+0 0:0:null\n", cwriter.manifest_text()) def test_writer_works_with_cache(self): @@ -279,13 +291,13 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers, f.write('foo') f.flush() cwriter = arv_put.ArvPutUploadJob([f.name]) - cwriter.start() - self.assertEqual(3, cwriter.bytes_written) + cwriter.start(save_collection=False) + self.assertEqual(3, cwriter.bytes_written - cwriter.bytes_skipped) # Don't destroy the cache, and start another upload cwriter_new = arv_put.ArvPutUploadJob([f.name]) - cwriter_new.start() + cwriter_new.start(save_collection=False) cwriter_new.destroy_cache() - self.assertEqual(0, cwriter_new.bytes_written) + self.assertEqual(0, cwriter_new.bytes_written - cwriter_new.bytes_skipped) def make_progress_tester(self): progression = [] @@ -301,13 +313,13 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers, progression, reporter = self.make_progress_tester() cwriter = arv_put.ArvPutUploadJob([f.name], reporter=reporter, bytes_expected=expect_count) - cwriter.start() + cwriter.start(save_collection=False) cwriter.destroy_cache() self.assertIn((3, expect_count), progression) def test_writer_upload_directory(self): cwriter = arv_put.ArvPutUploadJob([self.tempdir]) - cwriter.start() + cwriter.start(save_collection=False) cwriter.destroy_cache() self.assertEqual(1024*(1+2+3+4+5), cwriter.bytes_written) @@ -316,6 +328,8 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers, data = args[1] # Exit only on last block if len(data) < arvados.config.KEEP_BLOCK_SIZE: + # Simulate a checkpoint before quitting. Ensure block commit. + self.writer._update(final=True) raise SystemExit("Simulated error") return self.arvfile_write(*args, **kwargs) @@ -324,18 +338,166 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers, mocked_write.side_effect = wrapped_write writer = arv_put.ArvPutUploadJob([self.large_file_name], replication_desired=1) + # We'll be accessing from inside the wrapper + self.writer = writer with self.assertRaises(SystemExit): - writer.start() - self.assertLess(writer.bytes_written, - os.path.getsize(self.large_file_name)) + writer.start(save_collection=False) + # Confirm that the file was partially uploaded + self.assertGreater(writer.bytes_written, 0) + self.assertLess(writer.bytes_written, + os.path.getsize(self.large_file_name)) # Retry the upload writer2 = arv_put.ArvPutUploadJob([self.large_file_name], replication_desired=1) - writer2.start() - self.assertEqual(writer.bytes_written + writer2.bytes_written, + writer2.start(save_collection=False) + self.assertEqual(writer.bytes_written + writer2.bytes_written - writer2.bytes_skipped, os.path.getsize(self.large_file_name)) writer2.destroy_cache() + del(self.writer) + + # Test for bug #11002 + def test_graceful_exit_while_repacking_small_blocks(self): + def wrapped_commit(*args, **kwargs): + raise SystemExit("Simulated error") + + with mock.patch('arvados.arvfile._BlockManager.commit_bufferblock', + autospec=True) as mocked_commit: + mocked_commit.side_effect = wrapped_commit + # Upload a little more than 1 block, wrapped_commit will make the first block + # commit to fail. + # arv-put should not exit with an exception by trying to commit the collection + # as it's in an inconsistent state. + writer = arv_put.ArvPutUploadJob([self.small_files_dir], + replication_desired=1) + try: + with self.assertRaises(SystemExit): + writer.start(save_collection=False) + except arvados.arvfile.UnownedBlockError: + self.fail("arv-put command is trying to use a corrupted BlockManager. See https://dev.arvados.org/issues/11002") + writer.destroy_cache() + def test_no_resume_when_asked(self): + def wrapped_write(*args, **kwargs): + data = args[1] + # Exit only on last block + if len(data) < arvados.config.KEEP_BLOCK_SIZE: + # Simulate a checkpoint before quitting. + self.writer._update() + raise SystemExit("Simulated error") + return self.arvfile_write(*args, **kwargs) + + with mock.patch('arvados.arvfile.ArvadosFileWriter.write', + autospec=True) as mocked_write: + mocked_write.side_effect = wrapped_write + writer = arv_put.ArvPutUploadJob([self.large_file_name], + replication_desired=1) + # We'll be accessing from inside the wrapper + self.writer = writer + with self.assertRaises(SystemExit): + writer.start(save_collection=False) + # Confirm that the file was partially uploaded + self.assertGreater(writer.bytes_written, 0) + self.assertLess(writer.bytes_written, + os.path.getsize(self.large_file_name)) + # Retry the upload, this time without resume + writer2 = arv_put.ArvPutUploadJob([self.large_file_name], + replication_desired=1, + resume=False) + writer2.start(save_collection=False) + self.assertEqual(writer2.bytes_skipped, 0) + self.assertEqual(writer2.bytes_written, + os.path.getsize(self.large_file_name)) + writer2.destroy_cache() + del(self.writer) + + def test_no_resume_when_no_cache(self): + def wrapped_write(*args, **kwargs): + data = args[1] + # Exit only on last block + if len(data) < arvados.config.KEEP_BLOCK_SIZE: + # Simulate a checkpoint before quitting. + self.writer._update() + raise SystemExit("Simulated error") + return self.arvfile_write(*args, **kwargs) + + with mock.patch('arvados.arvfile.ArvadosFileWriter.write', + autospec=True) as mocked_write: + mocked_write.side_effect = wrapped_write + writer = arv_put.ArvPutUploadJob([self.large_file_name], + replication_desired=1) + # We'll be accessing from inside the wrapper + self.writer = writer + with self.assertRaises(SystemExit): + writer.start(save_collection=False) + # Confirm that the file was partially uploaded + self.assertGreater(writer.bytes_written, 0) + self.assertLess(writer.bytes_written, + os.path.getsize(self.large_file_name)) + # Retry the upload, this time without cache usage + writer2 = arv_put.ArvPutUploadJob([self.large_file_name], + replication_desired=1, + resume=False, + use_cache=False) + writer2.start(save_collection=False) + self.assertEqual(writer2.bytes_skipped, 0) + self.assertEqual(writer2.bytes_written, + os.path.getsize(self.large_file_name)) + writer2.destroy_cache() + del(self.writer) + + def test_dry_run_feature(self): + def wrapped_write(*args, **kwargs): + data = args[1] + # Exit only on last block + if len(data) < arvados.config.KEEP_BLOCK_SIZE: + # Simulate a checkpoint before quitting. + self.writer._update() + raise SystemExit("Simulated error") + return self.arvfile_write(*args, **kwargs) + + with mock.patch('arvados.arvfile.ArvadosFileWriter.write', + autospec=True) as mocked_write: + mocked_write.side_effect = wrapped_write + writer = arv_put.ArvPutUploadJob([self.large_file_name], + replication_desired=1) + # We'll be accessing from inside the wrapper + self.writer = writer + with self.assertRaises(SystemExit): + writer.start(save_collection=False) + # Confirm that the file was partially uploaded + self.assertGreater(writer.bytes_written, 0) + self.assertLess(writer.bytes_written, + os.path.getsize(self.large_file_name)) + # Retry the upload using dry_run to check if there is a pending upload + writer2 = arv_put.ArvPutUploadJob([self.large_file_name], + replication_desired=1, + dry_run=True) + with self.assertRaises(arv_put.ArvPutUploadIsPending): + writer2.start(save_collection=False) + # Complete the pending upload + writer3 = arv_put.ArvPutUploadJob([self.large_file_name], + replication_desired=1) + writer3.start(save_collection=False) + # Confirm there's no pending upload with dry_run=True + writer4 = arv_put.ArvPutUploadJob([self.large_file_name], + replication_desired=1, + dry_run=True) + with self.assertRaises(arv_put.ArvPutUploadNotPending): + writer4.start(save_collection=False) + writer4.destroy_cache() + # Test obvious cases + with self.assertRaises(arv_put.ArvPutUploadIsPending): + arv_put.ArvPutUploadJob([self.large_file_name], + replication_desired=1, + dry_run=True, + resume=False, + use_cache=False) + with self.assertRaises(arv_put.ArvPutUploadIsPending): + arv_put.ArvPutUploadJob([self.large_file_name], + replication_desired=1, + dry_run=True, + resume=False) + del(self.writer) class ArvadosExpectedBytesTest(ArvadosBaseTestCase): TEST_SIZE = os.path.getsize(__file__) @@ -368,7 +530,7 @@ class ArvadosPutReportTest(ArvadosBaseTestCase): def test_known_human_progress(self): for count, total in [(0, 1), (2, 4), (45, 60)]: - expect = '{:.1%}'.format(float(count) / total) + expect = '{:.1%}'.format(1.0*count/total) actual = arv_put.human_progress(count, total) self.assertTrue(actual.startswith('\r')) self.assertIn(expect, actual) @@ -409,6 +571,15 @@ class ArvadosPutTest(run_test_server.TestCaseWithServers, ArvadosBaseTestCase): delattr(self, outbuf) super(ArvadosPutTest, self).tearDown() + def test_version_argument(self): + err = io.BytesIO() + out = io.BytesIO() + with tutil.redirected_streams(stdout=out, stderr=err): + with self.assertRaises(SystemExit): + self.call_main_with_args(['--version']) + self.assertEqual(out.getvalue(), '') + self.assertRegexpMatches(err.getvalue(), "[0-9]+\.[0-9]+\.[0-9]+") + def test_simple_file_put(self): self.call_main_on_test_file()