X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/56a99279c3fa45c95db7daa726a3fc0c071dffe1..7fde9a0a090e5f092b946f7569f1350778a47574:/sdk/python/tests/test_arv_put.py diff --git a/sdk/python/tests/test_arv_put.py b/sdk/python/tests/test_arv_put.py index 108d6611f3..667abbe63e 100644 --- a/sdk/python/tests/test_arv_put.py +++ b/sdk/python/tests/test_arv_put.py @@ -1,6 +1,12 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +from __future__ import absolute_import +from __future__ import division +from future import standard_library +standard_library.install_aliases() +from builtins import str +from builtins import range import apiclient import io import mock @@ -18,14 +24,14 @@ import threading import hashlib import random -from cStringIO import StringIO +from io import StringIO import arvados import arvados.commands.put as arv_put -import arvados_testutil as tutil +from . import arvados_testutil as tutil -from arvados_testutil import ArvadosBaseTestCase, fake_httplib2_response -import run_test_server +from .arvados_testutil import ArvadosBaseTestCase, fake_httplib2_response +from . import run_test_server class ArvadosPutResumeCacheTest(ArvadosBaseTestCase): CACHE_ARGSET = [ @@ -257,8 +263,8 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers, _, self.large_file_name = tempfile.mkstemp() fileobj = open(self.large_file_name, 'w') # Make sure to write just a little more than one block - for _ in range((arvados.config.KEEP_BLOCK_SIZE/(1024*1024))+1): - data = random.choice(['x', 'y', 'z']) * 1024 * 1024 # 1 MB + for _ in range((arvados.config.KEEP_BLOCK_SIZE>>20)+1): + data = random.choice(['x', 'y', 'z']) * 1024 * 1024 # 1 MiB fileobj.write(data) fileobj.close() # Temp dir containing small files to be repacked @@ -322,6 +328,8 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers, data = args[1] # Exit only on last block if len(data) < arvados.config.KEEP_BLOCK_SIZE: + # Simulate a checkpoint before quitting. Ensure block commit. + self.writer._update(final=True) raise SystemExit("Simulated error") return self.arvfile_write(*args, **kwargs) @@ -330,6 +338,8 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers, mocked_write.side_effect = wrapped_write writer = arv_put.ArvPutUploadJob([self.large_file_name], replication_desired=1) + # We'll be accessing from inside the wrapper + self.writer = writer with self.assertRaises(SystemExit): writer.start(save_collection=False) # Confirm that the file was partially uploaded @@ -343,6 +353,7 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers, self.assertEqual(writer.bytes_written + writer2.bytes_written - writer2.bytes_skipped, os.path.getsize(self.large_file_name)) writer2.destroy_cache() + del(self.writer) # Test for bug #11002 def test_graceful_exit_while_repacking_small_blocks(self): @@ -370,6 +381,8 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers, data = args[1] # Exit only on last block if len(data) < arvados.config.KEEP_BLOCK_SIZE: + # Simulate a checkpoint before quitting. + self.writer._update() raise SystemExit("Simulated error") return self.arvfile_write(*args, **kwargs) @@ -378,6 +391,8 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers, mocked_write.side_effect = wrapped_write writer = arv_put.ArvPutUploadJob([self.large_file_name], replication_desired=1) + # We'll be accessing from inside the wrapper + self.writer = writer with self.assertRaises(SystemExit): writer.start(save_collection=False) # Confirm that the file was partially uploaded @@ -393,12 +408,15 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers, self.assertEqual(writer2.bytes_written, os.path.getsize(self.large_file_name)) writer2.destroy_cache() + del(self.writer) def test_no_resume_when_no_cache(self): def wrapped_write(*args, **kwargs): data = args[1] # Exit only on last block if len(data) < arvados.config.KEEP_BLOCK_SIZE: + # Simulate a checkpoint before quitting. + self.writer._update() raise SystemExit("Simulated error") return self.arvfile_write(*args, **kwargs) @@ -407,6 +425,8 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers, mocked_write.side_effect = wrapped_write writer = arv_put.ArvPutUploadJob([self.large_file_name], replication_desired=1) + # We'll be accessing from inside the wrapper + self.writer = writer with self.assertRaises(SystemExit): writer.start(save_collection=False) # Confirm that the file was partially uploaded @@ -423,13 +443,15 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers, self.assertEqual(writer2.bytes_written, os.path.getsize(self.large_file_name)) writer2.destroy_cache() - + del(self.writer) def test_dry_run_feature(self): def wrapped_write(*args, **kwargs): data = args[1] # Exit only on last block if len(data) < arvados.config.KEEP_BLOCK_SIZE: + # Simulate a checkpoint before quitting. + self.writer._update() raise SystemExit("Simulated error") return self.arvfile_write(*args, **kwargs) @@ -438,6 +460,8 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers, mocked_write.side_effect = wrapped_write writer = arv_put.ArvPutUploadJob([self.large_file_name], replication_desired=1) + # We'll be accessing from inside the wrapper + self.writer = writer with self.assertRaises(SystemExit): writer.start(save_collection=False) # Confirm that the file was partially uploaded @@ -473,7 +497,7 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers, replication_desired=1, dry_run=True, resume=False) - + del(self.writer) class ArvadosExpectedBytesTest(ArvadosBaseTestCase): TEST_SIZE = os.path.getsize(__file__) @@ -506,7 +530,7 @@ class ArvadosPutReportTest(ArvadosBaseTestCase): def test_known_human_progress(self): for count, total in [(0, 1), (2, 4), (45, 60)]: - expect = '{:.1%}'.format(float(count) / total) + expect = '{:.1%}'.format(1.0*count/total) actual = arv_put.human_progress(count, total) self.assertTrue(actual.startswith('\r')) self.assertIn(expect, actual)