# Stop the thread before doing anything else
self._stop_checkpointer.set()
self._checkpointer.join()
- # Commit all pending blocks & one last _update()
- self._local_collection.manifest_text()
- self._update(final=True)
- if save_collection:
- self.save_collection()
+ try:
+ # Commit all pending blocks & one last _update()
+ self._local_collection.manifest_text()
+ self._update(final=True)
+ if save_collection:
+ self.save_collection()
+ except AttributeError:
+ # Exception caught in inconsistent state, finish as is.
+ self.logger.warning("Couldn't save last checkpoint while exiting.")
if self.use_cache:
self._cache_file.close()
data = random.choice(['x', 'y', 'z']) * 1024 * 1024 # 1 MB
fileobj.write(data)
fileobj.close()
+ # Temp dir containing small files to be repacked
+ self.small_files_dir = tempfile.mkdtemp()
+ data = 'y' * 1024 * 1024 # 1 MB
+ for i in range(1, 70):
+ with open(os.path.join(self.small_files_dir, str(i)), 'w') as f:
+ f.write(data + str(i))
self.arvfile_write = getattr(arvados.arvfile.ArvadosFileWriter, 'write')
def tearDown(self):
super(ArvPutUploadJobTest, self).tearDown()
shutil.rmtree(self.tempdir)
os.unlink(self.large_file_name)
+ shutil.rmtree(self.small_files_dir)
def test_writer_works_without_cache(self):
cwriter = arv_put.ArvPutUploadJob(['/dev/null'], resume=False)
os.path.getsize(self.large_file_name))
writer2.destroy_cache()
+ # Test for bug #11002
+ def test_graceful_exit_while_repacking_small_blocks(self):
+ def wrapped_commit(*args, **kwargs):
+ raise SystemExit("Simulated error")
+
+ with mock.patch('arvados.arvfile._BlockManager.commit_bufferblock',
+ autospec=True) as mocked_commit:
+ mocked_commit.side_effect = wrapped_commit
+ # Upload a little more than 1 block, wrapped_commit will make the first block
+ # commit to fail.
+ # arv-put should not exit with an exception by trying to commit the collection
+ # as it's in an inconsistent state.
+ writer = arv_put.ArvPutUploadJob([self.small_files_dir],
+ replication_desired=1)
+ try:
+ with self.assertRaises(SystemExit):
+ writer.start(save_collection=False)
+ except AttributeError:
+ self.fail("arv-put command is trying to use a corrupted BlockManager. See https://dev.arvados.org/issues/11002")
+ writer.destroy_cache()
+
def test_no_resume_when_asked(self):
def wrapped_write(*args, **kwargs):
data = args[1]