2752: arv-put only caches state on failure.

This is slightly less robust, but writing the cache after every data
flush is too expensive.

diff --git a/sdk/python/arvados/commands/put.py b/sdk/python/arvados/commands/put.py
index 44f911e60b5ed354124b4f532feba157267859f7..86670266ae793f21d5456915ab1866daff05f1f2 100644 (file)
--- a/sdk/python/arvados/commands/put.py
+++ b/sdk/python/arvados/commands/put.py
@@ -250,9 +250,7 @@ class ArvPutCollectionWriter(arvados.ResumableCollectionWriter):
         print >>sys.stderr, "arv-put: Resuming previous upload.  Bypass with the --no-resume option."
         self.report_progress(self.bytes_written, self.bytes_expected)
 
-    def checkpoint_state(self):
-        if self.cache is None:
-            return
+    def cache_state(self):
         state = self.dump_state()
         # Transform attributes for serialization.
         for attr, value in state.items():
@@ -340,17 +338,21 @@ def main(arguments=None):
         print "arv-put: Another process is already uploading this data."
         sys.exit(1)
 
-    writer = ArvPutCollectionWriter.from_cache(
-        resume_cache, reporter, expected_bytes_for(args.paths))
-
-    # Copy file data to Keep.
-    for path in args.paths:
-        if os.path.isdir(path):
-            writer.write_directory_tree(
-                path, max_manifest_depth=args.max_manifest_depth)
-        else:
-            writer.start_new_stream()
-            writer.write_file(path, args.filename or os.path.basename(path))
+    try:
+        writer = ArvPutCollectionWriter.from_cache(
+            resume_cache, reporter, expected_bytes_for(args.paths))
+
+        # Copy file data to Keep.
+        for path in args.paths:
+            if os.path.isdir(path):
+                writer.write_directory_tree(
+                    path, max_manifest_depth=args.max_manifest_depth)
+            else:
+                writer.start_new_stream()
+                writer.write_file(path, args.filename or os.path.basename(path))
+    except (Exception, KeyboardInterrupt):
+        writer.cache_state()
+        raise
 
     if args.stream:
         print writer.manifest_text(),

diff --git a/sdk/python/tests/test_arv-put.py b/sdk/python/tests/test_arv-put.py
index 9623923dadedb2e015200a70702fcf9dab657a92..dde42e6f0ffa84e46e0ef9c0ebfcb621647e7538 100644 (file)
--- a/sdk/python/tests/test_arv-put.py
+++ b/sdk/python/tests/test_arv-put.py
@@ -199,6 +199,7 @@ class ArvadosPutCollectionWriterTest(ArvadosKeepLocalStoreTestCase):
     def test_writer_caches(self):
         cwriter = arv_put.ArvPutCollectionWriter(self.cache)
         cwriter.write_file('/dev/null')
+        cwriter.cache_state()
         self.assertTrue(self.cache.load())
         self.assertEquals(". 0:0:null\n", cwriter.manifest_text())
 
@@ -211,6 +212,7 @@ class ArvadosPutCollectionWriterTest(ArvadosKeepLocalStoreTestCase):
         cwriter = arv_put.ArvPutCollectionWriter(self.cache)
         with self.make_test_file() as testfile:
             cwriter.write_file(testfile.name, 'test')
+            cwriter.cache_state()
             new_writer = arv_put.ArvPutCollectionWriter.from_cache(
                 self.cache)
             self.assertEquals(
@@ -235,6 +237,7 @@ class ArvadosPutCollectionWriterTest(ArvadosKeepLocalStoreTestCase):
         # These bytes are intentionally not valid UTF-8.
         with self.make_test_file('\x00\x07\xe2') as testfile:
             cwriter.write_file(testfile.name, 'test')
+            cwriter.cache_state()
             new_writer = arv_put.ArvPutCollectionWriter.from_cache(
                 self.cache)
         self.assertEquals(cwriter.manifest_text(), new_writer.manifest_text())
@@ -261,7 +264,7 @@ class ArvadosPutCollectionWriterTest(ArvadosKeepLocalStoreTestCase):
             # Set up a writer with some flushed bytes.
             cwriter.write_file(testfile.name, 'test')
             cwriter.finish_current_stream()
-            cwriter.checkpoint_state()
+            cwriter.cache_state()
             # Restore a writer from that state and check its progress report.
             # We're also checking that progress is reported immediately after
             # resuming.