8281: Fix arv-mount ignoring --retries argument when writing file data.

"num_retries" arguments get passed around extensively in arvfile.py
and collection.py in the Python SDK, but ultimately the writing of
file data is done by a _BlockManager which doesn't have any way to
accept that argument or pass it along to a KeepClient, so PUT requests
always use the CollectionWriter's KeepClient's default num_retries.

In arv-mount's case, we have been telling CollectionWriter the
num_retries we want. When CollectionWriter creates a KeepClient,
num_retries gets passed along -- normally this works around the fact
that num_retries gets lost by the _BlockManager layer. However, we
provided our own KeepClient to use instead of letting CollectionWriter
create one, and we forgot to set num_retries on our own KeepClient, so
we weren't retrying PUT requests.

diff --git a/services/fuse/arvados_fuse/command.py b/services/fuse/arvados_fuse/command.py
index cd81a5390a2bdf53807711174677a2bc8f1b47be..71623a5f3d07364bb4de94c42023af3b5658fc55 100644 (file)
--- a/services/fuse/arvados_fuse/command.py
+++ b/services/fuse/arvados_fuse/command.py
@@ -151,7 +151,8 @@ class Mount(object):
         self.api = arvados.safeapi.ThreadSafeApiCache(
             apiconfig=arvados.config.settings(),
             keep_params={
-                "block_cache": arvados.keep.KeepBlockCache(self.args.file_cache)
+                'block_cache': arvados.keep.KeepBlockCache(self.args.file_cache),
+                'num_retries': self.args.retries,
             })
         # Do a sanity check that we have a working arvados host + token.
         self.api.users().current().execute()

diff --git a/services/fuse/arvados_fuse/fusedir.py b/services/fuse/arvados_fuse/fusedir.py
index 00efab765a6d59b52f33b62b80847247fb368c1e..093657e71ca88f18ecdf107f1a9e936dd17b9530 100644 (file)
--- a/services/fuse/arvados_fuse/fusedir.py
+++ b/services/fuse/arvados_fuse/fusedir.py
@@ -500,7 +500,8 @@ class TmpCollectionDirectory(CollectionDirectoryBase):
     def __init__(self, parent_inode, inodes, api_client, num_retries):
         collection = self.UnsaveableCollection(
             api_client=api_client,
-            keep_client=api_client.keep)
+            keep_client=api_client.keep,
+            num_retries=num_retries)
         super(TmpCollectionDirectory, self).__init__(
             parent_inode, inodes, collection)
         self.collection_record_file = None

diff --git a/services/fuse/tests/test_retry.py b/services/fuse/tests/test_retry.py
new file mode 100644 (file)
index 0000000..f6c0807
--- /dev/null
+++ b/services/fuse/tests/test_retry.py
@@ -0,0 +1,35 @@
+import arvados
+import arvados_fuse.command
+import mock
+import os
+import run_test_server
+import tempfile
+import unittest
+
+class KeepClientRetry(unittest.TestCase):
+    origKeepClient = arvados.keep.KeepClient
+
+    def setUp(self):
+        self.mnt = tempfile.mkdtemp()
+        run_test_server.authorize_with('active')
+
+    def tearDown(self):
+        os.rmdir(self.mnt)
+
+    @mock.patch('arvados_fuse.arvados.keep.KeepClient')
+    def _test_retry(self, num_retries, argv, kc):
+        kc.side_effect = lambda *args, **kw: self.origKeepClient(*args, **kw)
+        with arvados_fuse.command.Mount(
+                arvados_fuse.command.ArgumentParser().parse_args(
+                    argv+[self.mnt])):
+            pass
+        self.assertEqual(num_retries, kc.call_args[1].get('num_retries'))
+
+    def test_default_retry_3(self):
+        self._test_retry(3, [])
+
+    def test_retry_2(self):
+        self._test_retry(2, ['--retries=2'])
+
+    def test_no_retry(self):
+        self._test_retry(0, ['--retries=0'])