X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/b887df487c6cb465c3212e8ef99278350e58ff0a..fd0074f2200bc41bc63be770fffbe2446fb0cc03:/sdk/python/arvados/keep.py

diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index 62c6709bb6..6196b50202 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -27,6 +27,22 @@ import arvados.errors
 import arvados.retry as retry
 import arvados.util
 
+try:
+    # Workaround for urllib3 bug.
+    # The 'requests' library enables urllib3's SNI support by default, which uses pyopenssl.
+    # However, urllib3 prior to version 1.10 has a major bug in this feature
+    # (OpenSSL WantWriteError, https://github.com/shazow/urllib3/issues/412)
+    # Unfortunately Debian 8 is stabilizing on urllib3 1.9.1 which means the
+    # following workaround is necessary to be able to use
+    # the arvados python sdk with the distribution-provided packages.
+    import urllib3
+    from pkg_resources import parse_version
+    if parse_version(urllib3.__version__) < parse_version('1.10'):
+        from urllib3.contrib import pyopenssl
+        pyopenssl.extract_from_urllib3()
+except ImportError:
+    pass
+
 _logger = logging.getLogger('arvados.keep')
 global_client_object = None
 
@@ -433,16 +449,21 @@ class KeepClient(object):
           KeepClient does not use a proxy, pass in an empty string.
 
         :timeout:
-          The timeout (in seconds) for HTTP requests to Keep
+          The initial timeout (in seconds) for HTTP requests to Keep
           non-proxy servers.  A tuple of two floats is interpreted as
           (connection_timeout, read_timeout): see
           http://docs.python-requests.org/en/latest/user/advanced/#timeouts.
+          Because timeouts are often a result of transient server load, the
+          actual connection timeout will be increased by a factor of two on
+          each retry.
           Default: (2, 300).
 
         :proxy_timeout:
-          The timeout (in seconds) for HTTP requests to
+          The initial timeout (in seconds) for HTTP requests to
           Keep proxies. A tuple of two floats is interpreted as
-          (connection_timeout, read_timeout). Default: (20, 300).
+          (connection_timeout, read_timeout). The behavior described
+          above for adjusting connection timeouts on retry also applies.
+          Default: (20, 300).
 
         :api_token:
           If you're not using an API client, but only talking
@@ -514,14 +535,20 @@ class KeepClient(object):
                 self.using_proxy = None
                 self._static_services_list = False
 
-    def current_timeout(self):
-        """Return the appropriate timeout to use for this client: the proxy
-        timeout setting if the backend service is currently a proxy,
-        the regular timeout setting otherwise.
+    def current_timeout(self, attempt_number):
+        """Return the appropriate timeout to use for this client.
+
+        The proxy timeout setting if the backend service is currently a proxy,
+        the regular timeout setting otherwise.  The `attempt_number` indicates
+        how many times the operation has been tried already (starting from 0
+        for the first try), and scales the connection timeout portion of the
+        return value accordingly.
+
         """
         # TODO(twp): the timeout should be a property of a
         # KeepService, not a KeepClient. See #4488.
-        return self.proxy_timeout if self.using_proxy else self.timeout
+        t = self.proxy_timeout if self.using_proxy else self.timeout
+        return (t[0] * (1 << attempt_number), t[1])
 
     def build_services_list(self, force_rebuild=False):
         if (self._static_services_list or
@@ -670,7 +697,7 @@ class KeepClient(object):
                                for root in (local_roots + hint_roots)
                                if roots_map[root].usable()]
             for keep_service in services_to_try:
-                blob = keep_service.get(locator, timeout=self.current_timeout())
+                blob = keep_service.get(locator, timeout=self.current_timeout(num_retries-tries_left))
                 if blob is not None:
                     break
             loop.save_result((blob, len(services_to_try)))
@@ -759,7 +786,7 @@ class KeepClient(object):
                     data_hash=data_hash,
                     service_root=service_root,
                     thread_limiter=thread_limiter,
-                    timeout=self.current_timeout())
+                    timeout=self.current_timeout(num_retries-tries_left))
                 t.start()
                 threads.append(t)
             for t in threads: