Merge branch 'master' into 7454-azure-custom-data

[arvados.git] / sdk / python / arvados / api.py

diff --git a/sdk/python/arvados/api.py b/sdk/python/arvados/api.py
index 3968f0185e1a193a6b233c5033be4a0b7ad8794e..e2e8ba19b64823af233c25eba1f90880dc9f4067 100644 (file)
--- a/sdk/python/arvados/api.py
+++ b/sdk/python/arvados/api.py
@@ -1,8 +1,12 @@
+import collections
+import httplib
 import httplib2
 import json
 import logging
 import os
 import re
+import socket
+import time
 import types
 
 import apiclient
@@ -14,9 +18,32 @@ import util
 
 _logger = logging.getLogger('arvados.api')
 
-def _intercept_http_request(self, uri, **kwargs):
-    from httplib import BadStatusLine
+MAX_IDLE_CONNECTION_DURATION = 30
+RETRY_DELAY_INITIAL = 2
+RETRY_DELAY_BACKOFF = 2
+RETRY_COUNT = 2
+
+class OrderedJsonModel(apiclient.model.JsonModel):
+    """Model class for JSON that preserves the contents' order.
+
+    API clients that care about preserving the order of fields in API
+    server responses can use this model to do so, like this::
+
+        from arvados.api import OrderedJsonModel
+        client = arvados.api('v1', ..., model=OrderedJsonModel())
+    """
 
+    def deserialize(self, content):
+        # This is a very slightly modified version of the parent class'
+        # implementation.  Copyright (c) 2010 Google.
+        content = content.decode('utf-8')
+        body = json.loads(content, object_pairs_hook=collections.OrderedDict)
+        if self._data_wrapper and isinstance(body, dict) and 'data' in body:
+            body = body['data']
+        return body
+
+
+def _intercept_http_request(self, uri, **kwargs):
     if (self.max_request_size and
         kwargs.get('body') and
         self.max_request_size < len(kwargs['body'])):
@@ -29,22 +56,54 @@ def _intercept_http_request(self, uri, **kwargs):
         kwargs['headers']['X-External-Client'] = '1'
 
     kwargs['headers']['Authorization'] = 'OAuth2 %s' % self.arvados_api_token
-    try:
-        return self.orig_http_request(uri, **kwargs)
-    except BadStatusLine:
-        # This is how httplib tells us that it tried to reuse an
-        # existing connection but it was already closed by the
-        # server. In that case, yes, we would like to retry.
-        # Unfortunately, we are not absolutely certain that the
-        # previous call did not succeed, so this is slightly
-        # risky.
-        return self.orig_http_request(uri, **kwargs)
+
+    retryable = kwargs.get('method', 'GET') in [
+        'DELETE', 'GET', 'HEAD', 'OPTIONS', 'PUT']
+    retry_count = self._retry_count if retryable else 0
+
+    if (not retryable and
+        time.time() - self._last_request_time > self._max_keepalive_idle):
+        # High probability of failure due to connection atrophy. Make
+        # sure this request [re]opens a new connection by closing and
+        # forgetting all cached connections first.
+        for conn in self.connections.itervalues():
+            conn.close()
+        self.connections.clear()
+
+    delay = self._retry_delay_initial
+    for _ in range(retry_count):
+        self._last_request_time = time.time()
+        try:
+            return self.orig_http_request(uri, **kwargs)
+        except httplib.HTTPException:
+            _logger.debug("Retrying API request in %d s after HTTP error",
+                          delay, exc_info=True)
+        except socket.error:
+            # This is the one case where httplib2 doesn't close the
+            # underlying connection first.  Close all open
+            # connections, expecting this object only has the one
+            # connection to the API server.  This is safe because
+            # httplib2 reopens connections when needed.
+            _logger.debug("Retrying API request in %d s after socket error",
+                          delay, exc_info=True)
+            for conn in self.connections.itervalues():
+                conn.close()
+        time.sleep(delay)
+        delay = delay * self._retry_delay_backoff
+
+    self._last_request_time = time.time()
+    return self.orig_http_request(uri, **kwargs)
 
 def _patch_http_request(http, api_token):
     http.arvados_api_token = api_token
     http.max_request_size = 0
     http.orig_http_request = http.request
     http.request = types.MethodType(_intercept_http_request, http)
+    http._last_request_time = 0
+    http._max_keepalive_idle = MAX_IDLE_CONNECTION_DURATION
+    http._retry_delay_initial = RETRY_DELAY_INITIAL
+    http._retry_delay_backoff = RETRY_DELAY_BACKOFF
+    http._retry_count = RETRY_COUNT
     return http
 
 # Monkey patch discovery._cast() so objects and arrays get serialized
@@ -69,7 +128,10 @@ def _new_http_error(cls, *args, **kwargs):
 apiclient_errors.HttpError.__new__ = staticmethod(_new_http_error)
 
 def http_cache(data_type):
-    path = os.environ['HOME'] + '/.cache/arvados/' + data_type
+    homedir = os.environ.get('HOME')
+    if not homedir or len(homedir) == 0:
+        return None
+    path = homedir + '/.cache/arvados/' + data_type
     try:
         util.mkdir_dash_p(path)
     except OSError:
@@ -134,11 +196,7 @@ def api(version=None, cache=True, host=None, token=None, insecure=False, **kwarg
             'https://%s/discovery/v1/apis/{api}/{apiVersion}/rest' % (host,))
 
     if 'http' not in kwargs:
-        http_kwargs = {}
-        # Prefer system's CA certificates (if available) over httplib2's.
-        certs_path = '/etc/ssl/certs/ca-certificates.crt'
-        if os.path.exists(certs_path):
-            http_kwargs['ca_certs'] = certs_path
+        http_kwargs = {'ca_certs': util.ca_certs_path()}
         if cache:
             http_kwargs['cache'] = http_cache('discovery')
         if insecure: