12431: Use libcloud fork 2.2.2.dev1

[arvados.git] / services / nodemanager / arvnodeman / computenode / __init__.py

diff --git a/services/nodemanager/arvnodeman/computenode/__init__.py b/services/nodemanager/arvnodeman/computenode/__init__.py
index ce114e1279b4135fdb58bf7c7b2b07c6e70a07e3..4e46a438dbf1ea04fb066ccdc45b66f4b196ceae 100644 (file)
--- a/services/nodemanager/arvnodeman/computenode/__init__.py
+++ b/services/nodemanager/arvnodeman/computenode/__init__.py
@@ -61,10 +61,9 @@ class RetryMixin(object):
     is a timer actor.)
 
     """
-    def __init__(self, retry_wait, max_retry_wait,
-                 logger, cloud, timer=None):
-        self.min_retry_wait = retry_wait
-        self.max_retry_wait = max_retry_wait
+    def __init__(self, retry_wait, max_retry_wait, logger, cloud, timer=None):
+        self.min_retry_wait = max(1, retry_wait)
+        self.max_retry_wait = max(self.min_retry_wait, max_retry_wait)
         self.retry_wait = retry_wait
         self._logger = logger
         self._cloud = cloud
@@ -80,16 +79,25 @@ class RetryMixin(object):
                     try:
                         ret = orig_func(self, *args, **kwargs)
                     except RateLimitReachedError as error:
-                        self.retry_wait = error.retry_after
+                        # If retry-after is zero, continue with exponential
+                        # backoff.
+                        if error.retry_after != 0:
+                            self.retry_wait = error.retry_after
                         should_retry = True
                     except BaseHTTPError as error:
                         if error.headers and error.headers.get("retry-after"):
                             try:
-                                self.retry_wait = \
-                                    int(error.headers["retry-after"])
+                                retry_after = int(error.headers["retry-after"])
+                                # If retry-after is zero, continue with
+                                # exponential backoff.
+                                if retry_after != 0:
+                                    self.retry_wait = retry_after
                                 should_retry = True
                             except ValueError:
-                                pass
+                                self._logger.warning(
+                                    "Unrecognizable Retry-After header: %r",
+                                    error.headers["retry-after"],
+                                    exc_info=error)
                         if error.code == 429 or error.code >= 500:
                             should_retry = True
                     except CLOUD_ERRORS as error:
@@ -106,9 +114,6 @@ class RetryMixin(object):
                         self.retry_wait = self.min_retry_wait
                         return ret
 
-                    if self.retry_wait < 0 or self.retry_wait > self.max_retry_wait:
-                        self.retry_wait = self.max_retry_wait
-
                     # Only got here if an exception was caught.  Now determine what to do about it.
                     if not should_retry:
                         self.retry_wait = self.min_retry_wait
@@ -117,6 +122,12 @@ class RetryMixin(object):
                             error, exc_info=error)
                         raise
 
+                    # Retry wait out of bounds?
+                    if self.retry_wait < self.min_retry_wait:
+                        self.retry_wait = self.min_retry_wait
+                    elif self.retry_wait > self.max_retry_wait:
+                        self.retry_wait = self.max_retry_wait
+
                     self._logger.warning(
                         "Client error: %s - %s %s seconds",
                         error,