-#!/usr/bin/env python
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+from builtins import range
+from builtins import object
import functools
import inspect
+import pycurl
import time
from collections import deque
import arvados.errors
-_HTTP_SUCCESSES = set(xrange(200, 300))
+_HTTP_SUCCESSES = set(range(200, 300))
_HTTP_CAN_RETRY = set([408, 409, 422, 423, 500, 502, 503, 504])
class RetryLoop(object):
return loop.last_result()
"""
def __init__(self, num_retries, success_check=lambda r: True,
- backoff_start=0, backoff_growth=2, save_results=1):
+ backoff_start=0, backoff_growth=2, save_results=1,
+ max_wait=60):
"""Construct a new RetryLoop.
Arguments:
* save_results: Specify a number to save the last N results
that the loop recorded. These records are available through
the results attribute, oldest first. Default 1.
+ * max_wait: Maximum number of seconds to wait between retries.
"""
self.tries_left = num_retries + 1
self.check_result = success_check
self.backoff_wait = backoff_start
self.backoff_growth = backoff_growth
+ self.max_wait = max_wait
self.next_start_time = 0
self.results = deque(maxlen=save_results)
+ self._attempts = 0
self._running = None
self._success = None
def running(self):
return self._running and (self._success is None)
- def next(self):
+ def __next__(self):
if self._running is None:
self._running = True
if (self.tries_left < 1) or not self.running():
wait_time = max(0, self.next_start_time - time.time())
time.sleep(wait_time)
self.backoff_wait *= self.backoff_growth
+ if self.backoff_wait > self.max_wait:
+ self.backoff_wait = self.max_wait
self.next_start_time = time.time() + self.backoff_wait
self.tries_left -= 1
return self.tries_left
"recorded a loop result after the loop finished")
self.results.append(result)
self._success = self.check_result(result)
+ self._attempts += 1
def success(self):
"""Return the loop's end state.
raise arvados.errors.AssertionError(
"queried loop results before any were recorded")
+ def attempts(self):
+ """Return the number of attempts that have been made.
-def check_http_response_success(result):
- """Convert a 'requests' response to a loop control flag.
+ Includes successes and failures."""
+ return self._attempts
- Pass this method a requests.Response object. It returns True if
- the response indicates success, None if it indicates temporary
+ def attempts_str(self):
+ """Human-readable attempts(): 'N attempts' or '1 attempt'"""
+ if self._attempts == 1:
+ return '1 attempt'
+ else:
+ return '{} attempts'.format(self._attempts)
+
+
+def check_http_response_success(status_code):
+ """Convert an HTTP status code to a loop control flag.
+
+ Pass this method a numeric HTTP status code. It returns True if
+ the code indicates success, None if it indicates temporary
failure, and False otherwise. You can use this as the
success_check for a RetryLoop.
3xx status codes. They don't indicate success, and you can't
retry those requests verbatim.
"""
- try:
- status = result.status_code
- except Exception:
- return None
- if status in _HTTP_SUCCESSES:
+ if status_code in _HTTP_SUCCESSES:
return True
- elif status in _HTTP_CAN_RETRY:
+ elif status_code in _HTTP_CAN_RETRY:
return None
- elif 100 <= status < 600:
+ elif 100 <= status_code < 600:
return False
else:
return None # Get well soon, server.
"""
@functools.wraps(orig_func)
def num_retries_setter(self, *args, **kwargs):
- arg_vals = inspect.getcallargs(orig_func, self, *args, **kwargs)
- if arg_vals['num_retries'] is None:
+ if kwargs.get('num_retries') is None:
kwargs['num_retries'] = self.num_retries
return orig_func(self, *args, **kwargs)
return num_retries_setter
projects / arvados.git / blobdiff