X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/1a169a434494175b208d0d5055bb42333d9b64b9..HEAD:/sdk/python/arvados/retry.py diff --git a/sdk/python/arvados/retry.py b/sdk/python/arvados/retry.py index 2168146a4b..e9e574f5df 100644 --- a/sdk/python/arvados/retry.py +++ b/sdk/python/arvados/retry.py @@ -15,21 +15,28 @@ It also provides utility functions for common operations with `RetryLoop`: # # SPDX-License-Identifier: Apache-2.0 -from builtins import range -from builtins import object import functools import inspect import pycurl import time from collections import deque +from typing import ( + Callable, + Generic, + Optional, + TypeVar, +) import arvados.errors _HTTP_SUCCESSES = set(range(200, 300)) _HTTP_CAN_RETRY = set([408, 409, 423, 500, 502, 503, 504]) -class RetryLoop(object): +CT = TypeVar('CT', bound=Callable) +T = TypeVar('T') + +class RetryLoop(Generic[T]): """Coordinate limited retries of code. `RetryLoop` coordinates a loop that runs until it records a @@ -49,38 +56,39 @@ class RetryLoop(object): Arguments: - num_retries: int - : The maximum number of times to retry the loop if it - doesn't succeed. This means the loop body could run at most + * num_retries: int --- The maximum number of times to retry the loop if + it doesn't succeed. This means the loop body could run at most `num_retries + 1` times. - success_check: Callable - : This is a function that will be called each - time the loop saves a result. The function should return - `True` if the result indicates the code succeeded, `False` if it - represents a permanent failure, and `None` if it represents a - temporary failure. If no function is provided, the loop will - end after any result is saved. - - backoff_start: float - : The number of seconds that must pass before the loop's second - iteration. Default 0, which disables all waiting. - - backoff_growth: float - : The wait time multiplier after each iteration. - Default 2 (i.e., double the wait time each time). - - save_results: int - : Specify a number to store that many saved results from the loop. - These are available through the `results` attribute, oldest first. - Default 1. - - max_wait: float - : Maximum number of seconds to wait between retries. Default 60. + * success_check: Callable[[T], bool | None] --- This is a function that + will be called each time the loop saves a result. The function should + return `True` if the result indicates the code succeeded, `False` if + it represents a permanent failure, and `None` if it represents a + temporary failure. If no function is provided, the loop will end + after any result is saved. + + * backoff_start: float --- The number of seconds that must pass before + the loop's second iteration. Default 0, which disables all waiting. + + * backoff_growth: float --- The wait time multiplier after each + iteration. Default 2 (i.e., double the wait time each time). + + * save_results: int --- Specify a number to store that many saved + results from the loop. These are available through the `results` + attribute, oldest first. Default 1. + + * max_wait: float --- Maximum number of seconds to wait between + retries. Default 60. """ - def __init__(self, num_retries, success_check=lambda r: True, - backoff_start=0, backoff_growth=2, save_results=1, - max_wait=60): + def __init__( + self, + num_retries: int, + success_check: Callable[[T], Optional[bool]]=lambda r: True, + backoff_start: float=0, + backoff_growth: float=2, + save_results: int=1, + max_wait: float=60 + ) -> None: self.tries_left = num_retries + 1 self.check_result = success_check self.backoff_wait = backoff_start @@ -92,11 +100,11 @@ class RetryLoop(object): self._running = None self._success = None - def __iter__(self): + def __iter__(self) -> 'RetryLoop': """Return an iterator of retries.""" return self - def running(self): + def running(self) -> Optional[bool]: """Return whether this loop is running. Returns `None` if the loop has never run, `True` if it is still running, @@ -105,7 +113,7 @@ class RetryLoop(object): """ return self._running and (self._success is None) - def __next__(self): + def __next__(self) -> int: """Record a loop attempt. If the loop is still running, decrements the number of tries left and @@ -126,7 +134,7 @@ class RetryLoop(object): self.tries_left -= 1 return self.tries_left - def save_result(self, result): + def save_result(self, result: T) -> None: """Record a loop result. Save the given result, and end the loop if it indicates @@ -138,8 +146,7 @@ class RetryLoop(object): Arguments: - result: Any - : The result from this loop attempt to check and save. + * result: T --- The result from this loop attempt to check and save. """ if not self.running(): raise arvados.errors.AssertionError( @@ -148,7 +155,7 @@ class RetryLoop(object): self._success = self.check_result(result) self._attempts += 1 - def success(self): + def success(self) -> Optional[bool]: """Return the loop's end state. Returns `True` if the loop recorded a successful result, `False` if it @@ -156,7 +163,7 @@ class RetryLoop(object): """ return self._success - def last_result(self): + def last_result(self) -> T: """Return the most recent result the loop saved. Raises `arvados.errors.AssertionError` if called before any result has @@ -168,7 +175,7 @@ class RetryLoop(object): raise arvados.errors.AssertionError( "queried loop results before any were recorded") - def attempts(self): + def attempts(self) -> int: """Return the number of results that have been saved. This count includes all kinds of results: success, permanent failure, @@ -176,7 +183,7 @@ class RetryLoop(object): """ return self._attempts - def attempts_str(self): + def attempts_str(self) -> str: """Return a human-friendly string counting saved results. This method returns '1 attempt' or 'N attempts', where the number @@ -188,7 +195,7 @@ class RetryLoop(object): return '{} attempts'.format(self._attempts) -def check_http_response_success(status_code): +def check_http_response_success(status_code: int) -> Optional[bool]: """Convert a numeric HTTP status code to a loop control flag. This method takes a numeric HTTP status code and returns `True` if @@ -207,8 +214,7 @@ def check_http_response_success(status_code): Arguments: - status_code: int - : A numeric HTTP response code + * status_code: int --- A numeric HTTP response code """ if status_code in _HTTP_SUCCESSES: return True @@ -219,7 +225,7 @@ def check_http_response_success(status_code): else: return None # Get well soon, server. -def retry_method(orig_func): +def retry_method(orig_func: CT) -> CT: """Provide a default value for a method's num_retries argument. This is a decorator for instance and class methods that accept a @@ -229,8 +235,8 @@ def retry_method(orig_func): Arguments: - orig_func: Callable - : A class or instance method that accepts a `num_retries` keyword argument + * orig_func: Callable --- A class or instance method that accepts a + `num_retries` keyword argument """ @functools.wraps(orig_func) def num_retries_setter(self, *args, **kwargs):