1 # Copyright (C) The Arvados Authors. All rights reserved.
3 # SPDX-License-Identifier: Apache-2.0
5 from builtins import zip
6 from builtins import range
7 from builtins import object
11 from unittest import mock
13 import arvados.errors as arv_error
14 import arvados.retry as arv_retry
16 class RetryLoopTestMixin(object):
18 def loop_success(result):
19 # During the tests, we use integers that look like HTTP status
20 # codes as loop results. Then we define simplified HTTP
21 # heuristics here to decide whether the result is success (True),
22 # permanent failure (False), or temporary failure (None).
30 def run_loop(self, num_retries, *results, **kwargs):
31 responses = itertools.chain(results, itertools.repeat(None))
32 retrier = arv_retry.RetryLoop(num_retries, self.loop_success,
34 for tries_left, response in zip(retrier, responses):
35 retrier.save_result(response)
38 def check_result(self, retrier, expect_success, last_code):
39 self.assertIs(retrier.success(), expect_success,
40 "loop success flag is incorrect")
41 self.assertEqual(last_code, retrier.last_result())
44 class RetryLoopTestCase(unittest.TestCase, RetryLoopTestMixin):
45 def test_zero_retries_and_success(self):
46 retrier = self.run_loop(0, 200)
47 self.check_result(retrier, True, 200)
49 def test_zero_retries_and_tempfail(self):
50 retrier = self.run_loop(0, 500, 501)
51 self.check_result(retrier, None, 500)
53 def test_zero_retries_and_permfail(self):
54 retrier = self.run_loop(0, 400, 201)
55 self.check_result(retrier, False, 400)
57 def test_one_retry_with_immediate_success(self):
58 retrier = self.run_loop(1, 200, 201)
59 self.check_result(retrier, True, 200)
61 def test_one_retry_with_delayed_success(self):
62 retrier = self.run_loop(1, 500, 201)
63 self.check_result(retrier, True, 201)
65 def test_one_retry_with_no_success(self):
66 retrier = self.run_loop(1, 500, 501, 502)
67 self.check_result(retrier, None, 501)
69 def test_one_retry_but_permfail(self):
70 retrier = self.run_loop(1, 400, 201)
71 self.check_result(retrier, False, 400)
73 def test_two_retries_with_immediate_success(self):
74 retrier = self.run_loop(2, 200, 201, 202)
75 self.check_result(retrier, True, 200)
77 def test_two_retries_with_success_after_one(self):
78 retrier = self.run_loop(2, 500, 201, 502)
79 self.check_result(retrier, True, 201)
81 def test_two_retries_with_success_after_two(self):
82 retrier = self.run_loop(2, 500, 501, 202, 503)
83 self.check_result(retrier, True, 202)
85 def test_two_retries_with_no_success(self):
86 retrier = self.run_loop(2, 500, 501, 502, 503)
87 self.check_result(retrier, None, 502)
89 def test_two_retries_with_permfail(self):
90 retrier = self.run_loop(2, 500, 401, 202)
91 self.check_result(retrier, False, 401)
93 def test_save_result_before_start_is_error(self):
94 retrier = arv_retry.RetryLoop(0)
95 self.assertRaises(arv_error.AssertionError, retrier.save_result, 1)
97 def test_save_result_after_end_is_error(self):
98 retrier = arv_retry.RetryLoop(0)
101 self.assertRaises(arv_error.AssertionError, retrier.save_result, 1)
104 @mock.patch('time.time', side_effect=itertools.count())
105 @mock.patch('time.sleep')
106 class RetryLoopBackoffTestCase(unittest.TestCase, RetryLoopTestMixin):
107 def run_loop(self, num_retries, *results, **kwargs):
108 kwargs.setdefault('backoff_start', 8)
109 return super(RetryLoopBackoffTestCase, self).run_loop(
110 num_retries, *results, **kwargs)
112 def check_backoff(self, sleep_mock, sleep_count, multiplier=1):
113 # Figure out how much time we actually spent sleeping.
114 sleep_times = [arglist[0][0] for arglist in sleep_mock.call_args_list
115 if arglist[0][0] > 0]
116 self.assertEqual(sleep_count, len(sleep_times),
117 "loop did not back off correctly")
119 for this_wait in sleep_times:
120 self.assertGreater(this_wait, last_wait * multiplier,
121 "loop did not grow backoff times correctly")
122 last_wait = this_wait
124 def test_no_backoff_with_no_retries(self, sleep_mock, time_mock):
125 self.run_loop(0, 500, 201)
126 self.check_backoff(sleep_mock, 0)
128 def test_no_backoff_after_success(self, sleep_mock, time_mock):
129 self.run_loop(1, 200, 501)
130 self.check_backoff(sleep_mock, 0)
132 def test_no_backoff_after_permfail(self, sleep_mock, time_mock):
133 self.run_loop(1, 400, 201)
134 self.check_backoff(sleep_mock, 0)
136 def test_backoff_before_success(self, sleep_mock, time_mock):
137 self.run_loop(5, 500, 501, 502, 203, 504)
138 self.check_backoff(sleep_mock, 3)
140 def test_backoff_before_permfail(self, sleep_mock, time_mock):
141 self.run_loop(5, 500, 501, 502, 403, 504)
142 self.check_backoff(sleep_mock, 3)
144 def test_backoff_all_tempfail(self, sleep_mock, time_mock):
145 self.run_loop(3, 500, 501, 502, 503, 504)
146 self.check_backoff(sleep_mock, 3)
148 def test_backoff_multiplier(self, sleep_mock, time_mock):
149 self.run_loop(5, 500, 501, 502, 503, 504, 505,
150 backoff_start=5, backoff_growth=10, max_wait=1000000000)
151 self.check_backoff(sleep_mock, 5, 9)
154 class CheckHTTPResponseSuccessTestCase(unittest.TestCase):
155 def results_map(self, *codes):
157 yield code, arv_retry.check_http_response_success(code)
159 def check(assert_name):
160 def check_method(self, expected, *codes):
161 assert_func = getattr(self, assert_name)
162 for code, actual in self.results_map(*codes):
163 assert_func(expected, actual,
164 "{} status flagged {}".format(code, actual))
165 if assert_name != 'assertIs':
167 actual is True or actual is False or actual is None,
168 "{} status returned {}".format(code, actual))
171 check_is = check('assertIs')
172 check_is_not = check('assertIsNot')
174 def test_obvious_successes(self):
175 self.check_is(True, *list(range(200, 207)))
177 def test_obvious_stops(self):
178 self.check_is(False, 422, 424, 426, 428, 431,
179 *list(range(400, 408)) + list(range(410, 420)))
181 def test_obvious_retries(self):
182 self.check_is(None, 500, 502, 503, 504)
184 def test_4xx_retries(self):
185 self.check_is(None, 408, 409, 423)
187 def test_5xx_failures(self):
188 self.check_is(False, 501, *list(range(505, 512)))
190 def test_1xx_not_retried(self):
191 self.check_is_not(None, 100, 101)
193 def test_redirects_not_retried(self):
194 self.check_is_not(None, *list(range(300, 309)))
196 def test_wacky_code_retries(self):
197 self.check_is(None, 0, 99, 600, -200)
200 class RetryMethodTestCase(unittest.TestCase):
201 class Tester(object):
205 @arv_retry.retry_method
206 def check(self, a, num_retries=None, z=0):
207 return (a, num_retries, z)
210 def test_positional_arg_raises(self):
211 # unsupported use -- make sure we raise rather than ignore
212 with self.assertRaises(TypeError):
213 self.assertEqual((3, 2, 0), self.Tester().check(3, 2))
215 def test_keyword_arg_passed(self):
216 self.assertEqual((4, 3, 0), self.Tester().check(num_retries=3, a=4))
218 def test_not_specified(self):
219 self.assertEqual((0, 1, 0), self.Tester().check(0))
221 def test_not_specified_with_other_kwargs(self):
222 self.assertEqual((1, 1, 1), self.Tester().check(1, z=1))
224 def test_bad_call(self):
225 with self.assertRaises(TypeError):
226 self.Tester().check(num_retries=2)
229 if __name__ == '__main__':