X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/eae1286badb67ee63888633ff59bda9cb736131e..c3b26754a231ec909506f2ff28af1af9f2e27f2b:/services/nodemanager/arvnodeman/computenode/__init__.py diff --git a/services/nodemanager/arvnodeman/computenode/__init__.py b/services/nodemanager/arvnodeman/computenode/__init__.py index 4e46a438db..b124c66540 100644 --- a/services/nodemanager/arvnodeman/computenode/__init__.py +++ b/services/nodemanager/arvnodeman/computenode/__init__.py @@ -12,6 +12,7 @@ import re import time from ..config import CLOUD_ERRORS +from ..status import tracker from libcloud.common.exceptions import BaseHTTPError, RateLimitReachedError ARVADOS_TIMEFMT = '%Y-%m-%dT%H:%M:%SZ' @@ -32,7 +33,7 @@ def arvados_timestamp(timestr): subsecs = float(subsec_match.group(1)) timestr = timestr[:subsec_match.start()] + 'Z' return calendar.timegm(time.strptime(timestr + 'UTC', - ARVADOS_TIMEFMT + '%Z')) + ARVADOS_TIMEFMT + '%Z')) + subsecs def timestamp_fresh(timestamp, fresh_time): return (time.time() - timestamp) < fresh_time @@ -101,6 +102,7 @@ class RetryMixin(object): if error.code == 429 or error.code >= 500: should_retry = True except CLOUD_ERRORS as error: + tracker.counter_add('cloud_errors') should_retry = True except errors as error: should_retry = True @@ -108,9 +110,11 @@ class RetryMixin(object): # As a libcloud workaround for drivers that don't use # typed exceptions, consider bare Exception() objects # retryable. - should_retry = type(error) is Exception + if type(error) is Exception: + tracker.counter_add('cloud_errors') + should_retry = True else: - # No exception, + # No exception self.retry_wait = self.min_retry_wait return ret