8816: Handle cloud errors slightly differently from unrecognized errors.

[arvados.git] / services / nodemanager / arvnodeman / computenode / dispatch / __init__.py

diff --git a/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py b/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py
index 4848289e8bfed1fbf253f7c8589a29e5c548051b..26744c42db71576cdcf9814c0478548e80c8c4b8 100644 (file)
--- a/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py
+++ b/services/nodemanager/arvnodeman/computenode/dispatch/__init__.py
@@ -256,12 +256,6 @@ class ComputeNodeUpdateActor(config.actor_class):
     this to perform maintenance tasks on themselves.  Having a
     dedicated actor for this gives us the opportunity to control the
     flow of requests; e.g., by backing off when errors occur.
-
-    This actor is most like a "traditional" Pykka actor: there's no
-    subscribing, but instead methods return real driver results.  If
-    you're interested in those results, you should get them from the
-    Future that the proxy method returns.  Be prepared to handle exceptions
-    from the cloud driver when you do.
     """
     def __init__(self, cloud_factory, max_retry_wait=180):
         super(ComputeNodeUpdateActor, self).__init__()
@@ -270,6 +264,12 @@ class ComputeNodeUpdateActor(config.actor_class):
         self.error_streak = 0
         self.next_request_time = time.time()
 
+    def _set_logger(self):
+        self._logger = logging.getLogger("%s.%s" % (self.__class__.__name__, self.actor_urn[33:]))
+
+    def on_start(self):
+        self._set_logger()
+
     def _throttle_errors(orig_func):
         @functools.wraps(orig_func)
         def throttle_wrapper(self, *args, **kwargs):
@@ -279,11 +279,17 @@ class ComputeNodeUpdateActor(config.actor_class):
             self.next_request_time = time.time()
             try:
                 result = orig_func(self, *args, **kwargs)
-            except Exception as error:
+            except self._cloud.CLOUD_ERRORS as error:
                 self.error_streak += 1
                 self.next_request_time += min(2 ** self.error_streak,
                                               self.max_retry_wait)
-                raise
+                self._logger.error(
+                    "Caught cloud error (no retry): %s",
+                    error, exc_info=error)
+            except Exception as error:
+                self._logger.error(
+                    "Caught unknown error (no retry): %s",
+                    error, exc_info=error)
             else:
                 self.error_streak = 0
                 return result