Merge branch '8284-fix-slurm-queue-timestamp-check' closes #8284

[arvados.git] / services / nodemanager / arvnodeman / computenode / driver / __init__.py

diff --git a/services/nodemanager/arvnodeman/computenode/driver/__init__.py b/services/nodemanager/arvnodeman/computenode/driver/__init__.py
index e3298a70fac47e420c83c505bc1dd54681c7e335..c98c95af66d89b257be6f7e79d4fd4371138281c 100644 (file)
--- a/services/nodemanager/arvnodeman/computenode/driver/__init__.py
+++ b/services/nodemanager/arvnodeman/computenode/driver/__init__.py
@@ -2,14 +2,16 @@
 
 from __future__ import absolute_import, print_function
 
+import logging
 from operator import attrgetter
 
 import libcloud.common.types as cloud_types
-from libcloud.compute.base import NodeDriver
+from libcloud.compute.base import NodeDriver, NodeAuthSSHKey
 
 from ...config import NETWORK_ERRORS
+from .. import RetryMixin
 
-class BaseComputeNodeDriver(object):
+class BaseComputeNodeDriver(RetryMixin):
     """Abstract base class for compute node drivers.
 
     libcloud drivers abstract away many of the differences between
@@ -24,7 +26,16 @@ class BaseComputeNodeDriver(object):
     """
     CLOUD_ERRORS = NETWORK_ERRORS + (cloud_types.LibcloudError,)
 
-    def __init__(self, auth_kwargs, list_kwargs, create_kwargs, driver_class):
+    @RetryMixin._retry()
+    def _create_driver(self, driver_class, **auth_kwargs):
+        return driver_class(**auth_kwargs)
+
+    @RetryMixin._retry()
+    def _set_sizes(self):
+        self.sizes = {sz.id: sz for sz in self.real.list_sizes()}
+
+    def __init__(self, auth_kwargs, list_kwargs, create_kwargs,
+                 driver_class, retry_wait=1, max_retry_wait=180):
         """Base initializer for compute node drivers.
 
         Arguments:
@@ -37,7 +48,12 @@ class BaseComputeNodeDriver(object):
           libcloud driver's create_node method to create a new compute node.
         * driver_class: The class of a libcloud driver to use.
         """
-        self.real = driver_class(**auth_kwargs)
+
+        super(BaseComputeNodeDriver, self).__init__(retry_wait, max_retry_wait,
+                                         logging.getLogger(self.__class__.__name__),
+                                         type(self),
+                                         None)
+        self.real = self._create_driver(driver_class, **auth_kwargs)
         self.list_kwargs = list_kwargs
         self.create_kwargs = create_kwargs
         # Transform entries in create_kwargs.  For each key K, if this class
@@ -53,12 +69,14 @@ class BaseComputeNodeDriver(object):
                 if new_pair is not None:
                     self.create_kwargs[new_pair[0]] = new_pair[1]
 
+        self._set_sizes()
+
     def _init_ping_host(self, ping_host):
         self.ping_host = ping_host
 
     def _init_ssh_key(self, filename):
         with open(filename) as ssh_file:
-            key = cloud_base.NodeAuthSSHKey(ssh_file.read())
+            key = NodeAuthSSHKey(ssh_file.read())
         return 'auth', key
 
     def search_for(self, term, list_method, key=attrgetter('id'), **kwargs):
@@ -89,7 +107,7 @@ class BaseComputeNodeDriver(object):
     def list_nodes(self):
         return self.real.list_nodes(**self.list_kwargs)
 
-    def arvados_create_kwargs(self, arvados_node):
+    def arvados_create_kwargs(self, size, arvados_node):
         """Return dynamic keyword arguments for create_node.
 
         Subclasses must override this method.  It should return a dictionary
@@ -98,21 +116,50 @@ class BaseComputeNodeDriver(object):
         create_kwargs.
 
         Arguments:
+        * size: The node size that will be created (libcloud NodeSize object)
         * arvados_node: The Arvados node record that will be associated
           with this cloud node, as returned from the API server.
         """
         raise NotImplementedError("BaseComputeNodeDriver.arvados_create_kwargs")
 
+    def broken(self, cloud_node):
+        """Return true if libcloud has indicated the node is in a "broken" state."""
+        return False
+
     def _make_ping_url(self, arvados_node):
         return 'https://{}/arvados/v1/nodes/{}/ping?ping_secret={}'.format(
             self.ping_host, arvados_node['uuid'],
             arvados_node['info']['ping_secret'])
 
+    def find_node(self, name):
+        node = [n for n in self.list_nodes() if n.name == name]
+        if node:
+            return node[0]
+        else:
+            return None
+
     def create_node(self, size, arvados_node):
-        kwargs = self.create_kwargs.copy()
-        kwargs.update(self.arvados_create_kwargs(arvados_node))
-        kwargs['size'] = size
-        return self.real.create_node(**kwargs)
+        try:
+            kwargs = self.create_kwargs.copy()
+            kwargs.update(self.arvados_create_kwargs(size, arvados_node))
+            kwargs['size'] = size
+            return self.real.create_node(**kwargs)
+        except self.CLOUD_ERRORS:
+            # Workaround for bug #6702: sometimes the create node request
+            # succeeds but times out and raises an exception instead of
+            # returning a result.  If this happens, we get stuck in a retry
+            # loop forever because subsequent create_node attempts will fail
+            # due to node name collision.  So check if the node we intended to
+            # create shows up in the cloud node list and return it if found.
+            try:
+                node = self.find_node(kwargs['name'])
+                if node:
+                    return node
+            except:
+                # Ignore possible exception from find_node in favor of
+                # re-raising the original create_node exception.
+                pass
+            raise
 
     def post_create_node(self, cloud_node):
         # ComputeNodeSetupActor calls this method after the cloud node is