X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/bf3f076d4e1559937dfc4ac27747612df0f8761b..3fa4a2b6138e3e9e468dd885a743ca38f08f0755:/services/nodemanager/tests/test_computenode_dispatch.py

diff --git a/services/nodemanager/tests/test_computenode_dispatch.py b/services/nodemanager/tests/test_computenode_dispatch.py
index 227b5e5f34..c718dad6b6 100644
--- a/services/nodemanager/tests/test_computenode_dispatch.py
+++ b/services/nodemanager/tests/test_computenode_dispatch.py
@@ -11,7 +11,10 @@ import mock
 import pykka
 import threading
 
+from libcloud.common.exceptions import BaseHTTPError
+
 import arvnodeman.computenode.dispatch as dispatch
+from arvnodeman.computenode.driver import BaseComputeNodeDriver
 from . import testutil
 
 class ComputeNodeSetupActorTestCase(testutil.ActorTestMixin, unittest.TestCase):
@@ -25,6 +28,7 @@ class ComputeNodeSetupActorTestCase(testutil.ActorTestMixin, unittest.TestCase):
         self.api_client.nodes().update().execute.side_effect = arvados_effect
         self.cloud_client = mock.MagicMock(name='cloud_client')
         self.cloud_client.create_node.return_value = testutil.cloud_node_mock(1)
+        self.cloud_client.is_cloud_exception = BaseComputeNodeDriver.is_cloud_exception
 
     def make_actor(self, arv_node=None):
         if not hasattr(self, 'timer'):
@@ -86,6 +90,28 @@ class ComputeNodeSetupActorTestCase(testutil.ActorTestMixin, unittest.TestCase):
         self.make_actor()
         self.wait_for_assignment(self.setup_actor, 'cloud_node')
 
+    def test_unknown_basehttperror_not_retried(self):
+        self.make_mocks()
+        self.cloud_client.create_node.side_effect = [
+            BaseHTTPError(400, "Unknown"),
+            self.cloud_client.create_node.return_value,
+            ]
+        self.make_actor()
+        finished = threading.Event()
+        self.setup_actor.subscribe(lambda _: finished.set())
+        assert(finished.wait(self.TIMEOUT))
+        self.assertEqual(0, self.cloud_client.post_create_node.call_count)
+
+    def test_known_basehttperror_retried(self):
+        self.make_mocks()
+        self.cloud_client.create_node.side_effect = [
+            BaseHTTPError(400, "InstanceLimitExceeded"),
+            self.cloud_client.create_node.return_value,
+            ]
+        self.make_actor()
+        self.wait_for_assignment(self.setup_actor, 'cloud_node')
+        self.assertEqual(1, self.cloud_client.post_create_node.call_count)
+
     def test_failed_post_create_retried(self):
         self.make_mocks()
         self.cloud_client.post_create_node.side_effect = [
@@ -175,14 +201,19 @@ class ComputeNodeShutdownActorMixin(testutil.ActorTestMixin):
         else:
             self.fail("success flag {} is not {}".format(last_flag, expected))
 
+    def test_cancellable_shutdown(self, *mocks):
+        self.make_mocks(shutdown_open=True, arvados_node=testutil.arvados_node_mock(crunch_worker_state="busy"))
+        self.cloud_client.destroy_node.return_value = True
+        self.make_actor(cancellable=True)
+        self.check_success_flag(False)
+        self.assertFalse(self.cloud_client.destroy_node.called)
+
     def test_uncancellable_shutdown(self, *mocks):
-        self.make_mocks(shutdown_open=False)
-        self.cloud_client.destroy_node.return_value = False
-        self.make_actor(cancellable=False)
-        self.check_success_flag(None, 0)
-        self.shutdowns._set_state(True, 600)
+        self.make_mocks(shutdown_open=True, arvados_node=testutil.arvados_node_mock(crunch_worker_state="busy"))
         self.cloud_client.destroy_node.return_value = True
-        self.check_success_flag(True)
+        self.make_actor(cancellable=False)
+        self.check_success_flag(True, 2)
+        self.assertTrue(self.cloud_client.destroy_node.called)
 
     def test_arvados_node_cleaned_after_shutdown(self, *mocks):
         cloud_node = testutil.cloud_node_mock(62)
@@ -221,21 +252,13 @@ class ComputeNodeShutdownActorTestCase(ComputeNodeShutdownActorMixin,
         self.check_success_flag(True)
         self.assertTrue(self.cloud_client.destroy_node.called)
 
-    def test_shutdown_retries_when_cloud_fails(self):
-        self.make_mocks()
-        self.cloud_client.destroy_node.return_value = False
-        self.make_actor(start_time=0)
-        self.assertIsNone(self.shutdown_actor.success.get(self.TIMEOUT))
-        self.cloud_client.destroy_node.return_value = True
-        self.check_success_flag(True)
-
-    def test_shutdown_cancelled_when_cloud_fails_on_broken_node(self):
+    def test_shutdown_cancelled_when_destroy_node_fails(self):
         self.make_mocks(node_broken=True)
         self.cloud_client.destroy_node.return_value = False
         self.make_actor(start_time=0)
         self.check_success_flag(False, 2)
         self.assertEqual(1, self.cloud_client.destroy_node.call_count)
-        self.assertEqual(self.ACTOR_CLASS.NODE_BROKEN,
+        self.assertEqual(self.ACTOR_CLASS.DESTROY_FAILED,
                          self.shutdown_actor.cancel_reason.get(self.TIMEOUT))
 
     def test_late_subscribe(self):
@@ -250,11 +273,13 @@ class ComputeNodeShutdownActorTestCase(ComputeNodeShutdownActorMixin,
 
 class ComputeNodeUpdateActorTestCase(testutil.ActorTestMixin,
                                      unittest.TestCase):
+    ACTOR_CLASS = dispatch.ComputeNodeUpdateActor
+
     def make_actor(self):
         self.driver = mock.MagicMock(name='driver_mock')
-        self.updater = dispatch.ComputeNodeUpdateActor.start(self.driver).proxy()
+        self.updater = self.ACTOR_CLASS.start(self.driver).proxy()
 
-    def test_node_sync(self):
+    def test_node_sync(self, *args):
         self.make_actor()
         cloud_node = testutil.cloud_node_mock()
         arv_node = testutil.arvados_node_mock()
@@ -262,7 +287,7 @@ class ComputeNodeUpdateActorTestCase(testutil.ActorTestMixin,
         self.driver().sync_node.assert_called_with(cloud_node, arv_node)
 
     @testutil.no_sleep
-    def test_node_sync_error(self):
+    def test_node_sync_error(self, *args):
         self.make_actor()
         cloud_node = testutil.cloud_node_mock()
         arv_node = testutil.arvados_node_mock()