X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/85c71a36173550f14fb1d5f4092f2050ec8dc033..a95f899d7ac84f29b3d019aa410d265bb40833e5:/services/nodemanager/tests/testutil.py

diff --git a/services/nodemanager/tests/testutil.py b/services/nodemanager/tests/testutil.py
index b266ed62da..1b6aab3caf 100644
--- a/services/nodemanager/tests/testutil.py
+++ b/services/nodemanager/tests/testutil.py
@@ -6,6 +6,7 @@ import datetime
 import threading
 import time
 
+import libcloud.common.types as cloud_types
 import mock
 import pykka
 
@@ -31,7 +32,7 @@ def arvados_node_mock(node_num=99, job_uuid=None, age=-1, **kwargs):
             'job_uuid': job_uuid,
             'crunch_worker_state': crunch_worker_state,
             'properties': {},
-            'info': {'ping_secret': 'defaulttestsecret'}}
+            'info': {'ping_secret': 'defaulttestsecret', 'ec2_instance_id': str(node_num)}}
     node.update(kwargs)
     return node
 
@@ -85,7 +86,10 @@ class MockTimer(object):
             to_deliver = self.messages
             self.messages = []
         for callback, args, kwargs in to_deliver:
-            callback(*args, **kwargs)
+            try:
+                callback(*args, **kwargs)
+            except pykka.ActorDeadError:
+                pass
 
     def schedule(self, want_time, callback, *args, **kwargs):
         with self.lock:
@@ -115,6 +119,13 @@ class ActorTestMixin(object):
             if result is not unassigned:
                 return result
 
+    def busywait(self, f):
+        n = 0
+        while not f() and n < 10:
+            time.sleep(.1)
+            n += 1
+        self.assertTrue(f())
+
 
 class DriverTestMixin(object):
     def setUp(self):
@@ -131,12 +142,51 @@ class DriverTestMixin(object):
         return getattr(self.driver_mock(), method_name).call_args
 
     def test_driver_create_retry(self):
-        driver_mock2 = mock.MagicMock(name='driver_mock2')
-        self.driver_mock.side_effect = (Exception("oops"), driver_mock2)
-        kwargs = {'user_id': 'foo'}
-        driver = self.new_driver(auth_kwargs=kwargs)
-        self.assertTrue(self.driver_mock.called)
-        self.assertIs(driver.real, driver_mock2)
+        with mock.patch('time.sleep'):
+            driver_mock2 = mock.MagicMock(name='driver_mock2')
+            self.driver_mock.side_effect = (Exception("oops"), driver_mock2)
+            kwargs = {'user_id': 'foo'}
+            driver = self.new_driver(auth_kwargs=kwargs)
+            self.assertTrue(self.driver_mock.called)
+            self.assertIs(driver.real, driver_mock2)
+
+    def test_create_can_find_node_after_timeout(self, create_kwargs={}, node_extra={}):
+        driver = self.new_driver(create_kwargs=create_kwargs)
+        arv_node = arvados_node_mock()
+        cloud_node = cloud_node_mock(**node_extra)
+        cloud_node.name = driver.create_cloud_name(arv_node)
+        create_method = self.driver_mock().create_node
+        create_method.side_effect = cloud_types.LibcloudError("fake timeout")
+        list_method = self.driver_mock().list_nodes
+        list_method.return_value = [cloud_node]
+        actual = driver.create_node(MockSize(1), arv_node)
+        self.assertIs(cloud_node, actual)
+
+    def test_create_can_raise_exception_after_timeout(self):
+        driver = self.new_driver()
+        arv_node = arvados_node_mock()
+        create_method = self.driver_mock().create_node
+        create_method.side_effect = cloud_types.LibcloudError("fake timeout")
+        list_method = self.driver_mock().list_nodes
+        list_method.return_value = []
+        with self.assertRaises(cloud_types.LibcloudError) as exc_test:
+            driver.create_node(MockSize(1), arv_node)
+        self.assertIs(create_method.side_effect, exc_test.exception)
+
+    def check_node_found_after_timeout_has_fixed_size(self, size, cloud_node,
+                                                      create_kwargs={}):
+        # This method needs to be called explicitly by driver test suites
+        # that need it.
+        self.driver_mock().list_sizes.return_value = [size]
+        driver = self.new_driver(create_kwargs=create_kwargs)
+        arv_node = arvados_node_mock()
+        cloud_node.name = driver.create_cloud_name(arv_node)
+        create_method = self.driver_mock().create_node
+        create_method.side_effect = cloud_types.LibcloudError("fake timeout")
+        self.driver_mock().list_nodes.return_value = [cloud_node]
+        actual = driver.create_node(size, arv_node)
+        self.assertIs(size, actual.size)
+
 
 class RemotePollLoopActorTestMixin(ActorTestMixin):
     def build_monitor(self, *args, **kwargs):