X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/1128f6e0d62f71f4ee91ab609c918ae5bb291edd..788b8d7247da8c4592b1f9d482fff4e1509f57f3:/services/nodemanager/tests/testutil.py diff --git a/services/nodemanager/tests/testutil.py b/services/nodemanager/tests/testutil.py index f254e7280f..a17f0882e6 100644 --- a/services/nodemanager/tests/testutil.py +++ b/services/nodemanager/tests/testutil.py @@ -2,9 +2,11 @@ from __future__ import absolute_import, print_function +import datetime import threading import time +import libcloud.common.types as cloud_types import mock import pykka @@ -12,34 +14,43 @@ from . import pykka_timeout no_sleep = mock.patch('time.sleep', lambda n: None) -def arvados_node_mock(node_num=99, job_uuid=None, age=0, **kwargs): +def arvados_node_mock(node_num=99, job_uuid=None, age=-1, **kwargs): + mod_time = datetime.datetime.utcnow() - datetime.timedelta(seconds=age) + mod_time_s = mod_time.strftime('%Y-%m-%dT%H:%M:%S.%fZ') if job_uuid is True: job_uuid = 'zzzzz-jjjjj-jobjobjobjobjob' crunch_worker_state = 'idle' if (job_uuid is None) else 'busy' node = {'uuid': 'zzzzz-yyyyy-{:015x}'.format(node_num), - 'created_at': '2014-01-01T01:02:03Z', - 'modified_at': time.strftime('%Y-%m-%dT%H:%M:%SZ', - time.gmtime(time.time() - age)), + 'created_at': '2014-01-01T01:02:03.04050607Z', + 'modified_at': mod_time_s, + 'first_ping_at': kwargs.pop('first_ping_at', mod_time_s), + 'last_ping_at': mod_time_s, 'slot_number': node_num, 'hostname': 'compute{}'.format(node_num), 'domain': 'zzzzz.arvadosapi.com', 'ip_address': ip_address_mock(node_num), 'job_uuid': job_uuid, 'crunch_worker_state': crunch_worker_state, - 'info': {}} + 'properties': {}, + 'info': {'ping_secret': 'defaulttestsecret', 'ec2_instance_id': str(node_num)}} node.update(kwargs) return node -def cloud_node_mock(node_num=99): - node = mock.NonCallableMagicMock( - ['id', 'name', 'state', 'public_ips', 'private_ips', 'driver', 'size', - 'image', 'extra'], - name='cloud_node') - node.id = str(node_num) - node.name = node.id - node.public_ips = [] - node.private_ips = [ip_address_mock(node_num)] - return node +def cloud_object_mock(name_id, **extra): + # A very generic mock, useful for stubbing libcloud objects we + # only search for and pass around, like locations, subnets, etc. + cloud_object = mock.NonCallableMagicMock(['id', 'name'], + name='cloud_object') + cloud_object.name = str(name_id) + cloud_object.id = 'id_' + cloud_object.name + cloud_object.extra = extra + return cloud_object + + +def cloud_node_fqdn(node): + # We intentionally put the FQDN somewhere goofy to make sure tested code is + # using this function for lookups. + return node.extra.get('testname', 'NoTestName') def ip_address_mock(last_octet): return '10.20.30.{}'.format(last_octet) @@ -75,7 +86,10 @@ class MockTimer(object): to_deliver = self.messages self.messages = [] for callback, args, kwargs in to_deliver: - callback(*args, **kwargs) + try: + callback(*args, **kwargs) + except pykka.ActorDeadError: + pass def schedule(self, want_time, callback, *args, **kwargs): with self.lock: @@ -106,6 +120,67 @@ class ActorTestMixin(object): return result +class DriverTestMixin(object): + def setUp(self): + self.driver_mock = mock.MagicMock(name='driver_mock') + super(DriverTestMixin, self).setUp() + + def new_driver(self, auth_kwargs={}, list_kwargs={}, create_kwargs={}): + create_kwargs.setdefault('ping_host', '100::') + return self.TEST_CLASS( + auth_kwargs, list_kwargs, create_kwargs, + driver_class=self.driver_mock) + + def driver_method_args(self, method_name): + return getattr(self.driver_mock(), method_name).call_args + + def test_driver_create_retry(self): + with mock.patch('time.sleep'): + driver_mock2 = mock.MagicMock(name='driver_mock2') + self.driver_mock.side_effect = (Exception("oops"), driver_mock2) + kwargs = {'user_id': 'foo'} + driver = self.new_driver(auth_kwargs=kwargs) + self.assertTrue(self.driver_mock.called) + self.assertIs(driver.real, driver_mock2) + + def test_create_can_find_node_after_timeout(self, create_kwargs={}, node_extra={}): + driver = self.new_driver(create_kwargs=create_kwargs) + arv_node = arvados_node_mock() + cloud_node = cloud_node_mock(**node_extra) + cloud_node.name = driver.create_cloud_name(arv_node) + create_method = self.driver_mock().create_node + create_method.side_effect = cloud_types.LibcloudError("fake timeout") + list_method = self.driver_mock().list_nodes + list_method.return_value = [cloud_node] + actual = driver.create_node(MockSize(1), arv_node) + self.assertIs(cloud_node, actual) + + def test_create_can_raise_exception_after_timeout(self): + driver = self.new_driver() + arv_node = arvados_node_mock() + create_method = self.driver_mock().create_node + create_method.side_effect = cloud_types.LibcloudError("fake timeout") + list_method = self.driver_mock().list_nodes + list_method.return_value = [] + with self.assertRaises(cloud_types.LibcloudError) as exc_test: + driver.create_node(MockSize(1), arv_node) + self.assertIs(create_method.side_effect, exc_test.exception) + + def check_node_found_after_timeout_has_fixed_size(self, size, cloud_node, + create_kwargs={}): + # This method needs to be called explicitly by driver test suites + # that need it. + self.driver_mock().list_sizes.return_value = [size] + driver = self.new_driver(create_kwargs=create_kwargs) + arv_node = arvados_node_mock() + cloud_node.name = driver.create_cloud_name(arv_node) + create_method = self.driver_mock().create_node + create_method.side_effect = cloud_types.LibcloudError("fake timeout") + self.driver_mock().list_nodes.return_value = [cloud_node] + actual = driver.create_node(size, arv_node) + self.assertIs(size, actual.size) + + class RemotePollLoopActorTestMixin(ActorTestMixin): def build_monitor(self, *args, **kwargs): self.timer = mock.MagicMock(name='timer_mock') @@ -113,3 +188,16 @@ class RemotePollLoopActorTestMixin(ActorTestMixin): self.subscriber = mock.Mock(name='subscriber_mock') self.monitor = self.TEST_CLASS.start( self.client, self.timer, *args, **kwargs).proxy() + +def cloud_node_mock(node_num=99, size=MockSize(1), **extra): + node = mock.NonCallableMagicMock( + ['id', 'name', 'state', 'public_ips', 'private_ips', 'driver', 'size', + 'image', 'extra'], + name='cloud_node') + node.id = str(node_num) + node.name = node.id + node.size = size + node.public_ips = [] + node.private_ips = [ip_address_mock(node_num)] + node.extra = extra + return node