X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/a6837612f9678bb983f634b518bde16b8921a0a4..26751323e77005dc158b64e86c47bbb9459e6697:/services/nodemanager/tests/test_computenode_driver_gce.py diff --git a/services/nodemanager/tests/test_computenode_driver_gce.py b/services/nodemanager/tests/test_computenode_driver_gce.py index 0e10e2f84e..84e061d867 100644 --- a/services/nodemanager/tests/test_computenode_driver_gce.py +++ b/services/nodemanager/tests/test_computenode_driver_gce.py @@ -2,6 +2,7 @@ from __future__ import absolute_import, print_function +import json import time import unittest @@ -10,15 +11,22 @@ import mock import arvnodeman.computenode.driver.gce as gce from . import testutil -class GCEComputeNodeDriverTestCase(unittest.TestCase): +class GCEComputeNodeDriverTestCase(testutil.DriverTestMixin, unittest.TestCase): + TEST_CLASS = gce.ComputeNodeDriver + def setUp(self): - self.driver_mock = mock.MagicMock(name='driver_mock') + super(GCEComputeNodeDriverTestCase, self).setUp() + self.driver_mock().list_images.return_value = [ + testutil.cloud_object_mock('testimage', selfLink='image-link')] + self.driver_mock().ex_list_disktypes.return_value = [ + testutil.cloud_object_mock(name, selfLink=name + '-link') + for name in ['pd-standard', 'pd-ssd', 'local-ssd']] + self.driver_mock.reset_mock() def new_driver(self, auth_kwargs={}, list_kwargs={}, create_kwargs={}): - create_kwargs.setdefault('ping_host', '100::') - return gce.ComputeNodeDriver( - auth_kwargs, list_kwargs, create_kwargs, - driver_class=self.driver_mock) + create_kwargs.setdefault('image', 'testimage') + return super(GCEComputeNodeDriverTestCase, self).new_driver( + auth_kwargs, list_kwargs, create_kwargs) def test_driver_instantiation(self): kwargs = {'user_id': 'foo'} @@ -26,80 +34,208 @@ class GCEComputeNodeDriverTestCase(unittest.TestCase): self.assertTrue(self.driver_mock.called) self.assertEqual(kwargs, self.driver_mock.call_args[1]) - def test_create_location_loaded_at_initialization(self): - kwargs = {'location': 'testregion'} - driver = self.new_driver(create_kwargs=kwargs) - self.assertTrue(self.driver_mock().list_locations) - - def test_create_image_loaded_at_initialization(self): - kwargs = {'image': 'testimage'} - driver = self.new_driver(create_kwargs=kwargs) - self.assertTrue(self.driver_mock().list_images) + def test_create_image_loaded_at_initialization_by_name(self): + image_mocks = [testutil.cloud_object_mock(c) for c in 'abc'] + list_method = self.driver_mock().list_images + list_method.return_value = image_mocks + driver = self.new_driver(create_kwargs={'image': 'b'}) + self.assertEqual(1, list_method.call_count) def test_create_includes_ping_secret(self): arv_node = testutil.arvados_node_mock(info={'ping_secret': 'ssshh'}) driver = self.new_driver() driver.create_node(testutil.MockSize(1), arv_node) - create_method = self.driver_mock().create_node - self.assertTrue(create_method.called) - create_metadata = create_method.call_args[1].get('ex_metadata') - self.assertIsInstance(create_metadata, dict) - self.assertIn('ping_secret=ssshh', - create_metadata.get('pingUrl', 'arg missing')) - - def test_generate_metadata_for_new_arvados_node(self): - arv_node = testutil.arvados_node_mock(8) - driver = self.new_driver(list_kwargs={'list': 'test'}) - self.assertEqual({'ex_metadata': {'list': 'test'}}, - driver.arvados_create_kwargs(arv_node)) + metadata = self.driver_mock().create_node.call_args[1]['ex_metadata'] + self.assertIn('ping_secret=ssshh', metadata.get('arv-ping-url')) - def test_tags_set_default_hostname_from_new_arvados_node(self): - arv_node = testutil.arvados_node_mock(hostname=None) - cloud_node = testutil.cloud_node_mock(1) + def test_create_raises_but_actually_succeeded(self): + arv_node = testutil.arvados_node_mock(1, hostname=None) driver = self.new_driver() - driver.sync_node(cloud_node, arv_node) - tag_mock = self.driver_mock().ex_set_node_tags - self.assertTrue(tag_mock.called) - self.assertEqual(['hostname-dynamic.compute.zzzzz.arvadosapi.com'], - tag_mock.call_args[0][1]) - - def test_sync_node_sets_static_hostname(self): + nodelist = [testutil.cloud_node_mock(1)] + nodelist[0].name = 'compute-000000000000001-zzzzz' + self.driver_mock().list_nodes.return_value = nodelist + self.driver_mock().create_node.side_effect = IOError + n = driver.create_node(testutil.MockSize(1), arv_node) + self.assertEqual('compute-000000000000001-zzzzz', n.name) + + def test_create_sets_default_hostname(self): + driver = self.new_driver() + driver.create_node(testutil.MockSize(1), + testutil.arvados_node_mock(254, hostname=None)) + create_kwargs = self.driver_mock().create_node.call_args[1] + self.assertEqual('compute-0000000000000fe-zzzzz', + create_kwargs.get('name')) + self.assertEqual('dynamic.compute.zzzzz.arvadosapi.com', + create_kwargs.get('ex_metadata', {}).get('hostname')) + + def test_create_tags_from_list_tags(self): + driver = self.new_driver(list_kwargs={'tags': 'testA, testB'}) + driver.create_node(testutil.MockSize(1), testutil.arvados_node_mock()) + self.assertEqual(['testA', 'testB'], + self.driver_mock().create_node.call_args[1]['ex_tags']) + + def test_create_with_two_disks_attached(self): + driver = self.new_driver(create_kwargs={'image': 'testimage'}) + driver.create_node(testutil.MockSize(1), testutil.arvados_node_mock()) + create_disks = self.driver_mock().create_node.call_args[1].get( + 'ex_disks_gce_struct', []) + self.assertEqual(2, len(create_disks)) + self.assertTrue(create_disks[0].get('autoDelete')) + self.assertTrue(create_disks[0].get('boot')) + self.assertEqual('PERSISTENT', create_disks[0].get('type')) + init_params = create_disks[0].get('initializeParams', {}) + self.assertEqual('pd-standard-link', init_params.get('diskType')) + self.assertEqual('image-link', init_params.get('sourceImage')) + # Our node images expect the SSD to be named `tmp` to find and mount it. + self.assertEqual('tmp', create_disks[1].get('deviceName')) + self.assertTrue(create_disks[1].get('autoDelete')) + self.assertFalse(create_disks[1].get('boot', 'unset')) + self.assertEqual('SCRATCH', create_disks[1].get('type')) + init_params = create_disks[1].get('initializeParams', {}) + self.assertEqual('local-ssd-link', init_params.get('diskType')) + + def test_list_nodes_requires_tags_match(self): + # A node matches if our list tags are a subset of the node's tags. + # Test behavior with no tags, no match, partial matches, different + # order, and strict supersets. + cloud_mocks = [ + testutil.cloud_node_mock(node_num, tags=tag_set) + for node_num, tag_set in enumerate( + [[], ['bad'], ['good'], ['great'], ['great', 'ok'], + ['great', 'good'], ['good', 'fantastic', 'great']])] + cloud_mocks.append(testutil.cloud_node_mock()) + self.driver_mock().list_nodes.return_value = cloud_mocks + driver = self.new_driver(list_kwargs={'tags': 'good, great'}) + self.assertItemsEqual(['5', '6'], [n.id for n in driver.list_nodes()]) + + def build_gce_metadata(self, metadata_dict): + # Convert a plain metadata dictionary to the GCE data structure. + return { + 'kind': 'compute#metadata', + 'fingerprint': 'testprint', + 'items': [{'key': key, 'value': metadata_dict[key]} + for key in metadata_dict], + } + + def check_sync_node_updates_hostname_tag(self, plain_metadata): + start_metadata = self.build_gce_metadata(plain_metadata) arv_node = testutil.arvados_node_mock(1) - cloud_node = testutil.cloud_node_mock(2) + cloud_node = testutil.cloud_node_mock( + 2, metadata=start_metadata.copy(), + zone=testutil.cloud_object_mock('testzone')) driver = self.new_driver() driver.sync_node(cloud_node, arv_node) - tag_mock = self.driver_mock().ex_set_node_tags - self.assertTrue(tag_mock.called) - self.assertEqual(['hostname-compute1.zzzzz.arvadosapi.com'], - tag_mock.call_args[0][1]) - - def test_node_create_time(self): - refsecs = int(time.time()) - reftuple = time.gmtime(refsecs) + args, kwargs = self.driver_mock().connection.async_request.call_args + self.assertEqual('/zones/testzone/instances/2/setMetadata', args[0]) + for key in ['kind', 'fingerprint']: + self.assertEqual(start_metadata[key], kwargs['data'][key]) + plain_metadata['hostname'] = 'compute1.zzzzz.arvadosapi.com' + self.assertEqual( + plain_metadata, + {item['key']: item['value'] for item in kwargs['data']['items']}) + + def test_sync_node_updates_hostname_tag(self): + self.check_sync_node_updates_hostname_tag( + {'testkey': 'testvalue', 'hostname': 'startvalue'}) + + def test_sync_node_adds_hostname_tag(self): + self.check_sync_node_updates_hostname_tag({'testkey': 'testval'}) + + def test_sync_node_raises_exception_on_failure(self): + arv_node = testutil.arvados_node_mock(8) + cloud_node = testutil.cloud_node_mock( + 9, metadata={}, zone=testutil.cloud_object_mock('failzone')) + mock_response = self.driver_mock().connection.async_request() + mock_response.success.return_value = False + mock_response.error = 'sync error test' + driver = self.new_driver() + with self.assertRaises(Exception) as err_check: + driver.sync_node(cloud_node, arv_node) + self.assertIs(err_check.exception.__class__, Exception) + self.assertIn('sync error test', str(err_check.exception)) + + def test_node_create_time_zero_for_unknown_nodes(self): node = testutil.cloud_node_mock() - node.extra = {'launch_time': time.strftime('%Y-%m-%dT%H:%M:%S.000Z', - reftuple)} - self.assertEqual(refsecs, gce.ComputeNodeDriver.node_start_time(node)) + self.assertEqual(0, gce.ComputeNodeDriver.node_start_time(node)) - def test_generate_metadata_for_new_arvados_node(self): - arv_node = testutil.arvados_node_mock(8) - driver = self.new_driver(list_kwargs={'list': 'test'}) - self.assertEqual({'ex_metadata': {'list': 'test'}}, - driver.arvados_create_kwargs(arv_node)) + def test_node_create_time_for_known_node(self): + node = testutil.cloud_node_mock(metadata=self.build_gce_metadata( + {'booted_at': '1970-01-01T00:01:05Z'})) + self.assertEqual(65, gce.ComputeNodeDriver.node_start_time(node)) + + def test_node_create_time_recorded_when_node_boots(self): + start_time = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()) + arv_node = testutil.arvados_node_mock() + driver = self.new_driver() + driver.create_node(testutil.MockSize(1), arv_node) + metadata = self.driver_mock().create_node.call_args[1]['ex_metadata'] + self.assertLessEqual(start_time, metadata.get('booted_at')) + + def test_known_node_fqdn(self): + name = 'fqdntest.zzzzz.arvadosapi.com' + node = testutil.cloud_node_mock(metadata=self.build_gce_metadata( + {'hostname': name})) + self.assertEqual(name, gce.ComputeNodeDriver.node_fqdn(node)) + + def test_unknown_node_fqdn(self): + # Return an empty string. This lets fqdn be safely compared + # against an expected value, and ComputeNodeMonitorActor + # should try to update it. + node = testutil.cloud_node_mock(metadata=self.build_gce_metadata({})) + self.assertEqual('', gce.ComputeNodeDriver.node_fqdn(node)) def test_deliver_ssh_key_in_metadata(self): test_ssh_key = 'ssh-rsa-foo' arv_node = testutil.arvados_node_mock(1) - with mock.patch('__builtin__.open', mock.mock_open(read_data=test_ssh_key)) as mock_file: + with mock.patch('__builtin__.open', + mock.mock_open(read_data=test_ssh_key)) as mock_file: driver = self.new_driver(create_kwargs={'ssh_key': 'ssh-key-file'}) mock_file.assert_called_once_with('ssh-key-file') - self.assertEqual({'ex_metadata': {'sshKeys': 'root:ssh-rsa-foo'}}, - driver.arvados_create_kwargs(arv_node)) + driver.create_node(testutil.MockSize(1), arv_node) + metadata = self.driver_mock().create_node.call_args[1]['ex_metadata'] + self.assertEqual('root:ssh-rsa-foo', metadata.get('sshKeys')) def test_create_driver_with_service_accounts(self): - srv_acct_config = { 'service_accounts': '{ "email": "foo@bar", "scopes":["storage-full"]}' } + service_accounts = {'email': 'foo@bar', 'scopes': ['storage-full']} + srv_acct_config = {'service_accounts': json.dumps(service_accounts)} arv_node = testutil.arvados_node_mock(1) driver = self.new_driver(create_kwargs=srv_acct_config) - create_kwargs = driver.arvados_create_kwargs(arv_node) - self.assertEqual({u'email': u'foo@bar', u'scopes': [u'storage-full']}, - create_kwargs['ex_service_accounts']) + driver.create_node(testutil.MockSize(1), arv_node) + self.assertEqual( + service_accounts, + self.driver_mock().create_node.call_args[1]['ex_service_accounts']) + + def test_fix_string_size(self): + # As of 0.18, the libcloud GCE driver sets node.size to the size's name. + # It's supposed to be the actual size object. Make sure our driver + # patches that up in listings. + size = testutil.MockSize(2) + node = testutil.cloud_node_mock(size=size) + node.size = size.name + self.driver_mock().list_sizes.return_value = [size] + self.driver_mock().list_nodes.return_value = [node] + driver = self.new_driver() + nodelist = driver.list_nodes() + self.assertEqual(1, len(nodelist)) + self.assertIs(node, nodelist[0]) + self.assertIs(size, nodelist[0].size) + + def test_skip_fix_when_size_not_string(self): + # Ensure we don't monkeypatch node sizes unless we need to. + size = testutil.MockSize(3) + node = testutil.cloud_node_mock(size=size) + self.driver_mock().list_nodes.return_value = [node] + driver = self.new_driver() + nodelist = driver.list_nodes() + self.assertEqual(1, len(nodelist)) + self.assertIs(node, nodelist[0]) + self.assertIs(size, nodelist[0].size) + + def test_node_found_after_timeout_has_fixed_size(self): + size = testutil.MockSize(4) + cloud_node = testutil.cloud_node_mock(size=size.id) + self.check_node_found_after_timeout_has_fixed_size(size, cloud_node) + + def test_list_empty_nodes(self): + self.driver_mock().list_nodes.return_value = [] + self.assertEqual([], self.new_driver().list_nodes())