services/nodemanager/tests/test_computenode_driver_gce.py

   1 #!/usr/bin/env python
   2
   3 from __future__ import absolute_import, print_function
   4
   5 import json
   6 import time
   7 import unittest
   8
   9 import mock
  10
  11 import arvnodeman.computenode.driver.gce as gce
  12 from . import testutil
  13
  14 class GCEComputeNodeDriverTestCase(testutil.DriverTestMixin, unittest.TestCase):
  15     TEST_CLASS = gce.ComputeNodeDriver
  16
  17     def setUp(self):
  18         super(GCEComputeNodeDriverTestCase, self).setUp()
  19         self.driver_mock().list_images.return_value = [
  20             testutil.cloud_object_mock('testimage', selfLink='image-link')]
  21         self.driver_mock().ex_list_disktypes.return_value = [
  22             testutil.cloud_object_mock(name, selfLink=name + '-link')
  23             for name in ['pd-standard', 'pd-ssd', 'local-ssd']]
  24         self.driver_mock.reset_mock()
  25
  26     def new_driver(self, auth_kwargs={}, list_kwargs={}, create_kwargs={}):
  27         create_kwargs.setdefault('image', 'testimage')
  28         return super(GCEComputeNodeDriverTestCase, self).new_driver(
  29             auth_kwargs, list_kwargs, create_kwargs)
  30
  31     def test_driver_instantiation(self):
  32         kwargs = {'user_id': 'foo'}
  33         driver = self.new_driver(auth_kwargs=kwargs)
  34         self.assertTrue(self.driver_mock.called)
  35         self.assertEqual(kwargs, self.driver_mock.call_args[1])
  36
  37     def test_create_image_loaded_at_initialization_by_name(self):
  38         image_mocks = [testutil.cloud_object_mock(c) for c in 'abc']
  39         list_method = self.driver_mock().list_images
  40         list_method.return_value = image_mocks
  41         driver = self.new_driver(create_kwargs={'image': 'b'})
  42         self.assertEqual(1, list_method.call_count)
  43
  44     def test_create_includes_ping_secret(self):
  45         arv_node = testutil.arvados_node_mock(info={'ping_secret': 'ssshh'})
  46         driver = self.new_driver()
  47         driver.create_node(testutil.MockSize(1), arv_node)
  48         metadata = self.driver_mock().create_node.call_args[1]['ex_metadata']
  49         self.assertIn('ping_secret=ssshh', metadata.get('arv-ping-url'))
  50
  51     def test_create_raises_but_actually_succeeded(self):
  52         arv_node = testutil.arvados_node_mock(1, hostname=None)
  53         driver = self.new_driver()
  54         nodelist = [testutil.cloud_node_mock(1)]
  55         nodelist[0].name = 'compute-000000000000001-zzzzz'
  56         self.driver_mock().list_nodes.return_value = nodelist
  57         self.driver_mock().create_node.side_effect = IOError
  58         n = driver.create_node(testutil.MockSize(1), arv_node)
  59         self.assertEqual('compute-000000000000001-zzzzz', n.name)
  60
  61     def test_create_sets_default_hostname(self):
  62         driver = self.new_driver()
  63         driver.create_node(testutil.MockSize(1),
  64                            testutil.arvados_node_mock(254, hostname=None))
  65         create_kwargs = self.driver_mock().create_node.call_args[1]
  66         self.assertEqual('compute-0000000000000fe-zzzzz',
  67                          create_kwargs.get('name'))
  68         self.assertEqual('dynamic.compute.zzzzz.arvadosapi.com',
  69                          create_kwargs.get('ex_metadata', {}).get('hostname'))
  70
  71     def test_create_tags_from_list_tags(self):
  72         driver = self.new_driver(list_kwargs={'tags': 'testA, testB'})
  73         driver.create_node(testutil.MockSize(1), testutil.arvados_node_mock())
  74         self.assertEqual(['testA', 'testB'],
  75                          self.driver_mock().create_node.call_args[1]['ex_tags'])
  76
  77     def test_create_with_two_disks_attached(self):
  78         driver = self.new_driver(create_kwargs={'image': 'testimage'})
  79         driver.create_node(testutil.MockSize(1), testutil.arvados_node_mock())
  80         create_disks = self.driver_mock().create_node.call_args[1].get(
  81             'ex_disks_gce_struct', [])
  82         self.assertEqual(2, len(create_disks))
  83         self.assertTrue(create_disks[0].get('autoDelete'))
  84         self.assertTrue(create_disks[0].get('boot'))
  85         self.assertEqual('PERSISTENT', create_disks[0].get('type'))
  86         init_params = create_disks[0].get('initializeParams', {})
  87         self.assertEqual('pd-standard-link', init_params.get('diskType'))
  88         self.assertEqual('image-link', init_params.get('sourceImage'))
  89         # Our node images expect the SSD to be named `tmp` to find and mount it.
  90         self.assertEqual('tmp', create_disks[1].get('deviceName'))
  91         self.assertTrue(create_disks[1].get('autoDelete'))
  92         self.assertFalse(create_disks[1].get('boot', 'unset'))
  93         self.assertEqual('SCRATCH', create_disks[1].get('type'))
  94         init_params = create_disks[1].get('initializeParams', {})
  95         self.assertEqual('local-ssd-link', init_params.get('diskType'))
  96
  97     def test_list_nodes_requires_tags_match(self):
  98         # A node matches if our list tags are a subset of the node's tags.
  99         # Test behavior with no tags, no match, partial matches, different
 100         # order, and strict supersets.
 101         cloud_mocks = [
 102             testutil.cloud_node_mock(node_num, tags=tag_set)
 103             for node_num, tag_set in enumerate(
 104                 [[], ['bad'], ['good'], ['great'], ['great', 'ok'],
 105                  ['great', 'good'], ['good', 'fantastic', 'great']])]
 106         cloud_mocks.append(testutil.cloud_node_mock())
 107         self.driver_mock().list_nodes.return_value = cloud_mocks
 108         driver = self.new_driver(list_kwargs={'tags': 'good, great'})
 109         self.assertItemsEqual(['5', '6'], [n.id for n in driver.list_nodes()])
 110
 111     def build_gce_metadata(self, metadata_dict):
 112         # Convert a plain metadata dictionary to the GCE data structure.
 113         return {
 114             'kind': 'compute#metadata',
 115             'fingerprint': 'testprint',
 116             'items': [{'key': key, 'value': metadata_dict[key]}
 117                       for key in metadata_dict],
 118             }
 119
 120     def check_sync_node_updates_hostname_tag(self, plain_metadata):
 121         start_metadata = self.build_gce_metadata(plain_metadata)
 122         arv_node = testutil.arvados_node_mock(1)
 123         cloud_node = testutil.cloud_node_mock(
 124             2, metadata=start_metadata.copy(),
 125             zone=testutil.cloud_object_mock('testzone'))
 126         driver = self.new_driver()
 127         driver.sync_node(cloud_node, arv_node)
 128         args, kwargs = self.driver_mock().connection.async_request.call_args
 129         self.assertEqual('/zones/testzone/instances/2/setMetadata', args[0])
 130         for key in ['kind', 'fingerprint']:
 131             self.assertEqual(start_metadata[key], kwargs['data'][key])
 132         plain_metadata['hostname'] = 'compute1.zzzzz.arvadosapi.com'
 133         self.assertEqual(
 134             plain_metadata,
 135             {item['key']: item['value'] for item in kwargs['data']['items']})
 136
 137     def test_sync_node_updates_hostname_tag(self):
 138         self.check_sync_node_updates_hostname_tag(
 139             {'testkey': 'testvalue', 'hostname': 'startvalue'})
 140
 141     def test_sync_node_adds_hostname_tag(self):
 142         self.check_sync_node_updates_hostname_tag({'testkey': 'testval'})
 143
 144     def test_sync_node_raises_exception_on_failure(self):
 145         arv_node = testutil.arvados_node_mock(8)
 146         cloud_node = testutil.cloud_node_mock(
 147             9, metadata={}, zone=testutil.cloud_object_mock('failzone'))
 148         mock_response = self.driver_mock().connection.async_request()
 149         mock_response.success.return_value = False
 150         mock_response.error = 'sync error test'
 151         driver = self.new_driver()
 152         with self.assertRaises(Exception) as err_check:
 153             driver.sync_node(cloud_node, arv_node)
 154         self.assertIs(err_check.exception.__class__, Exception)
 155         self.assertIn('sync error test', str(err_check.exception))
 156
 157     def test_node_create_time_zero_for_unknown_nodes(self):
 158         node = testutil.cloud_node_mock()
 159         self.assertEqual(0, gce.ComputeNodeDriver.node_start_time(node))
 160
 161     def test_node_create_time_for_known_node(self):
 162         node = testutil.cloud_node_mock(metadata=self.build_gce_metadata(
 163                 {'booted_at': '1970-01-01T00:01:05Z'}))
 164         self.assertEqual(65, gce.ComputeNodeDriver.node_start_time(node))
 165
 166     def test_node_create_time_recorded_when_node_boots(self):
 167         start_time = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
 168         arv_node = testutil.arvados_node_mock()
 169         driver = self.new_driver()
 170         driver.create_node(testutil.MockSize(1), arv_node)
 171         metadata = self.driver_mock().create_node.call_args[1]['ex_metadata']
 172         self.assertLessEqual(start_time, metadata.get('booted_at'))
 173
 174     def test_known_node_fqdn(self):
 175         name = 'fqdntest.zzzzz.arvadosapi.com'
 176         node = testutil.cloud_node_mock(metadata=self.build_gce_metadata(
 177                 {'hostname': name}))
 178         self.assertEqual(name, gce.ComputeNodeDriver.node_fqdn(node))
 179
 180     def test_unknown_node_fqdn(self):
 181         # Return an empty string.  This lets fqdn be safely compared
 182         # against an expected value, and ComputeNodeMonitorActor
 183         # should try to update it.
 184         node = testutil.cloud_node_mock(metadata=self.build_gce_metadata({}))
 185         self.assertEqual('', gce.ComputeNodeDriver.node_fqdn(node))
 186
 187     def test_deliver_ssh_key_in_metadata(self):
 188         test_ssh_key = 'ssh-rsa-foo'
 189         arv_node = testutil.arvados_node_mock(1)
 190         with mock.patch('__builtin__.open',
 191                         mock.mock_open(read_data=test_ssh_key)) as mock_file:
 192             driver = self.new_driver(create_kwargs={'ssh_key': 'ssh-key-file'})
 193         mock_file.assert_called_once_with('ssh-key-file')
 194         driver.create_node(testutil.MockSize(1), arv_node)
 195         metadata = self.driver_mock().create_node.call_args[1]['ex_metadata']
 196         self.assertEqual('root:ssh-rsa-foo', metadata.get('sshKeys'))
 197
 198     def test_create_driver_with_service_accounts(self):
 199         service_accounts = {'email': 'foo@bar', 'scopes': ['storage-full']}
 200         srv_acct_config = {'service_accounts': json.dumps(service_accounts)}
 201         arv_node = testutil.arvados_node_mock(1)
 202         driver = self.new_driver(create_kwargs=srv_acct_config)
 203         driver.create_node(testutil.MockSize(1), arv_node)
 204         self.assertEqual(
 205             service_accounts,
 206             self.driver_mock().create_node.call_args[1]['ex_service_accounts'])
 207
 208     def test_fix_string_size(self):
 209         # As of 0.18, the libcloud GCE driver sets node.size to the size's name.
 210         # It's supposed to be the actual size object.  Make sure our driver
 211         # patches that up in listings.
 212         size = testutil.MockSize(2)
 213         node = testutil.cloud_node_mock(size=size)
 214         node.size = size.name
 215         self.driver_mock().list_sizes.return_value = [size]
 216         self.driver_mock().list_nodes.return_value = [node]
 217         driver = self.new_driver()
 218         nodelist = driver.list_nodes()
 219         self.assertEqual(1, len(nodelist))
 220         self.assertIs(node, nodelist[0])
 221         self.assertIs(size, nodelist[0].size)
 222
 223     def test_skip_fix_when_size_not_string(self):
 224         # Ensure we don't monkeypatch node sizes unless we need to.
 225         size = testutil.MockSize(3)
 226         node = testutil.cloud_node_mock(size=size)
 227         self.driver_mock().list_nodes.return_value = [node]
 228         driver = self.new_driver()
 229         nodelist = driver.list_nodes()
 230         self.assertEqual(1, len(nodelist))
 231         self.assertIs(node, nodelist[0])
 232         self.assertIs(size, nodelist[0].size)
 233
 234     def test_list_empty_nodes(self):
 235         self.driver_mock().list_nodes.return_value = []
 236         self.assertEqual([], self.new_driver().list_nodes())