X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/1ad098e2521d57ba6d66d0a0d9dfffab76061924..2ec2c8ed2c5db174f3a83dc257fa4c4b3190f47b:/services/nodemanager/tests/test_daemon.py diff --git a/services/nodemanager/tests/test_daemon.py b/services/nodemanager/tests/test_daemon.py index c1f3b3a9b0..554fb88b47 100644 --- a/services/nodemanager/tests/test_daemon.py +++ b/services/nodemanager/tests/test_daemon.py @@ -26,6 +26,7 @@ class NodeManagerDaemonActorTestCase(testutil.ActorTestMixin, cloud_size=get_cloud_size, actor_ref=mock_actor) mock_actor.proxy.return_value = mock_proxy + mock_actor.tell_proxy.return_value = mock_proxy self.last_setup = mock_proxy return mock_actor @@ -47,7 +48,10 @@ class NodeManagerDaemonActorTestCase(testutil.ActorTestMixin, return mock_actor def make_daemon(self, cloud_nodes=[], arvados_nodes=[], want_sizes=[], - avail_sizes=[(testutil.MockSize(1), {"cores": 1})], min_nodes=0, max_nodes=8): + avail_sizes=[(testutil.MockSize(1), {"cores": 1})], + min_nodes=0, max_nodes=8, + shutdown_windows=[54, 5, 1], + max_total_price=None): for name in ['cloud_nodes', 'arvados_nodes', 'server_wishlist']: setattr(self, name + '_poller', mock.MagicMock(name=name + '_mock')) self.arv_factory = mock.MagicMock(name='arvados_mock') @@ -67,9 +71,10 @@ class NodeManagerDaemonActorTestCase(testutil.ActorTestMixin, self.server_wishlist_poller, self.arvados_nodes_poller, self.cloud_nodes_poller, self.cloud_updates, self.timer, self.arv_factory, self.cloud_factory, - [54, 5, 1], ServerCalculator(avail_sizes), + shutdown_windows, ServerCalculator(avail_sizes), min_nodes, max_nodes, 600, 1800, 3600, - self.node_setup, self.node_shutdown).proxy() + self.node_setup, self.node_shutdown, + max_total_price=max_total_price).proxy() if cloud_nodes is not None: self.daemon.update_cloud_nodes(cloud_nodes).get(self.TIMEOUT) if arvados_nodes is not None: @@ -204,6 +209,7 @@ class NodeManagerDaemonActorTestCase(testutil.ActorTestMixin, mock_shutdown = self.node_shutdown.start(node_monitor=mock_node_monitor) self.daemon.shutdowns.get()[cloud_nodes[1].id] = mock_shutdown.proxy() + self.daemon.sizes_booting_shutdown.get()[cloud_nodes[1].id] = size self.assertEqual(2, self.alive_monitor_count()) for mon_ref in self.monitor_list(): @@ -594,3 +600,120 @@ class NodeManagerDaemonActorTestCase(testutil.ActorTestMixin, self.daemon.update_cloud_nodes([]).get(self.TIMEOUT) self.stop_proxy(self.daemon) self.assertEqual(1, self.last_shutdown.stop.call_count) + + def busywait(self, f): + n = 0 + while not f() and n < 10: + time.sleep(.1) + n += 1 + self.assertTrue(f()) + + def test_node_create_two_sizes(self): + small = testutil.MockSize(1) + big = testutil.MockSize(2) + avail_sizes = [(testutil.MockSize(1), {"cores":1}), + (testutil.MockSize(2), {"cores":2})] + self.make_daemon(want_sizes=[small, small, small, big], + avail_sizes=avail_sizes, max_nodes=4) + + # the daemon runs in another thread, so we need to wait and see + # if it does all the work we're expecting it to do before stopping it. + self.busywait(lambda: self.node_setup.start.call_count == 4) + booting = self.daemon.booting.get(self.TIMEOUT) + self.stop_proxy(self.daemon) + sizecounts = {a[0].id: 0 for a in avail_sizes} + for b in booting.itervalues(): + sizecounts[b.cloud_size.get().id] += 1 + logging.info(sizecounts) + self.assertEqual(3, sizecounts[small.id]) + self.assertEqual(1, sizecounts[big.id]) + + def test_node_max_nodes_two_sizes(self): + small = testutil.MockSize(1) + big = testutil.MockSize(2) + avail_sizes = [(testutil.MockSize(1), {"cores":1}), + (testutil.MockSize(2), {"cores":2})] + self.make_daemon(want_sizes=[small, small, small, big], + avail_sizes=avail_sizes, max_nodes=3) + + # the daemon runs in another thread, so we need to wait and see + # if it does all the work we're expecting it to do before stopping it. + self.busywait(lambda: self.node_setup.start.call_count == 3) + booting = self.daemon.booting.get(self.TIMEOUT) + self.stop_proxy(self.daemon) + sizecounts = {a[0].id: 0 for a in avail_sizes} + for b in booting.itervalues(): + sizecounts[b.cloud_size.get().id] += 1 + self.assertEqual(2, sizecounts[small.id]) + self.assertEqual(1, sizecounts[big.id]) + + def test_wishlist_reconfigure(self): + small = testutil.MockSize(1) + big = testutil.MockSize(2) + avail_sizes = [(small, {"cores":1}), (big, {"cores":2})] + + self.make_daemon(cloud_nodes=[testutil.cloud_node_mock(1, small), + testutil.cloud_node_mock(2, small), + testutil.cloud_node_mock(3, big)], + arvados_nodes=[testutil.arvados_node_mock(1), + testutil.arvados_node_mock(2), + testutil.arvados_node_mock(3)], + want_sizes=[small, small, big], + avail_sizes=avail_sizes) + + self.daemon.update_server_wishlist([small, big, big]).get(self.TIMEOUT) + + self.assertEqual(0, self.node_shutdown.start.call_count) + + for c in self.daemon.cloud_nodes.get().nodes.itervalues(): + self.daemon.node_can_shutdown(c.actor) + + booting = self.daemon.booting.get() + shutdowns = self.daemon.shutdowns.get() + + self.stop_proxy(self.daemon) + + self.assertEqual(1, self.node_setup.start.call_count) + self.assertEqual(1, self.node_shutdown.start.call_count) + + # booting a new big node + sizecounts = {a[0].id: 0 for a in avail_sizes} + for b in booting.itervalues(): + sizecounts[b.cloud_size.get().id] += 1 + self.assertEqual(0, sizecounts[small.id]) + self.assertEqual(1, sizecounts[big.id]) + + # shutting down a small node + sizecounts = {a[0].id: 0 for a in avail_sizes} + for b in shutdowns.itervalues(): + sizecounts[b.cloud_node.get().size.id] += 1 + self.assertEqual(1, sizecounts[small.id]) + self.assertEqual(0, sizecounts[big.id]) + + def test_node_max_price(self): + small = testutil.MockSize(1) + big = testutil.MockSize(2) + avail_sizes = [(testutil.MockSize(1), {"cores":1, "price":1}), + (testutil.MockSize(2), {"cores":2, "price":2})] + self.make_daemon(want_sizes=[small, small, small, big], + avail_sizes=avail_sizes, + max_nodes=4, + max_total_price=4) + # the daemon runs in another thread, so we need to wait and see + # if it does all the work we're expecting it to do before stopping it. + self.busywait(lambda: self.node_setup.start.call_count == 3) + booting = self.daemon.booting.get() + self.stop_proxy(self.daemon) + + sizecounts = {a[0].id: 0 for a in avail_sizes} + for b in booting.itervalues(): + sizecounts[b.cloud_size.get().id] += 1 + logging.info(sizecounts) + + # Booting 3 small nodes and not booting a big node would also partially + # satisfy the wishlist and come in under the price cap, however the way + # the update_server_wishlist() currently works effectively results in a + # round-robin creation of one node of each size in the wishlist, so + # test for that. + self.assertEqual(2, sizecounts[small.id]) + self.assertEqual(1, sizecounts[big.id])