self.make_actor()
self.wait_for_assignment(self.setup_actor, 'cloud_node')
- def test_unknown_basehttperror_not_retried(self):
+ def test_basehttperror_retried(self):
self.make_mocks()
self.cloud_client.create_node.side_effect = [
- BaseHTTPError(400, "Unknown"),
+ BaseHTTPError(500, "Try again"),
self.cloud_client.create_node.return_value,
]
self.make_actor()
- finished = threading.Event()
- self.setup_actor.subscribe(lambda _: finished.set())
- assert(finished.wait(self.TIMEOUT))
- self.assertEqual(0, self.cloud_client.post_create_node.call_count)
+ self.wait_for_assignment(self.setup_actor, 'cloud_node')
+ self.assertEqual(1, self.cloud_client.post_create_node.call_count)
- def test_known_basehttperror_retried(self):
+ def test_instance_exceeded_not_retried(self):
self.make_mocks()
self.cloud_client.create_node.side_effect = [
BaseHTTPError(400, "InstanceLimitExceeded"),
self.cloud_client.create_node.return_value,
]
self.make_actor()
- self.wait_for_assignment(self.setup_actor, 'cloud_node')
- self.assertEqual(1, self.cloud_client.post_create_node.call_count)
+ done = self.FUTURE_CLASS()
+ self.setup_actor.subscribe(done.set)
+ done.get(self.TIMEOUT)
+ self.assertEqual(0, self.cloud_client.post_create_node.call_count)
def test_failed_post_create_retried(self):
self.make_mocks()
cloud_node = testutil.cloud_node_mock(4)
arv_node = testutil.arvados_node_mock(4, crunch_worker_state='down')
setup = self.start_node_boot(cloud_node, arv_node)
- self.daemon.node_up(setup).get(self.TIMEOUT)
+ self.daemon.node_setup_finished(setup).get(self.TIMEOUT)
self.assertEqual(1, self.alive_monitor_count())
self.daemon.update_arvados_nodes([arv_node])
self.daemon.update_cloud_nodes([cloud_node])
cloud_node = testutil.cloud_node_mock(1)
setup = self.start_node_boot(cloud_node)
self.daemon.update_cloud_nodes([cloud_node])
- self.daemon.node_up(setup).get(self.TIMEOUT)
+ self.daemon.node_setup_finished(setup).get(self.TIMEOUT)
self.assertEqual(1, self.alive_monitor_count())
def test_no_duplication_when_booted_node_listed(self):
cloud_node = testutil.cloud_node_mock(2)
setup = self.start_node_boot(cloud_node, id_num=2)
- self.daemon.node_up(setup)
+ self.daemon.node_setup_finished(setup)
self.daemon.update_cloud_nodes([cloud_node]).get(self.TIMEOUT)
self.assertEqual(1, self.alive_monitor_count())
# even it doesn't appear in the listing (e.g., because of delays
# propagating tags).
setup = self.start_node_boot()
- self.daemon.node_up(setup).get(self.TIMEOUT)
+ self.daemon.node_setup_finished(setup).get(self.TIMEOUT)
self.assertEqual(1, self.alive_monitor_count())
self.daemon.update_cloud_nodes([]).get(self.TIMEOUT)
self.assertEqual(1, self.alive_monitor_count())
def test_booted_unlisted_node_counted(self):
setup = self.start_node_boot(id_num=1)
- self.daemon.node_up(setup)
+ self.daemon.node_setup_finished(setup)
self.daemon.update_server_wishlist(
[testutil.MockSize(1)]).get(self.TIMEOUT)
self.stop_proxy(self.daemon)
def test_booted_node_can_shutdown(self):
setup = self.start_node_boot()
- self.daemon.node_up(setup).get(self.TIMEOUT)
+ self.daemon.node_setup_finished(setup).get(self.TIMEOUT)
self.assertEqual(1, self.alive_monitor_count())
monitor = self.monitor_list()[0].proxy()
self.daemon.update_server_wishlist([])
def test_booted_node_lifecycle(self):
cloud_node = testutil.cloud_node_mock(6)
setup = self.start_node_boot(cloud_node, id_num=6)
- self.daemon.node_up(setup).get(self.TIMEOUT)
+ self.daemon.node_setup_finished(setup).get(self.TIMEOUT)
self.assertEqual(1, self.alive_monitor_count())
monitor = self.monitor_list()[0].proxy()
self.daemon.update_server_wishlist([])
def test_booted_node_shut_down_when_never_listed(self):
setup = self.start_node_boot()
self.cloud_factory().node_start_time.return_value = time.time() - 3601
- self.daemon.node_up(setup).get(self.TIMEOUT)
+ self.daemon.node_setup_finished(setup).get(self.TIMEOUT)
self.assertEqual(1, self.alive_monitor_count())
self.assertFalse(self.node_shutdown.start.called)
now = time.time()
cloud_node = testutil.cloud_node_mock(2)
setup = self.start_node_boot(cloud_node)
self.cloud_factory().node_start_time.return_value = time.time() - 3601
- self.daemon.node_up(setup).get(self.TIMEOUT)
+ self.daemon.node_setup_finished(setup).get(self.TIMEOUT)
self.assertEqual(1, self.alive_monitor_count())
self.daemon.update_cloud_nodes([cloud_node])
self.monitor_list()[0].tell_proxy().consider_shutdown()
arv_node = testutil.arvados_node_mock(4, crunch_worker_state='down')
setup = self.start_node_boot(cloud_node, arv_node)
self.daemon.update_arvados_nodes([arv_node]).get(self.TIMEOUT)
- self.daemon.node_up(setup).get(self.TIMEOUT)
+ self.daemon.node_setup_finished(setup).get(self.TIMEOUT)
self.assertEqual(1, self.alive_monitor_count())
self.monitor_list()[0].proxy().cloud_node_start_time = time.time()-3601
self.daemon.update_cloud_nodes([cloud_node])
cloud_node = testutil.cloud_node_mock(3)
arv_node = testutil.arvados_node_mock(3)
setup = self.start_node_boot(cloud_node, arv_node)
- self.daemon.node_up(setup).get(self.TIMEOUT)
+ self.daemon.node_setup_finished(setup).get(self.TIMEOUT)
self.assertEqual(1, self.alive_monitor_count())
self.daemon.update_cloud_nodes([cloud_node])
self.daemon.update_arvados_nodes([arv_node]).get(self.TIMEOUT)
cloud_node = testutil.cloud_node_mock(5)
arv_node = testutil.arvados_node_mock(5, job_uuid=True)
setup = self.start_node_boot(cloud_node, arv_node)
- self.daemon.node_up(setup).get(self.TIMEOUT)
+ self.daemon.node_setup_finished(setup).get(self.TIMEOUT)
self.assertEqual(1, self.alive_monitor_count())
self.daemon.update_cloud_nodes([cloud_node])
self.daemon.update_arvados_nodes([arv_node]).get(self.TIMEOUT)
new_node.stop_if_no_cloud_node.reset_mock()
self.daemon.shutdown().get(self.TIMEOUT)
self.assertTrue(new_node.stop_if_no_cloud_node.called)
- self.daemon.node_up(new_node).get(self.TIMEOUT)
+ self.daemon.node_setup_finished(new_node).get(self.TIMEOUT)
self.assertTrue(new_node.stop.called)
self.timer.deliver()
self.assertTrue(
def test_subscribers_get_server_lists(self, mock_squeue):
mock_squeue.return_value = ""
- self.build_monitor([{'items': [1, 2]}], self.MockCalculator())
+ self.build_monitor([{'items': [1, 2]}], self.MockCalculator(), True, True)
self.monitor.subscribe(self.subscriber).get(self.TIMEOUT)
self.stop_proxy(self.monitor)
self.subscriber.assert_called_with([testutil.MockSize(1),
"""
super(JobQueueMonitorActorTestCase, self).build_monitor(jobqueue.ServerCalculator(
- [(testutil.MockSize(n), {'cores': n, 'ram': n*1024, 'scratch': n}) for n in range(1, 3)]))
+ [(testutil.MockSize(n), {'cores': n, 'ram': n*1024, 'scratch': n}) for n in range(1, 3)]),
+ True, True)
self.monitor.subscribe(self.subscriber).get(self.TIMEOUT)
self.stop_proxy(self.monitor)
self.subscriber.assert_called_with([testutil.MockSize(1),
"""
super(JobQueueMonitorActorTestCase, self).build_monitor(jobqueue.ServerCalculator(
- [(testutil.MockSize(n), {'cores': n, 'ram': n*1024, 'scratch': n}) for n in range(1, 3)]))
+ [(testutil.MockSize(n), {'cores': n, 'ram': n*1024, 'scratch': n}) for n in range(1, 3)]),
+ True, True)
self.monitor.subscribe(self.subscriber).get(self.TIMEOUT)
self.stop_proxy(self.monitor)
self.subscriber.assert_called_with([testutil.MockSize(1),