services/nodemanager/tests/test_computenode_dispatch.py

   1 #!/usr/bin/env python
   2 # Copyright (C) The Arvados Authors. All rights reserved.
   3 #
   4 # SPDX-License-Identifier: AGPL-3.0
   5
   6 from __future__ import absolute_import, print_function
   7
   8 import time
   9 import unittest
  10
  11 import arvados.errors as arverror
  12 import httplib2
  13 import mock
  14 import pykka
  15 import threading
  16
  17 from libcloud.common.exceptions import BaseHTTPError
  18
  19 import arvnodeman.computenode.dispatch as dispatch
  20 from arvnodeman.computenode.driver import BaseComputeNodeDriver
  21 from . import testutil
  22
  23 class ComputeNodeSetupActorTestCase(testutil.ActorTestMixin, unittest.TestCase):
  24     def make_mocks(self, arvados_effect=None):
  25         if arvados_effect is None:
  26             arvados_effect = [testutil.arvados_node_mock()]
  27         self.arvados_effect = arvados_effect
  28         self.timer = testutil.MockTimer()
  29         self.api_client = mock.MagicMock(name='api_client')
  30         self.api_client.nodes().create().execute.side_effect = arvados_effect
  31         self.api_client.nodes().update().execute.side_effect = arvados_effect
  32         self.cloud_client = mock.MagicMock(name='cloud_client')
  33         self.cloud_client.create_node.return_value = testutil.cloud_node_mock(1)
  34
  35     def make_actor(self, arv_node=None):
  36         if not hasattr(self, 'timer'):
  37             self.make_mocks(arvados_effect=[arv_node] if arv_node else None)
  38         self.setup_actor = dispatch.ComputeNodeSetupActor.start(
  39             self.timer, self.api_client, self.cloud_client,
  40             testutil.MockSize(1), arv_node).proxy()
  41
  42     def assert_node_properties_updated(self, uuid=None,
  43                                        size=testutil.MockSize(1)):
  44         self.api_client.nodes().update.assert_any_call(
  45             uuid=(uuid or self.arvados_effect[-1]['uuid']),
  46             body={
  47                 'properties': {
  48                     'cloud_node': {
  49                         'size': size.id,
  50                         'price': size.price}}})
  51
  52     def test_creation_without_arvados_node(self):
  53         self.make_actor()
  54         finished = threading.Event()
  55         self.setup_actor.subscribe(lambda _: finished.set())
  56         self.assertEqual(self.arvados_effect[-1],
  57                          self.setup_actor.arvados_node.get(self.TIMEOUT))
  58         assert(finished.wait(self.TIMEOUT))
  59         self.assertEqual(1, self.api_client.nodes().create().execute.call_count)
  60         self.assertEqual(1, self.api_client.nodes().update().execute.call_count)
  61         self.assert_node_properties_updated()
  62         self.assertEqual(self.cloud_client.create_node(),
  63                          self.setup_actor.cloud_node.get(self.TIMEOUT))
  64
  65     def test_creation_with_arvados_node(self):
  66         self.make_mocks(arvados_effect=[testutil.arvados_node_mock()]*2)
  67         self.make_actor(testutil.arvados_node_mock())
  68         finished = threading.Event()
  69         self.setup_actor.subscribe(lambda _: finished.set())
  70         self.assertEqual(self.arvados_effect[-1],
  71                          self.setup_actor.arvados_node.get(self.TIMEOUT))
  72         assert(finished.wait(self.TIMEOUT))
  73         self.assert_node_properties_updated()
  74         self.assertEqual(2, self.api_client.nodes().update().execute.call_count)
  75         self.assertEqual(self.cloud_client.create_node(),
  76                          self.setup_actor.cloud_node.get(self.TIMEOUT))
  77
  78     def test_failed_arvados_calls_retried(self):
  79         self.make_mocks([
  80                 arverror.ApiError(httplib2.Response({'status': '500'}), ""),
  81                 testutil.arvados_node_mock(),
  82                 ])
  83         self.make_actor()
  84         self.wait_for_assignment(self.setup_actor, 'arvados_node')
  85
  86     def test_failed_cloud_calls_retried(self):
  87         self.make_mocks()
  88         self.cloud_client.create_node.side_effect = [
  89             Exception("test cloud creation error"),
  90             self.cloud_client.create_node.return_value,
  91             ]
  92         self.make_actor()
  93         self.wait_for_assignment(self.setup_actor, 'cloud_node')
  94
  95     def test_basehttperror_retried(self):
  96         self.make_mocks()
  97         self.cloud_client.create_node.side_effect = [
  98             BaseHTTPError(500, "Try again"),
  99             self.cloud_client.create_node.return_value,
 100             ]
 101         self.make_actor()
 102         self.wait_for_assignment(self.setup_actor, 'cloud_node')
 103         self.setup_actor.ping().get(self.TIMEOUT)
 104         self.assertEqual(1, self.cloud_client.post_create_node.call_count)
 105
 106     def test_instance_exceeded_not_retried(self):
 107         self.make_mocks()
 108         self.cloud_client.create_node.side_effect = [
 109             BaseHTTPError(400, "InstanceLimitExceeded"),
 110             self.cloud_client.create_node.return_value,
 111             ]
 112         self.make_actor()
 113         done = self.FUTURE_CLASS()
 114         self.setup_actor.subscribe(done.set)
 115         done.get(self.TIMEOUT)
 116         self.assertEqual(0, self.cloud_client.post_create_node.call_count)
 117
 118     def test_failed_post_create_retried(self):
 119         self.make_mocks()
 120         self.cloud_client.post_create_node.side_effect = [
 121             Exception("test cloud post-create error"), None]
 122         self.make_actor()
 123         done = self.FUTURE_CLASS()
 124         self.setup_actor.subscribe(done.set)
 125         done.get(self.TIMEOUT)
 126         self.assertEqual(2, self.cloud_client.post_create_node.call_count)
 127
 128     def test_stop_when_no_cloud_node(self):
 129         self.make_mocks(
 130             arverror.ApiError(httplib2.Response({'status': '500'}), ""))
 131         self.make_actor()
 132         self.assertTrue(
 133             self.setup_actor.stop_if_no_cloud_node().get(self.TIMEOUT))
 134         self.assertTrue(
 135             self.setup_actor.actor_ref.actor_stopped.wait(self.TIMEOUT))
 136
 137     def test_no_stop_when_cloud_node(self):
 138         self.make_actor()
 139         self.wait_for_assignment(self.setup_actor, 'cloud_node')
 140         self.assertFalse(
 141             self.setup_actor.stop_if_no_cloud_node().get(self.TIMEOUT))
 142         self.assertTrue(self.stop_proxy(self.setup_actor),
 143                         "actor was stopped by stop_if_no_cloud_node")
 144
 145     def test_subscribe(self):
 146         self.make_mocks(
 147             arverror.ApiError(httplib2.Response({'status': '500'}), ""))
 148         self.make_actor()
 149         subscriber = mock.Mock(name='subscriber_mock')
 150         self.setup_actor.subscribe(subscriber)
 151         retry_resp = [testutil.arvados_node_mock()]
 152         self.api_client.nodes().create().execute.side_effect = retry_resp
 153         self.api_client.nodes().update().execute.side_effect = retry_resp
 154         self.wait_for_assignment(self.setup_actor, 'cloud_node')
 155         self.setup_actor.ping().get(self.TIMEOUT)
 156         self.assertEqual(self.setup_actor.actor_ref.actor_urn,
 157                          subscriber.call_args[0][0].actor_ref.actor_urn)
 158
 159     def test_late_subscribe(self):
 160         self.make_actor()
 161         subscriber = mock.Mock(name='subscriber_mock')
 162         self.wait_for_assignment(self.setup_actor, 'cloud_node')
 163         self.setup_actor.subscribe(subscriber).get(self.TIMEOUT)
 164         self.stop_proxy(self.setup_actor)
 165         self.assertEqual(self.setup_actor.actor_ref.actor_urn,
 166                          subscriber.call_args[0][0].actor_ref.actor_urn)
 167
 168
 169 class ComputeNodeShutdownActorMixin(testutil.ActorTestMixin):
 170     def make_mocks(self, cloud_node=None, arvados_node=None,
 171                    shutdown_open=True, node_broken=False):
 172         self.timer = testutil.MockTimer()
 173         self.shutdowns = testutil.MockShutdownTimer()
 174         self.shutdowns._set_state(shutdown_open, 300)
 175         self.cloud_client = mock.MagicMock(name='cloud_client')
 176         self.cloud_client.broken.return_value = node_broken
 177         self.arvados_client = mock.MagicMock(name='arvados_client')
 178         self.updates = mock.MagicMock(name='update_mock')
 179         if cloud_node is None:
 180             cloud_node = testutil.cloud_node_mock()
 181         self.cloud_node = cloud_node
 182         self.arvados_node = arvados_node
 183
 184     def make_actor(self, cancellable=True, start_time=None):
 185         if not hasattr(self, 'timer'):
 186             self.make_mocks()
 187         if start_time is None:
 188             start_time = time.time()
 189         monitor_actor = dispatch.ComputeNodeMonitorActor.start(
 190             self.cloud_node, start_time, self.shutdowns,
 191             testutil.cloud_node_fqdn, self.timer, self.updates, self.cloud_client,
 192             self.arvados_node)
 193         self.shutdown_actor = self.ACTOR_CLASS.start(
 194             self.timer, self.cloud_client, self.arvados_client, monitor_actor,
 195             cancellable).proxy()
 196         self.monitor_actor = monitor_actor.proxy()
 197
 198     def check_success_flag(self, expected, allow_msg_count=1):
 199         # allow_msg_count is the number of internal messages that may
 200         # need to be handled for shutdown to finish.
 201         for try_num in range(1 + allow_msg_count):
 202             last_flag = self.shutdown_actor.success.get(self.TIMEOUT)
 203             if last_flag is expected:
 204                 break
 205         else:
 206             self.fail("success flag {} is not {}".format(last_flag, expected))
 207
 208     def test_cancellable_shutdown(self, *mocks):
 209         self.make_mocks(shutdown_open=True, arvados_node=testutil.arvados_node_mock(crunch_worker_state="busy"))
 210         self.cloud_client.destroy_node.return_value = True
 211         self.make_actor(cancellable=True)
 212         self.check_success_flag(False, 2)
 213         self.assertFalse(self.cloud_client.destroy_node.called)
 214
 215     def test_uncancellable_shutdown(self, *mocks):
 216         self.make_mocks(shutdown_open=True, arvados_node=testutil.arvados_node_mock(crunch_worker_state="busy"))
 217         self.cloud_client.destroy_node.return_value = True
 218         self.make_actor(cancellable=False)
 219         self.check_success_flag(True, 4)
 220         self.assertTrue(self.cloud_client.destroy_node.called)
 221
 222     def test_arvados_node_cleaned_after_shutdown(self, *mocks):
 223         if len(mocks) == 1:
 224             mocks[0].return_value = "drain\n"
 225         cloud_node = testutil.cloud_node_mock(62)
 226         arv_node = testutil.arvados_node_mock(62)
 227         self.make_mocks(cloud_node, arv_node)
 228         self.make_actor()
 229         self.check_success_flag(True, 3)
 230         update_mock = self.arvados_client.nodes().update
 231         self.assertTrue(update_mock.called)
 232         update_kwargs = update_mock.call_args_list[0][1]
 233         self.assertEqual(arv_node['uuid'], update_kwargs.get('uuid'))
 234         self.assertIn('body', update_kwargs)
 235         for clear_key in ['slot_number', 'hostname', 'ip_address',
 236                           'first_ping_at', 'last_ping_at']:
 237             self.assertIn(clear_key, update_kwargs['body'])
 238             self.assertIsNone(update_kwargs['body'][clear_key])
 239         self.assertTrue(update_mock().execute.called)
 240
 241     def test_arvados_node_not_cleaned_after_shutdown_cancelled(self, *mocks):
 242         if len(mocks) == 1:
 243             mocks[0].return_value = "idle\n"
 244         cloud_node = testutil.cloud_node_mock(61)
 245         arv_node = testutil.arvados_node_mock(61)
 246         self.make_mocks(cloud_node, arv_node, shutdown_open=False)
 247         self.cloud_client.destroy_node.return_value = False
 248         self.make_actor(cancellable=True)
 249         self.shutdown_actor.cancel_shutdown("test")
 250         self.shutdown_actor.ping().get(self.TIMEOUT)
 251         self.check_success_flag(False, 2)
 252         self.assertFalse(self.arvados_client.nodes().update.called)
 253
 254
 255 class ComputeNodeShutdownActorTestCase(ComputeNodeShutdownActorMixin,
 256                                        unittest.TestCase):
 257     ACTOR_CLASS = dispatch.ComputeNodeShutdownActor
 258
 259     def test_easy_shutdown(self):
 260         self.make_actor(start_time=0)
 261         self.check_success_flag(True)
 262         self.assertTrue(self.cloud_client.destroy_node.called)
 263
 264     def test_shutdown_cancelled_when_destroy_node_fails(self):
 265         self.make_mocks(node_broken=True)
 266         self.cloud_client.destroy_node.return_value = False
 267         self.make_actor(start_time=0)
 268         self.check_success_flag(False, 2)
 269         self.assertEqual(1, self.cloud_client.destroy_node.call_count)
 270         self.assertEqual(self.ACTOR_CLASS.DESTROY_FAILED,
 271                          self.shutdown_actor.cancel_reason.get(self.TIMEOUT))
 272
 273     def test_late_subscribe(self):
 274         self.make_actor()
 275         subscriber = mock.Mock(name='subscriber_mock')
 276         self.shutdown_actor.subscribe(subscriber).get(self.TIMEOUT)
 277         self.stop_proxy(self.shutdown_actor)
 278         self.assertTrue(subscriber.called)
 279         self.assertEqual(self.shutdown_actor.actor_ref.actor_urn,
 280                          subscriber.call_args[0][0].actor_ref.actor_urn)
 281
 282
 283 class ComputeNodeUpdateActorTestCase(testutil.ActorTestMixin,
 284                                      unittest.TestCase):
 285     ACTOR_CLASS = dispatch.ComputeNodeUpdateActor
 286
 287     def make_actor(self):
 288         self.driver = mock.MagicMock(name='driver_mock')
 289         self.timer = mock.MagicMock(name='timer_mock')
 290         self.updater = self.ACTOR_CLASS.start(self.driver, self.timer).proxy()
 291
 292     def test_node_sync(self, *args):
 293         self.make_actor()
 294         cloud_node = testutil.cloud_node_mock()
 295         arv_node = testutil.arvados_node_mock()
 296         self.updater.sync_node(cloud_node, arv_node).get(self.TIMEOUT)
 297         self.driver().sync_node.assert_called_with(cloud_node, arv_node)
 298
 299     @testutil.no_sleep
 300     def test_node_sync_error(self, *args):
 301         self.make_actor()
 302         cloud_node = testutil.cloud_node_mock()
 303         arv_node = testutil.arvados_node_mock()
 304         self.driver().sync_node.side_effect = (IOError, Exception, True)
 305         self.updater.sync_node(cloud_node, arv_node).get(self.TIMEOUT)
 306         self.updater.sync_node(cloud_node, arv_node).get(self.TIMEOUT)
 307         self.updater.sync_node(cloud_node, arv_node).get(self.TIMEOUT)
 308         self.driver().sync_node.assert_called_with(cloud_node, arv_node)
 309
 310 class ComputeNodeMonitorActorTestCase(testutil.ActorTestMixin,
 311                                       unittest.TestCase):
 312     def make_mocks(self, node_num):
 313         self.shutdowns = testutil.MockShutdownTimer()
 314         self.shutdowns._set_state(False, 300)
 315         self.timer = mock.MagicMock(name='timer_mock')
 316         self.updates = mock.MagicMock(name='update_mock')
 317         self.cloud_mock = testutil.cloud_node_mock(node_num)
 318         self.subscriber = mock.Mock(name='subscriber_mock')
 319         self.cloud_client = mock.MagicMock(name='cloud_client')
 320         self.cloud_client.broken.return_value = False
 321
 322     def make_actor(self, node_num=1, arv_node=None, start_time=None):
 323         if not hasattr(self, 'cloud_mock'):
 324             self.make_mocks(node_num)
 325         if start_time is None:
 326             start_time = time.time()
 327         self.node_actor = dispatch.ComputeNodeMonitorActor.start(
 328             self.cloud_mock, start_time, self.shutdowns,
 329             testutil.cloud_node_fqdn, self.timer, self.updates, self.cloud_client,
 330             arv_node, boot_fail_after=300).proxy()
 331         self.node_actor.subscribe(self.subscriber).get(self.TIMEOUT)
 332
 333     def node_state(self, *states):
 334         return self.node_actor.in_state(*states).get(self.TIMEOUT)
 335
 336     def test_in_state_when_unpaired(self):
 337         self.make_actor()
 338         self.assertTrue(self.node_state('unpaired'))
 339
 340     def test_in_state_when_pairing_stale(self):
 341         self.make_actor(arv_node=testutil.arvados_node_mock(
 342                 job_uuid=None, age=90000))
 343         self.assertTrue(self.node_state('down'))
 344
 345     def test_in_state_when_no_state_available(self):
 346         self.make_actor(arv_node=testutil.arvados_node_mock(
 347                 crunch_worker_state=None))
 348         self.assertTrue(self.node_state('idle'))
 349
 350     def test_in_state_when_no_state_available_old(self):
 351         self.make_actor(arv_node=testutil.arvados_node_mock(
 352                 crunch_worker_state=None, age=90000))
 353         self.assertTrue(self.node_state('down'))
 354
 355     def test_in_idle_state(self):
 356         self.make_actor(2, arv_node=testutil.arvados_node_mock(job_uuid=None))
 357         self.assertTrue(self.node_state('idle'))
 358         self.assertFalse(self.node_state('busy'))
 359         self.assertTrue(self.node_state('idle', 'busy'))
 360
 361     def test_in_busy_state(self):
 362         self.make_actor(3, arv_node=testutil.arvados_node_mock(job_uuid=True))
 363         self.assertFalse(self.node_state('idle'))
 364         self.assertTrue(self.node_state('busy'))
 365         self.assertTrue(self.node_state('idle', 'busy'))
 366
 367     def test_init_shutdown_scheduling(self):
 368         self.make_actor()
 369         self.assertTrue(self.timer.schedule.called)
 370         self.assertEqual(300, self.timer.schedule.call_args[0][0])
 371
 372     def test_shutdown_window_close_scheduling(self):
 373         self.make_actor()
 374         self.shutdowns._set_state(False, 600)
 375         self.timer.schedule.reset_mock()
 376         self.node_actor.consider_shutdown().get(self.TIMEOUT)
 377         self.stop_proxy(self.node_actor)
 378         self.assertTrue(self.timer.schedule.called)
 379         self.assertEqual(600, self.timer.schedule.call_args[0][0])
 380         self.assertFalse(self.subscriber.called)
 381
 382     def test_shutdown_subscription(self):
 383         self.make_actor(start_time=0)
 384         self.shutdowns._set_state(True, 600)
 385         self.node_actor.consider_shutdown().get(self.TIMEOUT)
 386         self.assertTrue(self.subscriber.called)
 387         self.assertEqual(self.node_actor.actor_ref.actor_urn,
 388                          self.subscriber.call_args[0][0].actor_ref.actor_urn)
 389
 390     def test_no_shutdown_booting(self):
 391         self.make_actor()
 392         self.shutdowns._set_state(True, 600)
 393         self.assertEquals(self.node_actor.shutdown_eligible().get(self.TIMEOUT),
 394                           (False, "node state is ('unpaired', 'open', 'boot wait', 'idle exceeded')"))
 395
 396     def test_shutdown_without_arvados_node(self):
 397         self.make_actor(start_time=0)
 398         self.shutdowns._set_state(True, 600)
 399         self.assertEquals((True, "node state is ('down', 'open', 'boot exceeded', 'idle exceeded')"),
 400                           self.node_actor.shutdown_eligible().get(self.TIMEOUT))
 401
 402     def test_shutdown_missing(self):
 403         arv_node = testutil.arvados_node_mock(10, job_uuid=None,
 404                                               crunch_worker_state="down",
 405                                               last_ping_at='1970-01-01T01:02:03.04050607Z')
 406         self.make_actor(10, arv_node)
 407         self.shutdowns._set_state(True, 600)
 408         self.assertEquals((True, "node state is ('down', 'open', 'boot wait', 'idle exceeded')"),
 409                           self.node_actor.shutdown_eligible().get(self.TIMEOUT))
 410
 411     def test_shutdown_running_broken(self):
 412         arv_node = testutil.arvados_node_mock(12, job_uuid=None,
 413                                               crunch_worker_state="down")
 414         self.make_actor(12, arv_node)
 415         self.shutdowns._set_state(True, 600)
 416         self.cloud_client.broken.return_value = True
 417         self.assertEquals((True, "node state is ('down', 'open', 'boot wait', 'idle exceeded')"),
 418                           self.node_actor.shutdown_eligible().get(self.TIMEOUT))
 419
 420     def test_shutdown_missing_broken(self):
 421         arv_node = testutil.arvados_node_mock(11, job_uuid=None,
 422                                               crunch_worker_state="down",
 423                                               last_ping_at='1970-01-01T01:02:03.04050607Z')
 424         self.make_actor(11, arv_node)
 425         self.shutdowns._set_state(True, 600)
 426         self.cloud_client.broken.return_value = True
 427         self.assertEquals(self.node_actor.shutdown_eligible().get(self.TIMEOUT), (True, "node state is ('down', 'open', 'boot wait', 'idle exceeded')"))
 428
 429     def test_no_shutdown_when_window_closed(self):
 430         self.make_actor(3, testutil.arvados_node_mock(3, job_uuid=None))
 431         self.assertEquals((False, "node state is ('idle', 'closed', 'boot wait', 'idle exceeded')"),
 432                           self.node_actor.shutdown_eligible().get(self.TIMEOUT))
 433
 434     def test_no_shutdown_when_node_running_job(self):
 435         self.make_actor(4, testutil.arvados_node_mock(4, job_uuid=True))
 436         self.shutdowns._set_state(True, 600)
 437         self.assertEquals((False, "node state is ('busy', 'open', 'boot wait', 'idle exceeded')"),
 438                           self.node_actor.shutdown_eligible().get(self.TIMEOUT))
 439
 440     def test_shutdown_when_node_state_unknown(self):
 441         self.make_actor(5, testutil.arvados_node_mock(
 442             5, crunch_worker_state=None))
 443         self.shutdowns._set_state(True, 600)
 444         self.assertEquals((True, "node state is ('idle', 'open', 'boot wait', 'idle exceeded')"),
 445                           self.node_actor.shutdown_eligible().get(self.TIMEOUT))
 446
 447     def test_shutdown_when_node_state_fail(self):
 448         self.make_actor(5, testutil.arvados_node_mock(
 449             5, crunch_worker_state='fail'))
 450         self.shutdowns._set_state(True, 600)
 451         self.assertEquals((True, "node state is ('fail', 'open', 'boot wait', 'idle exceeded')"),
 452                           self.node_actor.shutdown_eligible().get(self.TIMEOUT))
 453
 454     def test_no_shutdown_when_node_state_stale(self):
 455         self.make_actor(6, testutil.arvados_node_mock(6, age=90000))
 456         self.shutdowns._set_state(True, 600)
 457         self.assertEquals((False, "node state is stale"),
 458                           self.node_actor.shutdown_eligible().get(self.TIMEOUT))
 459
 460     def test_arvados_node_match(self):
 461         self.make_actor(2)
 462         arv_node = testutil.arvados_node_mock(
 463             2, hostname='compute-two.zzzzz.arvadosapi.com')
 464         self.cloud_client.node_id.return_value = '2'
 465         pair_id = self.node_actor.offer_arvados_pair(arv_node).get(self.TIMEOUT)
 466         self.assertEqual(self.cloud_mock.id, pair_id)
 467         self.stop_proxy(self.node_actor)
 468         self.updates.sync_node.assert_called_with(self.cloud_mock, arv_node)
 469
 470     def test_arvados_node_mismatch(self):
 471         self.make_actor(3)
 472         arv_node = testutil.arvados_node_mock(1)
 473         self.assertIsNone(
 474             self.node_actor.offer_arvados_pair(arv_node).get(self.TIMEOUT))
 475
 476     def test_arvados_node_mismatch_first_ping_too_early(self):
 477         self.make_actor(4)
 478         arv_node = testutil.arvados_node_mock(
 479             4, first_ping_at='1971-03-02T14:15:16.1717282Z')
 480         self.assertIsNone(
 481             self.node_actor.offer_arvados_pair(arv_node).get(self.TIMEOUT))
 482
 483     def test_update_cloud_node(self):
 484         self.make_actor(1)
 485         self.make_mocks(2)
 486         self.cloud_mock.id = '1'
 487         self.node_actor.update_cloud_node(self.cloud_mock)
 488         current_cloud = self.node_actor.cloud_node.get(self.TIMEOUT)
 489         self.assertEqual([testutil.ip_address_mock(2)],
 490                          current_cloud.private_ips)
 491
 492     def test_missing_cloud_node_update(self):
 493         self.make_actor(1)
 494         self.node_actor.update_cloud_node(None)
 495         current_cloud = self.node_actor.cloud_node.get(self.TIMEOUT)
 496         self.assertEqual([testutil.ip_address_mock(1)],
 497                          current_cloud.private_ips)
 498
 499     def test_update_arvados_node(self):
 500         self.make_actor(3)
 501         job_uuid = 'zzzzz-jjjjj-updatejobnode00'
 502         new_arvados = testutil.arvados_node_mock(3, job_uuid)
 503         self.node_actor.update_arvados_node(new_arvados)
 504         current_arvados = self.node_actor.arvados_node.get(self.TIMEOUT)
 505         self.assertEqual(job_uuid, current_arvados['job_uuid'])
 506
 507     def test_missing_arvados_node_update(self):
 508         self.make_actor(4, testutil.arvados_node_mock(4))
 509         self.node_actor.update_arvados_node(None)
 510         current_arvados = self.node_actor.arvados_node.get(self.TIMEOUT)
 511         self.assertEqual(testutil.ip_address_mock(4),
 512                          current_arvados['ip_address'])
 513
 514     def test_update_arvados_node_syncs_when_fqdn_mismatch(self):
 515         self.make_mocks(5)
 516         self.cloud_mock.extra['testname'] = 'cloudfqdn.zzzzz.arvadosapi.com'
 517         self.make_actor()
 518         arv_node = testutil.arvados_node_mock(5)
 519         self.node_actor.update_arvados_node(arv_node).get(self.TIMEOUT)
 520         self.assertEqual(1, self.updates.sync_node.call_count)
 521
 522     def test_update_arvados_node_skips_sync_when_fqdn_match(self):
 523         self.make_mocks(6)
 524         arv_node = testutil.arvados_node_mock(6)
 525         self.cloud_mock.extra['testname'] ='{n[hostname]}.{n[domain]}'.format(
 526             n=arv_node)
 527         self.make_actor()
 528         self.node_actor.update_arvados_node(arv_node).get(self.TIMEOUT)
 529         self.assertEqual(0, self.updates.sync_node.call_count)