services/nodemanager/tests/test_computenode_dispatch.py

   1 #!/usr/bin/env python
   2 # Copyright (C) The Arvados Authors. All rights reserved.
   3 #
   4 # SPDX-License-Identifier: AGPL-3.0
   5
   6 from __future__ import absolute_import, print_function
   7
   8 import time
   9 import unittest
  10
  11 import arvados.errors as arverror
  12 import httplib2
  13 import mock
  14 import pykka
  15 import threading
  16
  17 from libcloud.common.exceptions import BaseHTTPError
  18
  19 import arvnodeman.computenode.dispatch as dispatch
  20 import arvnodeman.status as status
  21 from arvnodeman.computenode.driver import BaseComputeNodeDriver
  22 from . import testutil
  23
  24 class ComputeNodeSetupActorTestCase(testutil.ActorTestMixin, unittest.TestCase):
  25     ACTOR_CLASS = dispatch.ComputeNodeSetupActor
  26
  27     def make_mocks(self, arvados_effect=None):
  28         if arvados_effect is None:
  29             arvados_effect = [testutil.arvados_node_mock(
  30                 slot_number=None,
  31                 hostname=None,
  32                 first_ping_at=None,
  33                 last_ping_at=None,
  34             )]
  35         self.arvados_effect = arvados_effect
  36         self.timer = testutil.MockTimer()
  37         self.api_client = mock.MagicMock(name='api_client')
  38         self.api_client.nodes().create().execute.side_effect = arvados_effect
  39         self.api_client.nodes().update().execute.side_effect = arvados_effect
  40         self.cloud_client = mock.MagicMock(name='cloud_client')
  41         self.cloud_client.create_node.return_value = testutil.cloud_node_mock(1)
  42
  43     def make_actor(self, arv_node=None):
  44         if not hasattr(self, 'timer'):
  45             self.make_mocks(arvados_effect=[arv_node] if arv_node else None)
  46         self.setup_actor = self.ACTOR_CLASS.start(
  47             self.timer, self.api_client, self.cloud_client,
  48             testutil.MockSize(1), arv_node).proxy()
  49
  50     def assert_node_properties_updated(self, uuid=None,
  51                                        size=testutil.MockSize(1)):
  52         self.api_client.nodes().update.assert_any_call(
  53             uuid=(uuid or self.arvados_effect[-1]['uuid']),
  54             body={
  55                 'properties': {
  56                     'cloud_node': {
  57                         'size': size.id,
  58                         'price': size.price}}})
  59
  60     def test_creation_without_arvados_node(self):
  61         self.make_actor()
  62         finished = threading.Event()
  63         self.setup_actor.subscribe(lambda _: finished.set())
  64         self.assertEqual(self.arvados_effect[-1],
  65                          self.setup_actor.arvados_node.get(self.TIMEOUT))
  66         assert(finished.wait(self.TIMEOUT))
  67         self.api_client.nodes().create.called_with(body={}, assign_slot=True)
  68         self.assertEqual(1, self.api_client.nodes().create().execute.call_count)
  69         self.assertEqual(1, self.api_client.nodes().update().execute.call_count)
  70         self.assert_node_properties_updated()
  71         self.assertEqual(self.cloud_client.create_node(),
  72                          self.setup_actor.cloud_node.get(self.TIMEOUT))
  73
  74     def test_creation_with_arvados_node(self):
  75         self.make_mocks(arvados_effect=[testutil.arvados_node_mock()]*2)
  76         self.make_actor(testutil.arvados_node_mock())
  77         finished = threading.Event()
  78         self.setup_actor.subscribe(lambda _: finished.set())
  79         self.assertEqual(self.arvados_effect[-1],
  80                          self.setup_actor.arvados_node.get(self.TIMEOUT))
  81         assert(finished.wait(self.TIMEOUT))
  82         self.assert_node_properties_updated()
  83         self.api_client.nodes().create.called_with(body={}, assign_slot=True)
  84         self.assertEqual(3, self.api_client.nodes().update().execute.call_count)
  85         self.assertEqual(self.cloud_client.create_node(),
  86                          self.setup_actor.cloud_node.get(self.TIMEOUT))
  87
  88     def test_failed_arvados_calls_retried(self):
  89         self.make_mocks([
  90                 arverror.ApiError(httplib2.Response({'status': '500'}), ""),
  91                 testutil.arvados_node_mock(),
  92                 ])
  93         self.make_actor()
  94         self.wait_for_assignment(self.setup_actor, 'arvados_node')
  95
  96     def test_failed_cloud_calls_retried(self):
  97         self.make_mocks()
  98         self.cloud_client.create_node.side_effect = [
  99             Exception("test cloud creation error"),
 100             self.cloud_client.create_node.return_value,
 101             ]
 102         self.make_actor()
 103         self.wait_for_assignment(self.setup_actor, 'cloud_node')
 104
 105     def test_basehttperror_retried(self):
 106         self.make_mocks()
 107         self.cloud_client.create_node.side_effect = [
 108             BaseHTTPError(500, "Try again"),
 109             self.cloud_client.create_node.return_value,
 110             ]
 111         self.make_actor()
 112         self.wait_for_assignment(self.setup_actor, 'cloud_node')
 113         self.setup_actor.ping().get(self.TIMEOUT)
 114         self.assertEqual(1, self.cloud_client.post_create_node.call_count)
 115
 116     def test_instance_exceeded_not_retried(self):
 117         self.make_mocks()
 118         self.cloud_client.create_node.side_effect = [
 119             BaseHTTPError(400, "InstanceLimitExceeded"),
 120             self.cloud_client.create_node.return_value,
 121             ]
 122         self.make_actor()
 123         done = self.FUTURE_CLASS()
 124         self.setup_actor.subscribe(done.set)
 125         done.get(self.TIMEOUT)
 126         self.assertEqual(0, self.cloud_client.post_create_node.call_count)
 127
 128     def test_failed_post_create_retried(self):
 129         self.make_mocks()
 130         self.cloud_client.post_create_node.side_effect = [
 131             Exception("test cloud post-create error"), None]
 132         self.make_actor()
 133         done = self.FUTURE_CLASS()
 134         self.setup_actor.subscribe(done.set)
 135         done.get(self.TIMEOUT)
 136         self.assertEqual(2, self.cloud_client.post_create_node.call_count)
 137
 138     def test_stop_when_no_cloud_node(self):
 139         self.make_mocks(
 140             arverror.ApiError(httplib2.Response({'status': '500'}), ""))
 141         self.make_actor()
 142         self.assertTrue(
 143             self.setup_actor.stop_if_no_cloud_node().get(self.TIMEOUT))
 144         self.assertTrue(
 145             self.setup_actor.actor_ref.actor_stopped.wait(self.TIMEOUT))
 146
 147     def test_no_stop_when_cloud_node(self):
 148         self.make_actor()
 149         self.wait_for_assignment(self.setup_actor, 'cloud_node')
 150         self.assertFalse(
 151             self.setup_actor.stop_if_no_cloud_node().get(self.TIMEOUT))
 152         self.assertTrue(self.stop_proxy(self.setup_actor),
 153                         "actor was stopped by stop_if_no_cloud_node")
 154
 155     def test_subscribe(self):
 156         self.make_mocks(
 157             arverror.ApiError(httplib2.Response({'status': '500'}), ""))
 158         self.make_actor()
 159         subscriber = mock.Mock(name='subscriber_mock')
 160         self.setup_actor.subscribe(subscriber)
 161         retry_resp = [testutil.arvados_node_mock()]
 162         self.api_client.nodes().create().execute.side_effect = retry_resp
 163         self.api_client.nodes().update().execute.side_effect = retry_resp
 164         self.wait_for_assignment(self.setup_actor, 'cloud_node')
 165         self.setup_actor.ping().get(self.TIMEOUT)
 166         self.assertEqual(self.setup_actor.actor_ref.actor_urn,
 167                          subscriber.call_args[0][0].actor_ref.actor_urn)
 168
 169     def test_late_subscribe(self):
 170         self.make_actor()
 171         subscriber = mock.Mock(name='subscriber_mock')
 172         self.wait_for_assignment(self.setup_actor, 'cloud_node')
 173         self.setup_actor.subscribe(subscriber).get(self.TIMEOUT)
 174         self.stop_proxy(self.setup_actor)
 175         self.assertEqual(self.setup_actor.actor_ref.actor_urn,
 176                          subscriber.call_args[0][0].actor_ref.actor_urn)
 177
 178
 179 class ComputeNodeShutdownActorMixin(testutil.ActorTestMixin):
 180     def make_mocks(self, cloud_node=None, arvados_node=None,
 181                    shutdown_open=True, node_broken=False):
 182         self.timer = testutil.MockTimer()
 183         self.shutdowns = testutil.MockShutdownTimer()
 184         self.shutdowns._set_state(shutdown_open, 300)
 185         self.cloud_client = mock.MagicMock(name='cloud_client')
 186         self.cloud_client.broken.return_value = node_broken
 187         self.arvados_client = mock.MagicMock(name='arvados_client')
 188         self.updates = mock.MagicMock(name='update_mock')
 189         if cloud_node is None:
 190             cloud_node = testutil.cloud_node_mock()
 191         self.cloud_node = cloud_node
 192         self.arvados_node = arvados_node
 193
 194     def make_actor(self, cancellable=True, start_time=None):
 195         if not hasattr(self, 'timer'):
 196             self.make_mocks()
 197         if start_time is None:
 198             start_time = time.time()
 199         monitor_actor = dispatch.ComputeNodeMonitorActor.start(
 200             self.cloud_node, start_time, self.shutdowns,
 201             self.timer, self.updates, self.cloud_client,
 202             self.arvados_node)
 203         self.shutdown_actor = self.ACTOR_CLASS.start(
 204             self.timer, self.cloud_client, self.arvados_client, monitor_actor,
 205             cancellable).proxy()
 206         self.monitor_actor = monitor_actor.proxy()
 207
 208     def check_success_flag(self, expected, allow_msg_count=1):
 209         # allow_msg_count is the number of internal messages that may
 210         # need to be handled for shutdown to finish.
 211         for _ in range(1 + allow_msg_count):
 212             last_flag = self.shutdown_actor.success.get(self.TIMEOUT)
 213             if last_flag is expected:
 214                 break
 215         else:
 216             self.fail("success flag {} is not {}".format(last_flag, expected))
 217
 218     def test_boot_failure_counting(self, *mocks):
 219         # A boot failure happens when a node transitions from unpaired to shutdown
 220         status.tracker.update({'boot_failures': 0})
 221         self.make_mocks(shutdown_open=True, arvados_node=testutil.arvados_node_mock(crunch_worker_state="unpaired"))
 222         self.cloud_client.destroy_node.return_value = True
 223         self.make_actor(cancellable=False)
 224         self.check_success_flag(True, 2)
 225         self.assertTrue(self.cloud_client.destroy_node.called)
 226         self.assertEqual(1, status.tracker.get('boot_failures'))
 227
 228     def test_cancellable_shutdown(self, *mocks):
 229         self.make_mocks(shutdown_open=True, arvados_node=testutil.arvados_node_mock(crunch_worker_state="busy"))
 230         self.cloud_client.destroy_node.return_value = True
 231         self.make_actor(cancellable=True)
 232         self.check_success_flag(False, 2)
 233         self.assertFalse(self.cloud_client.destroy_node.called)
 234
 235     def test_uncancellable_shutdown(self, *mocks):
 236         status.tracker.update({'boot_failures': 0})
 237         self.make_mocks(shutdown_open=True, arvados_node=testutil.arvados_node_mock(crunch_worker_state="busy"))
 238         self.cloud_client.destroy_node.return_value = True
 239         self.make_actor(cancellable=False)
 240         self.check_success_flag(True, 4)
 241         self.assertTrue(self.cloud_client.destroy_node.called)
 242         # A normal shutdown shouldn't be counted as boot failure
 243         self.assertEqual(0, status.tracker.get('boot_failures'))
 244
 245     def test_arvados_node_cleaned_after_shutdown(self, *mocks):
 246         if len(mocks) == 1:
 247             mocks[0].return_value = "drain\n"
 248         cloud_node = testutil.cloud_node_mock(62)
 249         arv_node = testutil.arvados_node_mock(62)
 250         self.make_mocks(cloud_node, arv_node)
 251         self.make_actor()
 252         self.check_success_flag(True, 3)
 253         update_mock = self.arvados_client.nodes().update
 254         self.assertTrue(update_mock.called)
 255         update_kwargs = update_mock.call_args_list[0][1]
 256         self.assertEqual(arv_node['uuid'], update_kwargs.get('uuid'))
 257         self.assertIn('body', update_kwargs)
 258         for clear_key in ['slot_number', 'hostname', 'ip_address',
 259                           'first_ping_at', 'last_ping_at']:
 260             self.assertIn(clear_key, update_kwargs['body'])
 261             self.assertIsNone(update_kwargs['body'][clear_key])
 262         self.assertTrue(update_mock().execute.called)
 263
 264     def test_arvados_node_not_cleaned_after_shutdown_cancelled(self, *mocks):
 265         if len(mocks) == 1:
 266             mocks[0].return_value = "idle\n"
 267         cloud_node = testutil.cloud_node_mock(61)
 268         arv_node = testutil.arvados_node_mock(61)
 269         self.make_mocks(cloud_node, arv_node, shutdown_open=False)
 270         self.cloud_client.destroy_node.return_value = False
 271         self.make_actor(cancellable=True)
 272         self.shutdown_actor.cancel_shutdown("test")
 273         self.shutdown_actor.ping().get(self.TIMEOUT)
 274         self.check_success_flag(False, 2)
 275         self.assertFalse(self.arvados_client.nodes().update.called)
 276
 277
 278 class ComputeNodeShutdownActorTestCase(ComputeNodeShutdownActorMixin,
 279                                        unittest.TestCase):
 280     ACTOR_CLASS = dispatch.ComputeNodeShutdownActor
 281
 282     def test_easy_shutdown(self):
 283         self.make_actor(start_time=0)
 284         self.check_success_flag(True)
 285         self.assertTrue(self.cloud_client.destroy_node.called)
 286
 287     def test_shutdown_cancelled_when_destroy_node_fails(self):
 288         self.make_mocks(node_broken=True)
 289         self.cloud_client.destroy_node.return_value = False
 290         self.make_actor(start_time=0)
 291         self.check_success_flag(False, 2)
 292         self.assertEqual(1, self.cloud_client.destroy_node.call_count)
 293         self.assertEqual(self.ACTOR_CLASS.DESTROY_FAILED,
 294                          self.shutdown_actor.cancel_reason.get(self.TIMEOUT))
 295
 296     def test_late_subscribe(self):
 297         self.make_actor()
 298         subscriber = mock.Mock(name='subscriber_mock')
 299         self.shutdown_actor.subscribe(subscriber).get(self.TIMEOUT)
 300         self.stop_proxy(self.shutdown_actor)
 301         self.assertTrue(subscriber.called)
 302         self.assertEqual(self.shutdown_actor.actor_ref.actor_urn,
 303                          subscriber.call_args[0][0].actor_ref.actor_urn)
 304
 305
 306 class ComputeNodeUpdateActorTestCase(testutil.ActorTestMixin,
 307                                      unittest.TestCase):
 308     ACTOR_CLASS = dispatch.ComputeNodeUpdateActor
 309
 310     def make_actor(self):
 311         self.driver = mock.MagicMock(name='driver_mock')
 312         self.timer = mock.MagicMock(name='timer_mock')
 313         self.updater = self.ACTOR_CLASS.start(self.driver, self.timer).proxy()
 314
 315     def test_node_sync(self, *args):
 316         self.make_actor()
 317         cloud_node = testutil.cloud_node_mock()
 318         arv_node = testutil.arvados_node_mock()
 319         self.updater.sync_node(cloud_node, arv_node).get(self.TIMEOUT)
 320         self.driver().sync_node.assert_called_with(cloud_node, arv_node)
 321
 322     @testutil.no_sleep
 323     def test_node_sync_error(self, *args):
 324         self.make_actor()
 325         cloud_node = testutil.cloud_node_mock()
 326         arv_node = testutil.arvados_node_mock()
 327         self.driver().sync_node.side_effect = (IOError, Exception, True)
 328         self.updater.sync_node(cloud_node, arv_node).get(self.TIMEOUT)
 329         self.updater.sync_node(cloud_node, arv_node).get(self.TIMEOUT)
 330         self.updater.sync_node(cloud_node, arv_node).get(self.TIMEOUT)
 331         self.driver().sync_node.assert_called_with(cloud_node, arv_node)
 332
 333 class ComputeNodeMonitorActorTestCase(testutil.ActorTestMixin,
 334                                       unittest.TestCase):
 335     def make_mocks(self, node_num):
 336         self.shutdowns = testutil.MockShutdownTimer()
 337         self.shutdowns._set_state(False, 300)
 338         self.timer = mock.MagicMock(name='timer_mock')
 339         self.updates = mock.MagicMock(name='update_mock')
 340         self.cloud_mock = testutil.cloud_node_mock(node_num)
 341         self.subscriber = mock.Mock(name='subscriber_mock')
 342         self.cloud_client = mock.MagicMock(name='cloud_client')
 343         self.cloud_client.broken.return_value = False
 344
 345     def make_actor(self, node_num=1, arv_node=None, start_time=None):
 346         if not hasattr(self, 'cloud_mock'):
 347             self.make_mocks(node_num)
 348         if start_time is None:
 349             start_time = time.time()
 350         self.node_actor = dispatch.ComputeNodeMonitorActor.start(
 351             self.cloud_mock, start_time, self.shutdowns,
 352             self.timer, self.updates, self.cloud_client,
 353             arv_node, boot_fail_after=300).proxy()
 354         self.node_actor.subscribe(self.subscriber).get(self.TIMEOUT)
 355
 356     def node_state(self, *states):
 357         return self.node_actor.in_state(*states).get(self.TIMEOUT)
 358
 359     def test_in_state_when_unpaired(self):
 360         self.make_actor()
 361         self.assertTrue(self.node_state('unpaired'))
 362
 363     def test_in_state_when_pairing_stale(self):
 364         self.make_actor(arv_node=testutil.arvados_node_mock(
 365                 job_uuid=None, age=90000))
 366         self.assertTrue(self.node_state('down'))
 367
 368     def test_in_state_when_no_state_available(self):
 369         self.make_actor(arv_node=testutil.arvados_node_mock(
 370                 crunch_worker_state=None))
 371         self.assertTrue(self.node_state('idle'))
 372
 373     def test_in_state_when_no_state_available_old(self):
 374         self.make_actor(arv_node=testutil.arvados_node_mock(
 375                 crunch_worker_state=None, age=90000))
 376         self.assertTrue(self.node_state('down'))
 377
 378     def test_in_idle_state(self):
 379         self.make_actor(2, arv_node=testutil.arvados_node_mock(job_uuid=None))
 380         self.assertTrue(self.node_state('idle'))
 381         self.assertFalse(self.node_state('busy'))
 382         self.assertTrue(self.node_state('idle', 'busy'))
 383
 384     def test_in_busy_state(self):
 385         self.make_actor(3, arv_node=testutil.arvados_node_mock(job_uuid=True))
 386         self.assertFalse(self.node_state('idle'))
 387         self.assertTrue(self.node_state('busy'))
 388         self.assertTrue(self.node_state('idle', 'busy'))
 389
 390     def test_init_shutdown_scheduling(self):
 391         self.make_actor()
 392         self.assertTrue(self.timer.schedule.called)
 393         self.assertEqual(300, self.timer.schedule.call_args[0][0])
 394
 395     def test_shutdown_window_close_scheduling(self):
 396         self.make_actor()
 397         self.shutdowns._set_state(False, 600)
 398         self.timer.schedule.reset_mock()
 399         self.node_actor.consider_shutdown().get(self.TIMEOUT)
 400         self.stop_proxy(self.node_actor)
 401         self.assertTrue(self.timer.schedule.called)
 402         self.assertEqual(600, self.timer.schedule.call_args[0][0])
 403         self.assertFalse(self.subscriber.called)
 404
 405     def test_shutdown_subscription(self):
 406         self.make_actor(start_time=0)
 407         self.shutdowns._set_state(True, 600)
 408         self.node_actor.consider_shutdown().get(self.TIMEOUT)
 409         self.assertTrue(self.subscriber.called)
 410         self.assertEqual(self.node_actor.actor_ref.actor_urn,
 411                          self.subscriber.call_args[0][0].actor_ref.actor_urn)
 412
 413     def test_no_shutdown_booting(self):
 414         self.make_actor()
 415         self.shutdowns._set_state(True, 600)
 416         self.assertEquals(self.node_actor.shutdown_eligible().get(self.TIMEOUT),
 417                           (False, "node state is ('unpaired', 'open', 'boot wait', 'idle exceeded')"))
 418
 419     def test_shutdown_without_arvados_node(self):
 420         self.make_actor(start_time=0)
 421         self.shutdowns._set_state(True, 600)
 422         self.assertEquals((True, "node state is ('down', 'open', 'boot exceeded', 'idle exceeded')"),
 423                           self.node_actor.shutdown_eligible().get(self.TIMEOUT))
 424
 425     def test_shutdown_missing(self):
 426         arv_node = testutil.arvados_node_mock(10, job_uuid=None,
 427                                               crunch_worker_state="down",
 428                                               last_ping_at='1970-01-01T01:02:03.04050607Z')
 429         self.make_actor(10, arv_node)
 430         self.shutdowns._set_state(True, 600)
 431         self.assertEquals((True, "node state is ('down', 'open', 'boot wait', 'idle exceeded')"),
 432                           self.node_actor.shutdown_eligible().get(self.TIMEOUT))
 433
 434     def test_shutdown_running_broken(self):
 435         arv_node = testutil.arvados_node_mock(12, job_uuid=None,
 436                                               crunch_worker_state="down")
 437         self.make_actor(12, arv_node)
 438         self.shutdowns._set_state(True, 600)
 439         self.cloud_client.broken.return_value = True
 440         self.assertEquals((True, "node state is ('down', 'open', 'boot wait', 'idle exceeded')"),
 441                           self.node_actor.shutdown_eligible().get(self.TIMEOUT))
 442
 443     def test_shutdown_missing_broken(self):
 444         arv_node = testutil.arvados_node_mock(11, job_uuid=None,
 445                                               crunch_worker_state="down",
 446                                               last_ping_at='1970-01-01T01:02:03.04050607Z')
 447         self.make_actor(11, arv_node)
 448         self.shutdowns._set_state(True, 600)
 449         self.cloud_client.broken.return_value = True
 450         self.assertEquals(self.node_actor.shutdown_eligible().get(self.TIMEOUT), (True, "node state is ('down', 'open', 'boot wait', 'idle exceeded')"))
 451
 452     def test_no_shutdown_when_window_closed(self):
 453         self.make_actor(3, testutil.arvados_node_mock(3, job_uuid=None))
 454         self.assertEquals((False, "node state is ('idle', 'closed', 'boot wait', 'idle exceeded')"),
 455                           self.node_actor.shutdown_eligible().get(self.TIMEOUT))
 456
 457     def test_no_shutdown_when_node_running_job(self):
 458         self.make_actor(4, testutil.arvados_node_mock(4, job_uuid=True))
 459         self.shutdowns._set_state(True, 600)
 460         self.assertEquals((False, "node state is ('busy', 'open', 'boot wait', 'idle exceeded')"),
 461                           self.node_actor.shutdown_eligible().get(self.TIMEOUT))
 462
 463     def test_shutdown_when_node_state_unknown(self):
 464         self.make_actor(5, testutil.arvados_node_mock(
 465             5, crunch_worker_state=None))
 466         self.shutdowns._set_state(True, 600)
 467         self.assertEquals((True, "node state is ('idle', 'open', 'boot wait', 'idle exceeded')"),
 468                           self.node_actor.shutdown_eligible().get(self.TIMEOUT))
 469
 470     def test_shutdown_when_node_state_fail(self):
 471         self.make_actor(5, testutil.arvados_node_mock(
 472             5, crunch_worker_state='fail'))
 473         self.shutdowns._set_state(True, 600)
 474         self.assertEquals((True, "node state is ('fail', 'open', 'boot wait', 'idle exceeded')"),
 475                           self.node_actor.shutdown_eligible().get(self.TIMEOUT))
 476
 477     def test_no_shutdown_when_node_state_stale(self):
 478         self.make_actor(6, testutil.arvados_node_mock(6, age=90000))
 479         self.shutdowns._set_state(True, 600)
 480         self.assertEquals((False, "node state is stale"),
 481                           self.node_actor.shutdown_eligible().get(self.TIMEOUT))
 482
 483     def test_arvados_node_match(self):
 484         self.make_actor(2)
 485         arv_node = testutil.arvados_node_mock(
 486             2, hostname='compute-two.zzzzz.arvadosapi.com')
 487         self.cloud_client.node_id.return_value = '2'
 488         pair_id = self.node_actor.offer_arvados_pair(arv_node).get(self.TIMEOUT)
 489         self.assertEqual(self.cloud_mock.id, pair_id)
 490         self.stop_proxy(self.node_actor)
 491         self.updates.sync_node.assert_called_with(self.cloud_mock, arv_node)
 492
 493     def test_arvados_node_mismatch(self):
 494         self.make_actor(3)
 495         arv_node = testutil.arvados_node_mock(1)
 496         self.assertIsNone(
 497             self.node_actor.offer_arvados_pair(arv_node).get(self.TIMEOUT))
 498
 499     def test_arvados_node_mismatch_first_ping_too_early(self):
 500         self.make_actor(4)
 501         arv_node = testutil.arvados_node_mock(
 502             4, first_ping_at='1971-03-02T14:15:16.1717282Z')
 503         self.assertIsNone(
 504             self.node_actor.offer_arvados_pair(arv_node).get(self.TIMEOUT))
 505
 506     def test_update_cloud_node(self):
 507         self.make_actor(1)
 508         self.make_mocks(2)
 509         self.cloud_mock.id = '1'
 510         self.node_actor.update_cloud_node(self.cloud_mock)
 511         current_cloud = self.node_actor.cloud_node.get(self.TIMEOUT)
 512         self.assertEqual([testutil.ip_address_mock(2)],
 513                          current_cloud.private_ips)
 514
 515     def test_missing_cloud_node_update(self):
 516         self.make_actor(1)
 517         self.node_actor.update_cloud_node(None)
 518         current_cloud = self.node_actor.cloud_node.get(self.TIMEOUT)
 519         self.assertEqual([testutil.ip_address_mock(1)],
 520                          current_cloud.private_ips)
 521
 522     def test_update_arvados_node(self):
 523         self.make_actor(3)
 524         job_uuid = 'zzzzz-jjjjj-updatejobnode00'
 525         new_arvados = testutil.arvados_node_mock(3, job_uuid)
 526         self.node_actor.update_arvados_node(new_arvados)
 527         current_arvados = self.node_actor.arvados_node.get(self.TIMEOUT)
 528         self.assertEqual(job_uuid, current_arvados['job_uuid'])
 529
 530     def test_missing_arvados_node_update(self):
 531         self.make_actor(4, testutil.arvados_node_mock(4))
 532         self.node_actor.update_arvados_node(None)
 533         current_arvados = self.node_actor.arvados_node.get(self.TIMEOUT)
 534         self.assertEqual(testutil.ip_address_mock(4),
 535                          current_arvados['ip_address'])
 536
 537     def test_update_arvados_node_calls_sync_node(self):
 538         self.make_mocks(5)
 539         self.cloud_mock.extra['testname'] = 'cloudfqdn.zzzzz.arvadosapi.com'
 540         self.make_actor()
 541         arv_node = testutil.arvados_node_mock(5)
 542         self.node_actor.update_arvados_node(arv_node).get(self.TIMEOUT)
 543         self.assertEqual(1, self.updates.sync_node.call_count)