services/dockercleaner/tests/test_cleaner.py

   1 #!/usr/bin/env python3
   2
   3 import collections
   4 import itertools
   5 import json
   6 import os
   7 import random
   8 import tempfile
   9 import time
  10 import unittest
  11
  12 import docker
  13 import mock
  14
  15 from arvados_docker import cleaner
  16
  17 MAX_DOCKER_ID = (16 ** 64) - 1
  18
  19
  20 def MockDockerId():
  21     return '{:064x}'.format(random.randint(0, MAX_DOCKER_ID))
  22
  23
  24 def MockContainer(image_hash):
  25     return {'Id': MockDockerId(),
  26             'Image': image_hash['Id']}
  27
  28
  29 def MockImage(*, size=0, vsize=None, tags=[]):
  30     if vsize is None:
  31         vsize = random.randint(100, 2000000)
  32     return {'Id': MockDockerId(),
  33             'ParentId': MockDockerId(),
  34             'RepoTags': list(tags),
  35             'Size': size,
  36             'VirtualSize': vsize}
  37
  38
  39 class MockEvent(dict):
  40     ENCODING = 'utf-8'
  41     event_seq = itertools.count(1)
  42
  43     def __init__(self, status, docker_id=None, **event_data):
  44         if docker_id is None:
  45             docker_id = MockDockerId()
  46         super().__init__(self, **event_data)
  47         self['status'] = status
  48         self['id'] = docker_id
  49         self.setdefault('time', next(self.event_seq))
  50
  51     def encoded(self):
  52         return json.dumps(self).encode(self.ENCODING)
  53
  54
  55 class MockException(docker.errors.APIError):
  56
  57     def __init__(self, status_code):
  58         response = mock.Mock(name='response')
  59         response.status_code = status_code
  60         super().__init__("mock exception", response)
  61
  62
  63 class DockerImageTestCase(unittest.TestCase):
  64
  65     def test_used_at_sets_last_used(self):
  66         image = cleaner.DockerImage(MockImage())
  67         image.used_at(5)
  68         self.assertEqual(5, image.last_used)
  69
  70     def test_used_at_moves_forward(self):
  71         image = cleaner.DockerImage(MockImage())
  72         image.used_at(6)
  73         image.used_at(8)
  74         self.assertEqual(8, image.last_used)
  75
  76     def test_used_at_does_not_go_backward(self):
  77         image = cleaner.DockerImage(MockImage())
  78         image.used_at(4)
  79         image.used_at(2)
  80         self.assertEqual(4, image.last_used)
  81
  82
  83 class DockerImagesTestCase(unittest.TestCase):
  84
  85     def setUp(self):
  86         self.mock_images = []
  87
  88     def setup_mock_images(self, *vsizes):
  89         self.mock_images.extend(MockImage(vsize=vsize) for vsize in vsizes)
  90
  91     def setup_images(self, *vsizes, target_size=1000000):
  92         self.setup_mock_images(*vsizes)
  93         images = cleaner.DockerImages(target_size)
  94         for image in self.mock_images:
  95             images.add_image(image)
  96         return images
  97
  98     def test_has_image(self):
  99         images = self.setup_images(None)
 100         self.assertTrue(images.has_image(self.mock_images[0]['Id']))
 101         self.assertFalse(images.has_image(MockDockerId()))
 102
 103     def test_del_image(self):
 104         images = self.setup_images(None)
 105         images.del_image(self.mock_images[0]['Id'])
 106         self.assertFalse(images.has_image(self.mock_images[0]['Id']))
 107
 108     def test_del_nonexistent_image(self):
 109         images = self.setup_images(None)
 110         images.del_image(MockDockerId())
 111         self.assertTrue(images.has_image(self.mock_images[0]['Id']))
 112
 113     def test_one_image_always_kept(self):
 114         # When crunch-job starts a job, it makes sure each compute node
 115         # has the Docker image loaded, then it runs all the tasks with
 116         # the assumption the image is on each node.  As long as that's
 117         # true, the cleaner should avoid removing every installed image:
 118         # crunch-job might be counting on the most recent one to be
 119         # available, even if it's not currently in use.
 120         images = self.setup_images(None, None, target_size=1)
 121         for use_time, image in enumerate(self.mock_images, 1):
 122             user = MockContainer(image)
 123             images.add_user(user, use_time)
 124             images.end_user(user['Id'])
 125         self.assertEqual([self.mock_images[0]['Id']],
 126                          list(images.should_delete()))
 127
 128     def test_images_under_target_not_deletable(self):
 129         # The images are used in this order.  target_size is set so it
 130         # could hold the largest image, but not after the most recently
 131         # used image is kept; then we have to fall back to the previous one.
 132         images = self.setup_images(20, 30, 40, 10, target_size=45)
 133         for use_time, image in enumerate(self.mock_images, 1):
 134             user = MockContainer(image)
 135             images.add_user(user, use_time)
 136             images.end_user(user['Id'])
 137         self.assertEqual([self.mock_images[ii]['Id'] for ii in [0, 2]],
 138                          list(images.should_delete()))
 139
 140     def test_images_in_use_not_deletable(self):
 141         images = self.setup_images(None, None, target_size=1)
 142         users = [MockContainer(image) for image in self.mock_images]
 143         images.add_user(users[0], 1)
 144         images.add_user(users[1], 2)
 145         images.end_user(users[1]['Id'])
 146         self.assertEqual([self.mock_images[1]['Id']],
 147                          list(images.should_delete()))
 148
 149     def test_image_deletable_after_unused(self):
 150         images = self.setup_images(None, None, target_size=1)
 151         users = [MockContainer(image) for image in self.mock_images]
 152         images.add_user(users[0], 1)
 153         images.add_user(users[1], 2)
 154         images.end_user(users[0]['Id'])
 155         self.assertEqual([self.mock_images[0]['Id']],
 156                          list(images.should_delete()))
 157
 158     def test_image_not_deletable_if_user_restarts(self):
 159         images = self.setup_images(None, target_size=1)
 160         user = MockContainer(self.mock_images[-1])
 161         images.add_user(user, 1)
 162         images.end_user(user['Id'])
 163         images.add_user(user, 2)
 164         self.assertEqual([], list(images.should_delete()))
 165
 166     def test_image_not_deletable_if_any_user_remains(self):
 167         images = self.setup_images(None, target_size=1)
 168         users = [MockContainer(self.mock_images[0]) for ii in range(2)]
 169         images.add_user(users[0], 1)
 170         images.add_user(users[1], 2)
 171         images.end_user(users[0]['Id'])
 172         self.assertEqual([], list(images.should_delete()))
 173
 174     def test_image_deletable_after_all_users_end(self):
 175         images = self.setup_images(None, None, target_size=1)
 176         users = [MockContainer(self.mock_images[ii]) for ii in [0, 1, 1]]
 177         images.add_user(users[0], 1)
 178         images.add_user(users[1], 2)
 179         images.add_user(users[2], 3)
 180         images.end_user(users[1]['Id'])
 181         images.end_user(users[2]['Id'])
 182         self.assertEqual([self.mock_images[-1]['Id']],
 183                          list(images.should_delete()))
 184
 185     def test_images_suggested_for_deletion_by_lru(self):
 186         images = self.setup_images(10, 10, 10, target_size=1)
 187         users = [MockContainer(image) for image in self.mock_images]
 188         images.add_user(users[0], 3)
 189         images.add_user(users[1], 1)
 190         images.add_user(users[2], 2)
 191         for user in users:
 192             images.end_user(user['Id'])
 193         self.assertEqual([self.mock_images[ii]['Id'] for ii in [1, 2]],
 194                          list(images.should_delete()))
 195
 196     def test_adding_user_without_image_does_not_implicitly_add_image(self):
 197         images = self.setup_images(10)
 198         images.add_user(MockContainer(MockImage()), 1)
 199         self.assertEqual([], list(images.should_delete()))
 200
 201     def test_nonexistent_user_removed(self):
 202         images = self.setup_images()
 203         images.end_user('nonexistent')
 204         # No exception should be raised.
 205
 206     def test_del_image_effective_with_users_present(self):
 207         images = self.setup_images(None, target_size=1)
 208         user = MockContainer(self.mock_images[0])
 209         images.add_user(user, 1)
 210         images.del_image(self.mock_images[0]['Id'])
 211         images.end_user(user['Id'])
 212         self.assertEqual([], list(images.should_delete()))
 213
 214     def setup_from_daemon(self, *vsizes, target_size=1500000):
 215         self.setup_mock_images(*vsizes)
 216         docker_client = mock.MagicMock(name='docker_client')
 217         docker_client.images.return_value = iter(self.mock_images)
 218         return cleaner.DockerImages.from_daemon(target_size, docker_client)
 219
 220     def test_images_loaded_from_daemon(self):
 221         images = self.setup_from_daemon(None, None)
 222         for image in self.mock_images:
 223             self.assertTrue(images.has_image(image['Id']))
 224
 225     def test_target_size_set_from_daemon(self):
 226         images = self.setup_from_daemon(20, 10, 5, target_size=15)
 227         user = MockContainer(self.mock_images[-1])
 228         images.add_user(user, 1)
 229         self.assertEqual([self.mock_images[0]['Id']],
 230                          list(images.should_delete()))
 231
 232
 233 class DockerImageUseRecorderTestCase(unittest.TestCase):
 234     TEST_CLASS = cleaner.DockerImageUseRecorder
 235     TEST_CLASS_INIT_KWARGS = {}
 236
 237     def setUp(self):
 238         self.images = mock.MagicMock(name='images')
 239         self.docker_client = mock.MagicMock(name='docker_client')
 240         self.events = []
 241         self.recorder = self.TEST_CLASS(self.images, self.docker_client,
 242                                         self.encoded_events, **self.TEST_CLASS_INIT_KWARGS)
 243
 244     @property
 245     def encoded_events(self):
 246         return (event.encoded() for event in self.events)
 247
 248     def test_unknown_events_ignored(self):
 249         self.events.append(MockEvent('mock!event'))
 250         self.recorder.run()
 251         # No exception should be raised.
 252
 253     def test_fetches_container_on_create(self):
 254         self.events.append(MockEvent('create'))
 255         self.recorder.run()
 256         self.docker_client.inspect_container.assert_called_with(
 257             self.events[0]['id'])
 258
 259     def test_adds_user_on_container_create(self):
 260         self.events.append(MockEvent('create'))
 261         self.recorder.run()
 262         self.images.add_user.assert_called_with(
 263             self.docker_client.inspect_container(), self.events[0]['time'])
 264
 265     def test_unknown_image_handling(self):
 266         # The use recorder should not fetch any images.
 267         self.events.append(MockEvent('create'))
 268         self.recorder.run()
 269         self.assertFalse(self.docker_client.inspect_image.called)
 270
 271     def test_unfetchable_containers_ignored(self):
 272         self.events.append(MockEvent('create'))
 273         self.docker_client.inspect_container.side_effect = MockException(404)
 274         self.recorder.run()
 275         self.assertFalse(self.images.add_user.called)
 276
 277     def test_ends_user_on_container_destroy(self):
 278         self.events.append(MockEvent('destroy'))
 279         self.recorder.run()
 280         self.images.end_user.assert_called_with(self.events[0]['id'])
 281
 282
 283 class DockerImageCleanerTestCase(DockerImageUseRecorderTestCase):
 284     TEST_CLASS = cleaner.DockerImageCleaner
 285
 286     def test_unknown_image_handling(self):
 287         # The image cleaner should fetch and record new images.
 288         self.images.has_image.return_value = False
 289         self.events.append(MockEvent('create'))
 290         self.recorder.run()
 291         self.docker_client.inspect_image.assert_called_with(
 292             self.docker_client.inspect_container()['Image'])
 293         self.images.add_image.assert_called_with(
 294             self.docker_client.inspect_image())
 295
 296     def test_unfetchable_images_ignored(self):
 297         self.images.has_image.return_value = False
 298         self.docker_client.inspect_image.side_effect = MockException(404)
 299         self.events.append(MockEvent('create'))
 300         self.recorder.run()
 301         self.docker_client.inspect_image.assert_called_with(
 302             self.docker_client.inspect_container()['Image'])
 303         self.assertFalse(self.images.add_image.called)
 304
 305     def test_deletions_after_destroy(self):
 306         delete_id = MockDockerId()
 307         self.images.should_delete.return_value = [delete_id]
 308         self.events.append(MockEvent('destroy'))
 309         self.recorder.run()
 310         self.docker_client.remove_image.assert_called_with(delete_id)
 311         self.images.del_image.assert_called_with(delete_id)
 312
 313     def test_failed_deletion_handling(self):
 314         delete_id = MockDockerId()
 315         self.images.should_delete.return_value = [delete_id]
 316         self.docker_client.remove_image.side_effect = MockException(500)
 317         self.events.append(MockEvent('destroy'))
 318         self.recorder.run()
 319         self.docker_client.remove_image.assert_called_with(delete_id)
 320         self.assertFalse(self.images.del_image.called)
 321
 322
 323 class DockerContainerCleanerTestCase(DockerImageUseRecorderTestCase):
 324     TEST_CLASS = cleaner.DockerImageCleaner
 325     TEST_CLASS_INIT_KWARGS = {'remove_containers_onexit': True}
 326
 327     def test_container_deletion_deletes_volumes(self):
 328         cid = MockDockerId()
 329         self.events.append(MockEvent('die', docker_id=cid))
 330         self.recorder.run()
 331         self.docker_client.remove_container.assert_called_with(cid, v=True)
 332
 333     @mock.patch('arvados_docker.cleaner.logger')
 334     def test_failed_container_deletion_handling(self, mockLogger):
 335         cid = MockDockerId()
 336         self.docker_client.remove_container.side_effect = MockException(500)
 337         self.events.append(MockEvent('die', docker_id=cid))
 338         self.recorder.run()
 339         self.docker_client.remove_container.assert_called_with(cid, v=True)
 340         self.assertEqual("Failed to remove container %s: %s",
 341                          mockLogger.warning.call_args[0][0])
 342         self.assertEqual(cid,
 343                          mockLogger.warning.call_args[0][1])
 344
 345
 346 class HumanSizeTestCase(unittest.TestCase):
 347
 348     def check(self, human_str, count, exp):
 349         self.assertEqual(count * (1024 ** exp),
 350                          cleaner.human_size(human_str))
 351
 352     def test_bytes(self):
 353         self.check('1', 1, 0)
 354         self.check('82', 82, 0)
 355
 356     def test_kibibytes(self):
 357         self.check('2K', 2, 1)
 358         self.check('3k', 3, 1)
 359
 360     def test_mebibytes(self):
 361         self.check('4M', 4, 2)
 362         self.check('5m', 5, 2)
 363
 364     def test_gibibytes(self):
 365         self.check('6G', 6, 3)
 366         self.check('7g', 7, 3)
 367
 368     def test_tebibytes(self):
 369         self.check('8T', 8, 4)
 370         self.check('9t', 9, 4)
 371
 372
 373 class RunTestCase(unittest.TestCase):
 374
 375     def setUp(self):
 376         self.config = cleaner.default_config()
 377         self.config['Quota'] = 1000000
 378         self.docker_client = mock.MagicMock(name='docker_client')
 379
 380     def test_run(self):
 381         test_start_time = int(time.time())
 382         self.docker_client.events.return_value = []
 383         cleaner.run(self.config, self.docker_client)
 384         self.assertEqual(2, self.docker_client.events.call_count)
 385         event_kwargs = [args[1] for args in
 386                         self.docker_client.events.call_args_list]
 387         self.assertIn('since', event_kwargs[0])
 388         self.assertIn('until', event_kwargs[0])
 389         self.assertLessEqual(test_start_time, event_kwargs[0]['until'])
 390         self.assertIn('since', event_kwargs[1])
 391         self.assertEqual(event_kwargs[0]['until'], event_kwargs[1]['since'])
 392
 393
 394 @mock.patch('docker.Client', name='docker_client')
 395 @mock.patch('arvados_docker.cleaner.run', name='cleaner_run')
 396 class MainTestCase(unittest.TestCase):
 397
 398     def test_client_api_version(self, run_mock, docker_client):
 399         with tempfile.NamedTemporaryFile(mode='wt') as cf:
 400             cf.write('{"Quota":"1000T"}')
 401             cf.flush()
 402             cleaner.main(['--config', cf.name])
 403         self.assertEqual(1, docker_client.call_count)
 404         # 1.14 is the first version that's well defined, going back to
 405         # Docker 1.2, and still supported up to at least Docker 1.9.
 406         # See
 407         # <https://docs.docker.com/engine/reference/api/docker_remote_api/>.
 408         self.assertEqual('1.14',
 409                          docker_client.call_args[1].get('version'))
 410         self.assertEqual(1, run_mock.call_count)
 411         self.assertIs(run_mock.call_args[0][1], docker_client())
 412
 413
 414 class ConfigTestCase(unittest.TestCase):
 415
 416     def test_load_config(self):
 417         with tempfile.NamedTemporaryFile(mode='wt') as cf:
 418             cf.write(
 419                 '{"Quota":"1000T", "RemoveStoppedContainers":"always", "Verbose":2}')
 420             cf.flush()
 421             config = cleaner.load_config(['--config', cf.name])
 422         self.assertEqual(1000 << 40, config['Quota'])
 423         self.assertEqual("always", config['RemoveStoppedContainers'])
 424         self.assertEqual(2, config['Verbose'])
 425
 426     def test_args_override_config(self):
 427         with tempfile.NamedTemporaryFile(mode='wt') as cf:
 428             cf.write(
 429                 '{"Quota":"1000T", "RemoveStoppedContainers":"always", "Verbose":2}')
 430             cf.flush()
 431             config = cleaner.load_config([
 432                 '--config', cf.name,
 433                 '--quota', '1G',
 434                 '--remove-stopped-containers', 'never',
 435                 '--verbose',
 436             ])
 437         self.assertEqual(1 << 30, config['Quota'])
 438         self.assertEqual('never', config['RemoveStoppedContainers'])
 439         self.assertEqual(1, config['Verbose'])
 440
 441     def test_args_no_config(self):
 442         self.assertEqual(False, os.path.exists(cleaner.DEFAULT_CONFIG_FILE))
 443         config = cleaner.load_config(['--quota', '1G'])
 444         self.assertEqual(1 << 30, config['Quota'])
 445
 446
 447 class ContainerRemovalTestCase(unittest.TestCase):
 448     LIFECYCLE = ['create', 'attach', 'start', 'resize', 'die', 'destroy']
 449
 450     def setUp(self):
 451         self.config = cleaner.default_config()
 452         self.docker_client = mock.MagicMock(name='docker_client')
 453         self.existingCID = MockDockerId()
 454         self.docker_client.containers.return_value = [{
 455             'Id': self.existingCID,
 456             'Status': 'Exited (0) 6 weeks ago',
 457         }, {
 458             # If docker_client.containers() returns non-exited
 459             # containers for some reason, do not remove them.
 460             'Id': MockDockerId(),
 461             'Status': 'Running',
 462         }]
 463         self.newCID = MockDockerId()
 464         self.docker_client.events.return_value = [
 465             MockEvent(e, docker_id=self.newCID).encoded()
 466             for e in self.LIFECYCLE]
 467
 468     def test_remove_onexit(self):
 469         self.config['RemoveStoppedContainers'] = 'onexit'
 470         cleaner.run(self.config, self.docker_client)
 471         self.docker_client.remove_container.assert_called_once_with(
 472             self.newCID, v=True)
 473
 474     def test_remove_always(self):
 475         self.config['RemoveStoppedContainers'] = 'always'
 476         cleaner.run(self.config, self.docker_client)
 477         self.docker_client.remove_container.assert_any_call(
 478             self.existingCID, v=True)
 479         self.docker_client.remove_container.assert_any_call(
 480             self.newCID, v=True)
 481         self.assertEqual(2, self.docker_client.remove_container.call_count)
 482
 483     def test_remove_never(self):
 484         self.config['RemoveStoppedContainers'] = 'never'
 485         cleaner.run(self.config, self.docker_client)
 486         self.assertEqual(0, self.docker_client.remove_container.call_count)
 487
 488     def test_container_exited_between_subscribe_events_and_check_existing(self):
 489         self.config['RemoveStoppedContainers'] = 'always'
 490         self.docker_client.events.return_value = [
 491             MockEvent(e, docker_id=self.existingCID).encoded()
 492             for e in ['die', 'destroy']]
 493         cleaner.run(self.config, self.docker_client)
 494         # Subscribed to events before getting the list of existing
 495         # exited containers?
 496         self.docker_client.assert_has_calls([
 497             mock.call.events(since=mock.ANY),
 498             mock.call.containers(filters={'status': 'exited'})])
 499         # Asked to delete the container twice?
 500         self.docker_client.remove_container.assert_has_calls(
 501             [mock.call(self.existingCID, v=True)] * 2)
 502         self.assertEqual(2, self.docker_client.remove_container.call_count)