X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/4d3b8d2eff10995295b12fdc39aa60af95ae7ef7..0eb72b526bf8bbb011551ecf019f604e17a534f1:/services/dockercleaner/tests/test_cleaner.py diff --git a/services/dockercleaner/tests/test_cleaner.py b/services/dockercleaner/tests/test_cleaner.py index fd959de776..7580b0128a 100644 --- a/services/dockercleaner/tests/test_cleaner.py +++ b/services/dockercleaner/tests/test_cleaner.py @@ -1,9 +1,14 @@ #!/usr/bin/env python3 +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 import collections import itertools import json +import os import random +import tempfile import time import unittest @@ -14,13 +19,16 @@ from arvados_docker import cleaner MAX_DOCKER_ID = (16 ** 64) - 1 + def MockDockerId(): return '{:064x}'.format(random.randint(0, MAX_DOCKER_ID)) + def MockContainer(image_hash): return {'Id': MockDockerId(), 'Image': image_hash['Id']} + def MockImage(*, size=0, vsize=None, tags=[]): if vsize is None: vsize = random.randint(100, 2000000) @@ -30,6 +38,7 @@ def MockImage(*, size=0, vsize=None, tags=[]): 'Size': size, 'VirtualSize': vsize} + class MockEvent(dict): ENCODING = 'utf-8' event_seq = itertools.count(1) @@ -47,6 +56,7 @@ class MockEvent(dict): class MockException(docker.errors.APIError): + def __init__(self, status_code): response = mock.Mock(name='response') response.status_code = status_code @@ -54,6 +64,7 @@ class MockException(docker.errors.APIError): class DockerImageTestCase(unittest.TestCase): + def test_used_at_sets_last_used(self): image = cleaner.DockerImage(MockImage()) image.used_at(5) @@ -73,6 +84,7 @@ class DockerImageTestCase(unittest.TestCase): class DockerImagesTestCase(unittest.TestCase): + def setUp(self): self.mock_images = [] @@ -223,13 +235,14 @@ class DockerImagesTestCase(unittest.TestCase): class DockerImageUseRecorderTestCase(unittest.TestCase): TEST_CLASS = cleaner.DockerImageUseRecorder + TEST_CLASS_INIT_KWARGS = {} def setUp(self): self.images = mock.MagicMock(name='images') self.docker_client = mock.MagicMock(name='docker_client') self.events = [] self.recorder = self.TEST_CLASS(self.images, self.docker_client, - self.encoded_events) + self.encoded_events, **self.TEST_CLASS_INIT_KWARGS) @property def encoded_events(self): @@ -310,7 +323,31 @@ class DockerImageCleanerTestCase(DockerImageUseRecorderTestCase): self.assertFalse(self.images.del_image.called) +class DockerContainerCleanerTestCase(DockerImageUseRecorderTestCase): + TEST_CLASS = cleaner.DockerImageCleaner + TEST_CLASS_INIT_KWARGS = {'remove_containers_onexit': True} + + def test_container_deletion_deletes_volumes(self): + cid = MockDockerId() + self.events.append(MockEvent('die', docker_id=cid)) + self.recorder.run() + self.docker_client.remove_container.assert_called_with(cid, v=True) + + @mock.patch('arvados_docker.cleaner.logger') + def test_failed_container_deletion_handling(self, mockLogger): + cid = MockDockerId() + self.docker_client.remove_container.side_effect = MockException(500) + self.events.append(MockEvent('die', docker_id=cid)) + self.recorder.run() + self.docker_client.remove_container.assert_called_with(cid, v=True) + self.assertEqual("Failed to remove container %s: %s", + mockLogger.warning.call_args[0][0]) + self.assertEqual(cid, + mockLogger.warning.call_args[0][1]) + + class HumanSizeTestCase(unittest.TestCase): + def check(self, human_str, count, exp): self.assertEqual(count * (1024 ** exp), cleaner.human_size(human_str)) @@ -337,15 +374,16 @@ class HumanSizeTestCase(unittest.TestCase): class RunTestCase(unittest.TestCase): + def setUp(self): - self.args = mock.MagicMock(name='args') - self.args.quota = 1000000 + self.config = cleaner.default_config() + self.config['Quota'] = 1000000 self.docker_client = mock.MagicMock(name='docker_client') def test_run(self): test_start_time = int(time.time()) self.docker_client.events.return_value = [] - cleaner.run(self.args, self.docker_client) + cleaner.run(self.config, self.docker_client) self.assertEqual(2, self.docker_client.events.call_count) event_kwargs = [args[1] for args in self.docker_client.events.call_args_list] @@ -354,3 +392,114 @@ class RunTestCase(unittest.TestCase): self.assertLessEqual(test_start_time, event_kwargs[0]['until']) self.assertIn('since', event_kwargs[1]) self.assertEqual(event_kwargs[0]['until'], event_kwargs[1]['since']) + + +@mock.patch('docker.Client', name='docker_client') +@mock.patch('arvados_docker.cleaner.run', name='cleaner_run') +class MainTestCase(unittest.TestCase): + + def test_client_api_version(self, run_mock, docker_client): + with tempfile.NamedTemporaryFile(mode='wt') as cf: + cf.write('{"Quota":"1000T"}') + cf.flush() + cleaner.main(['--config', cf.name]) + self.assertEqual(1, docker_client.call_count) + # 1.14 is the first version that's well defined, going back to + # Docker 1.2, and still supported up to at least Docker 1.9. + # See + # . + self.assertEqual('1.14', + docker_client.call_args[1].get('version')) + self.assertEqual(1, run_mock.call_count) + self.assertIs(run_mock.call_args[0][1], docker_client()) + + +class ConfigTestCase(unittest.TestCase): + + def test_load_config(self): + with tempfile.NamedTemporaryFile(mode='wt') as cf: + cf.write( + '{"Quota":"1000T", "RemoveStoppedContainers":"always", "Verbose":2}') + cf.flush() + config = cleaner.load_config(['--config', cf.name]) + self.assertEqual(1000 << 40, config['Quota']) + self.assertEqual("always", config['RemoveStoppedContainers']) + self.assertEqual(2, config['Verbose']) + + def test_args_override_config(self): + with tempfile.NamedTemporaryFile(mode='wt') as cf: + cf.write( + '{"Quota":"1000T", "RemoveStoppedContainers":"always", "Verbose":2}') + cf.flush() + config = cleaner.load_config([ + '--config', cf.name, + '--quota', '1G', + '--remove-stopped-containers', 'never', + '--verbose', + ]) + self.assertEqual(1 << 30, config['Quota']) + self.assertEqual('never', config['RemoveStoppedContainers']) + self.assertEqual(1, config['Verbose']) + + def test_args_no_config(self): + self.assertEqual(False, os.path.exists(cleaner.DEFAULT_CONFIG_FILE)) + config = cleaner.load_config(['--quota', '1G']) + self.assertEqual(1 << 30, config['Quota']) + + +class ContainerRemovalTestCase(unittest.TestCase): + LIFECYCLE = ['create', 'attach', 'start', 'resize', 'die', 'destroy'] + + def setUp(self): + self.config = cleaner.default_config() + self.docker_client = mock.MagicMock(name='docker_client') + self.existingCID = MockDockerId() + self.docker_client.containers.return_value = [{ + 'Id': self.existingCID, + 'Status': 'Exited (0) 6 weeks ago', + }, { + # If docker_client.containers() returns non-exited + # containers for some reason, do not remove them. + 'Id': MockDockerId(), + 'Status': 'Running', + }] + self.newCID = MockDockerId() + self.docker_client.events.return_value = [ + MockEvent(e, docker_id=self.newCID).encoded() + for e in self.LIFECYCLE] + + def test_remove_onexit(self): + self.config['RemoveStoppedContainers'] = 'onexit' + cleaner.run(self.config, self.docker_client) + self.docker_client.remove_container.assert_called_once_with( + self.newCID, v=True) + + def test_remove_always(self): + self.config['RemoveStoppedContainers'] = 'always' + cleaner.run(self.config, self.docker_client) + self.docker_client.remove_container.assert_any_call( + self.existingCID, v=True) + self.docker_client.remove_container.assert_any_call( + self.newCID, v=True) + self.assertEqual(2, self.docker_client.remove_container.call_count) + + def test_remove_never(self): + self.config['RemoveStoppedContainers'] = 'never' + cleaner.run(self.config, self.docker_client) + self.assertEqual(0, self.docker_client.remove_container.call_count) + + def test_container_exited_between_subscribe_events_and_check_existing(self): + self.config['RemoveStoppedContainers'] = 'always' + self.docker_client.events.return_value = [ + MockEvent(e, docker_id=self.existingCID).encoded() + for e in ['die', 'destroy']] + cleaner.run(self.config, self.docker_client) + # Subscribed to events before getting the list of existing + # exited containers? + self.docker_client.assert_has_calls([ + mock.call.events(since=mock.ANY), + mock.call.containers(filters={'status': 'exited'})]) + # Asked to delete the container twice? + self.docker_client.remove_container.assert_has_calls( + [mock.call(self.existingCID, v=True)] * 2) + self.assertEqual(2, self.docker_client.remove_container.call_count)