#!/usr/bin/env python3
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
import collections
import itertools
import json
+import os
import random
+import tempfile
import time
import unittest
import docker
-import mock
+from unittest import mock
from arvados_docker import cleaner
MAX_DOCKER_ID = (16 ** 64) - 1
+
def MockDockerId():
return '{:064x}'.format(random.randint(0, MAX_DOCKER_ID))
+
def MockContainer(image_hash):
return {'Id': MockDockerId(),
'Image': image_hash['Id']}
+
def MockImage(*, size=0, vsize=None, tags=[]):
if vsize is None:
vsize = random.randint(100, 2000000)
'Size': size,
'VirtualSize': vsize}
+
class MockEvent(dict):
ENCODING = 'utf-8'
event_seq = itertools.count(1)
class MockException(docker.errors.APIError):
+
def __init__(self, status_code):
response = mock.Mock(name='response')
response.status_code = status_code
class DockerImageTestCase(unittest.TestCase):
+
def test_used_at_sets_last_used(self):
image = cleaner.DockerImage(MockImage())
image.used_at(5)
class DockerImagesTestCase(unittest.TestCase):
+
def setUp(self):
self.mock_images = []
class DockerImageUseRecorderTestCase(unittest.TestCase):
TEST_CLASS = cleaner.DockerImageUseRecorder
+ TEST_CLASS_INIT_KWARGS = {}
def setUp(self):
self.images = mock.MagicMock(name='images')
self.docker_client = mock.MagicMock(name='docker_client')
self.events = []
self.recorder = self.TEST_CLASS(self.images, self.docker_client,
- self.encoded_events)
+ self.encoded_events, **self.TEST_CLASS_INIT_KWARGS)
@property
def encoded_events(self):
self.assertFalse(self.images.del_image.called)
+class DockerContainerCleanerTestCase(DockerImageUseRecorderTestCase):
+ TEST_CLASS = cleaner.DockerImageCleaner
+ TEST_CLASS_INIT_KWARGS = {'remove_containers_onexit': True}
+
+ def test_container_deletion_deletes_volumes(self):
+ cid = MockDockerId()
+ self.events.append(MockEvent('die', docker_id=cid))
+ self.recorder.run()
+ self.docker_client.remove_container.assert_called_with(cid, v=True)
+
+ @mock.patch('arvados_docker.cleaner.logger')
+ def test_failed_container_deletion_handling(self, mockLogger):
+ cid = MockDockerId()
+ self.docker_client.remove_container.side_effect = MockException(500)
+ self.events.append(MockEvent('die', docker_id=cid))
+ self.recorder.run()
+ self.docker_client.remove_container.assert_called_with(cid, v=True)
+ self.assertEqual("Failed to remove container %s: %s",
+ mockLogger.warning.call_args[0][0])
+ self.assertEqual(cid,
+ mockLogger.warning.call_args[0][1])
+
+
class HumanSizeTestCase(unittest.TestCase):
+
def check(self, human_str, count, exp):
self.assertEqual(count * (1024 ** exp),
cleaner.human_size(human_str))
class RunTestCase(unittest.TestCase):
+
def setUp(self):
- self.args = mock.MagicMock(name='args')
- self.args.quota = 1000000
+ self.config = cleaner.default_config()
+ self.config['Quota'] = 1000000
self.docker_client = mock.MagicMock(name='docker_client')
def test_run(self):
test_start_time = int(time.time())
self.docker_client.events.return_value = []
- cleaner.run(self.args, self.docker_client)
+ cleaner.run(self.config, self.docker_client)
self.assertEqual(2, self.docker_client.events.call_count)
event_kwargs = [args[1] for args in
self.docker_client.events.call_args_list]
self.assertLessEqual(test_start_time, event_kwargs[0]['until'])
self.assertIn('since', event_kwargs[1])
self.assertEqual(event_kwargs[0]['until'], event_kwargs[1]['since'])
+
+
+@mock.patch('docker.APIClient', name='docker_client')
+@mock.patch('arvados_docker.cleaner.run', name='cleaner_run')
+class MainTestCase(unittest.TestCase):
+
+ def test_client_api_version(self, run_mock, docker_client):
+ with tempfile.NamedTemporaryFile(mode='wt') as cf:
+ cf.write('{"Quota":"1000T"}')
+ cf.flush()
+ cleaner.main(['--config', cf.name])
+ self.assertEqual(1, docker_client.call_count)
+ # We are standardized on Docker API version 1.35.
+ # See DockerAPIVersion in lib/crunchrun/docker.go.
+ self.assertEqual('1.35',
+ docker_client.call_args[1].get('version'))
+ self.assertEqual(1, run_mock.call_count)
+ self.assertIs(run_mock.call_args[0][1], docker_client())
+
+
+class ConfigTestCase(unittest.TestCase):
+
+ def test_load_config(self):
+ with tempfile.NamedTemporaryFile(mode='wt') as cf:
+ cf.write(
+ '{"Quota":"1000T", "RemoveStoppedContainers":"always", "Verbose":2}')
+ cf.flush()
+ config = cleaner.load_config(['--config', cf.name])
+ self.assertEqual(1000 << 40, config['Quota'])
+ self.assertEqual("always", config['RemoveStoppedContainers'])
+ self.assertEqual(2, config['Verbose'])
+
+ def test_args_override_config(self):
+ with tempfile.NamedTemporaryFile(mode='wt') as cf:
+ cf.write(
+ '{"Quota":"1000T", "RemoveStoppedContainers":"always", "Verbose":2}')
+ cf.flush()
+ config = cleaner.load_config([
+ '--config', cf.name,
+ '--quota', '1G',
+ '--remove-stopped-containers', 'never',
+ '--verbose',
+ ])
+ self.assertEqual(1 << 30, config['Quota'])
+ self.assertEqual('never', config['RemoveStoppedContainers'])
+ self.assertEqual(1, config['Verbose'])
+
+ def test_args_no_config(self):
+ self.assertEqual(False, os.path.exists(cleaner.DEFAULT_CONFIG_FILE))
+ config = cleaner.load_config(['--quota', '1G'])
+ self.assertEqual(1 << 30, config['Quota'])
+
+
+class ContainerRemovalTestCase(unittest.TestCase):
+ LIFECYCLE = ['create', 'attach', 'start', 'resize', 'die', 'destroy']
+
+ def setUp(self):
+ self.config = cleaner.default_config()
+ self.docker_client = mock.MagicMock(name='docker_client')
+ self.existingCID = MockDockerId()
+ self.docker_client.containers.return_value = [{
+ 'Id': self.existingCID,
+ 'Status': 'Exited (0) 6 weeks ago',
+ }, {
+ # If docker_client.containers() returns non-exited
+ # containers for some reason, do not remove them.
+ 'Id': MockDockerId(),
+ 'Status': 'Running',
+ }]
+ self.newCID = MockDockerId()
+ self.docker_client.events.return_value = [
+ MockEvent(e, docker_id=self.newCID).encoded()
+ for e in self.LIFECYCLE]
+
+ def test_remove_onexit(self):
+ self.config['RemoveStoppedContainers'] = 'onexit'
+ cleaner.run(self.config, self.docker_client)
+ self.docker_client.remove_container.assert_called_once_with(
+ self.newCID, v=True)
+
+ def test_remove_always(self):
+ self.config['RemoveStoppedContainers'] = 'always'
+ cleaner.run(self.config, self.docker_client)
+ self.docker_client.remove_container.assert_any_call(
+ self.existingCID, v=True)
+ self.docker_client.remove_container.assert_any_call(
+ self.newCID, v=True)
+ self.assertEqual(2, self.docker_client.remove_container.call_count)
+
+ def test_remove_never(self):
+ self.config['RemoveStoppedContainers'] = 'never'
+ cleaner.run(self.config, self.docker_client)
+ self.assertEqual(0, self.docker_client.remove_container.call_count)
+
+ def test_container_exited_between_subscribe_events_and_check_existing(self):
+ self.config['RemoveStoppedContainers'] = 'always'
+ self.docker_client.events.return_value = [
+ MockEvent(e, docker_id=self.existingCID).encoded()
+ for e in ['die', 'destroy']]
+ cleaner.run(self.config, self.docker_client)
+ # Subscribed to events before getting the list of existing
+ # exited containers?
+ self.docker_client.assert_has_calls([
+ mock.call.events(since=mock.ANY),
+ mock.call.containers(filters={'status': 'exited'})])
+ # Asked to delete the container twice?
+ self.docker_client.remove_container.assert_has_calls(
+ [mock.call(self.existingCID, v=True)] * 2)
+ self.assertEqual(2, self.docker_client.remove_container.call_count)
projects / arvados.git / blobdiff