The arvados-docker-cleaner program removes least recently used docker images as needed to keep disk usage below a configured limit.
{% include 'notebox_begin' %}
-This also removes all containers as soon as they exit, as if they were run with `docker run --rm`. If you need to debug or inspect containers after they stop, temporarily stop arvados-docker-cleaner or run it with the `--no-remove-stopped-containers` flag.
+This also removes all containers as soon as they exit, as if they were run with `docker run --rm`. If you need to debug or inspect containers after they stop, temporarily stop arvados-docker-cleaner or run it with `--remove-stopped-containers never`.
{% include 'notebox_end' %}
On Debian-based systems, install runit:
self.images.add_image(image_hash)
return super().new_container(event, container_hash)
- @event_handlers.on('die')
- def clean_container(self, event=None):
- if not self.remove_stopped_containers:
- return
- cid = event['id']
+ def _remove_container(self, cid):
try:
self.docker_client.remove_container(cid)
except docker.errors.APIError as error:
else:
logger.info("Removed container %s", cid)
+ @event_handlers.on('die')
+ def clean_container(self, event=None):
+ if not self.remove_stopped_containers:
+ return
+ self._remove_container(event['id'])
+
+ def check_stopped_containers(self, remove=False):
+ logger.info("Checking for stopped containers")
+ for c in self.docker_client.containers(filters={'status': 'exited'}):
+ logger.info("Container %s %s", c['Id'], c['Status'])
+ if c['Status'][:6] != 'Exited':
+ logger.error("Unexpected status %s for container %s",
+ c['Status'], c['Id'])
+ elif remove:
+ self._remove_container(c['Id'])
+
@event_handlers.on('destroy')
def clean_images(self, event=None):
for image_id in self.images.should_delete():
'--quota', action='store', type=human_size, required=True,
help="space allowance for Docker images, suffixed with K/M/G/T")
parser.add_argument(
- '--no-remove-stopped-containers', action='store_false', default=True,
- dest='remove_stopped_containers',
- help="do not remove containers (default: remove on exit)")
+ '--remove-stopped-containers', type=str, default='always',
+ choices=['never', 'onexit', 'always'],
+ help="""when to remove stopped containers (default: always, i.e., remove
+ stopped containers found at startup, and remove containers as
+ soon as they exit)""")
parser.add_argument(
'--verbose', '-v', action='count', default=0,
help="log more information")
use_recorder.run()
cleaner = DockerImageCleaner(
images, docker_client, docker_client.events(since=start_time),
- remove_stopped_containers=args.remove_stopped_containers)
- logger.info("Starting cleanup loop")
+ remove_stopped_containers=args.remove_stopped_containers != 'never')
+ cleaner.check_stopped_containers(
+ remove=args.remove_stopped_containers == 'always')
+ logger.info("Checking image quota at startup")
cleaner.clean_images()
+ logger.info("Listening for docker events")
cleaner.run()
def main(arguments):
class ContainerRemovalTestCase(unittest.TestCase):
+ LIFECYCLE = ['create', 'attach', 'start', 'resize', 'die', 'destroy']
+
def setUp(self):
self.args = mock.MagicMock(name='args')
self.docker_client = mock.MagicMock(name='docker_client')
-
- def test_remove_on_die(self):
- mockID = MockDockerId()
+ self.existingCID = MockDockerId()
+ self.docker_client.containers.return_value = [{
+ 'Id': self.existingCID,
+ 'Status': 'Exited (0) 6 weeks ago',
+ }, {
+ # If docker_client.containers() returns non-exited
+ # containers for some reason, do not remove them.
+ 'Id': MockDockerId(),
+ 'Status': 'Running',
+ }]
+ self.newCID = MockDockerId()
self.docker_client.events.return_value = [
- MockEvent(x, docker_id=mockID).encoded()
- for x in ['create', 'attach', 'start', 'resize', 'die', 'destroy']]
+ MockEvent(e, docker_id=self.newCID).encoded()
+ for e in self.LIFECYCLE]
+
+ def test_remove_onexit(self):
+ self.args.remove_stopped_containers = 'onexit'
+ cleaner.run(self.args, self.docker_client)
+ self.docker_client.remove_container.assert_called_once_with(self.newCID)
+
+ def test_remove_always(self):
+ self.args.remove_stopped_containers = 'always'
cleaner.run(self.args, self.docker_client)
- self.docker_client.remove_container.assert_called_once_with(mockID)
+ self.docker_client.remove_container.assert_any_call(self.existingCID)
+ self.docker_client.remove_container.assert_any_call(self.newCID)
+ self.assertEqual(2, self.docker_client.remove_container.call_count)
- def test_disabled_flag(self):
- self.args.remove_stopped_containers = False
- self.docker_client.events.return_value = [MockEvent('die').encoded()]
+ def test_remove_never(self):
+ self.args.remove_stopped_containers = 'never'
cleaner.run(self.args, self.docker_client)
self.assertEqual(0, self.docker_client.remove_container.call_count)
+
+ def test_container_exited_between_subscribe_events_and_check_existing(self):
+ self.args.remove_stopped_containers = 'always'
+ self.docker_client.events.return_value = [
+ MockEvent(e, docker_id=self.existingCID).encoded()
+ for e in ['die', 'destroy']]
+ cleaner.run(self.args, self.docker_client)
+ # Subscribed to events before getting the list of existing
+ # exited containers?
+ self.docker_client.assert_has_calls([
+ mock.call.events(since=mock.ANY),
+ mock.call.containers(filters={'status':'exited'})])
+ # Asked to delete the container twice?
+ self.docker_client.remove_container.assert_has_calls([mock.call(self.existingCID)] * 2)
+ self.assertEqual(2, self.docker_client.remove_container.call_count)
projects / arvados.git / commitdiff