15 from arvados_docker import cleaner
17 MAX_DOCKER_ID = (16 ** 64) - 1
21 return '{:064x}'.format(random.randint(0, MAX_DOCKER_ID))
24 def MockContainer(image_hash):
25 return {'Id': MockDockerId(),
26 'Image': image_hash['Id']}
29 def MockImage(*, size=0, vsize=None, tags=[]):
31 vsize = random.randint(100, 2000000)
32 return {'Id': MockDockerId(),
33 'ParentId': MockDockerId(),
34 'RepoTags': list(tags),
39 class MockEvent(dict):
41 event_seq = itertools.count(1)
43 def __init__(self, status, docker_id=None, **event_data):
45 docker_id = MockDockerId()
46 super().__init__(self, **event_data)
47 self['status'] = status
48 self['id'] = docker_id
49 self.setdefault('time', next(self.event_seq))
52 return json.dumps(self).encode(self.ENCODING)
55 class MockException(docker.errors.APIError):
57 def __init__(self, status_code):
58 response = mock.Mock(name='response')
59 response.status_code = status_code
60 super().__init__("mock exception", response)
63 class DockerImageTestCase(unittest.TestCase):
65 def test_used_at_sets_last_used(self):
66 image = cleaner.DockerImage(MockImage())
68 self.assertEqual(5, image.last_used)
70 def test_used_at_moves_forward(self):
71 image = cleaner.DockerImage(MockImage())
74 self.assertEqual(8, image.last_used)
76 def test_used_at_does_not_go_backward(self):
77 image = cleaner.DockerImage(MockImage())
80 self.assertEqual(4, image.last_used)
83 class DockerImagesTestCase(unittest.TestCase):
88 def setup_mock_images(self, *vsizes):
89 self.mock_images.extend(MockImage(vsize=vsize) for vsize in vsizes)
91 def setup_images(self, *vsizes, target_size=1000000):
92 self.setup_mock_images(*vsizes)
93 images = cleaner.DockerImages(target_size)
94 for image in self.mock_images:
95 images.add_image(image)
98 def test_has_image(self):
99 images = self.setup_images(None)
100 self.assertTrue(images.has_image(self.mock_images[0]['Id']))
101 self.assertFalse(images.has_image(MockDockerId()))
103 def test_del_image(self):
104 images = self.setup_images(None)
105 images.del_image(self.mock_images[0]['Id'])
106 self.assertFalse(images.has_image(self.mock_images[0]['Id']))
108 def test_del_nonexistent_image(self):
109 images = self.setup_images(None)
110 images.del_image(MockDockerId())
111 self.assertTrue(images.has_image(self.mock_images[0]['Id']))
113 def test_one_image_always_kept(self):
114 # When crunch-job starts a job, it makes sure each compute node
115 # has the Docker image loaded, then it runs all the tasks with
116 # the assumption the image is on each node. As long as that's
117 # true, the cleaner should avoid removing every installed image:
118 # crunch-job might be counting on the most recent one to be
119 # available, even if it's not currently in use.
120 images = self.setup_images(None, None, target_size=1)
121 for use_time, image in enumerate(self.mock_images, 1):
122 user = MockContainer(image)
123 images.add_user(user, use_time)
124 images.end_user(user['Id'])
125 self.assertEqual([self.mock_images[0]['Id']],
126 list(images.should_delete()))
128 def test_images_under_target_not_deletable(self):
129 # The images are used in this order. target_size is set so it
130 # could hold the largest image, but not after the most recently
131 # used image is kept; then we have to fall back to the previous one.
132 images = self.setup_images(20, 30, 40, 10, target_size=45)
133 for use_time, image in enumerate(self.mock_images, 1):
134 user = MockContainer(image)
135 images.add_user(user, use_time)
136 images.end_user(user['Id'])
137 self.assertEqual([self.mock_images[ii]['Id'] for ii in [0, 2]],
138 list(images.should_delete()))
140 def test_images_in_use_not_deletable(self):
141 images = self.setup_images(None, None, target_size=1)
142 users = [MockContainer(image) for image in self.mock_images]
143 images.add_user(users[0], 1)
144 images.add_user(users[1], 2)
145 images.end_user(users[1]['Id'])
146 self.assertEqual([self.mock_images[1]['Id']],
147 list(images.should_delete()))
149 def test_image_deletable_after_unused(self):
150 images = self.setup_images(None, None, target_size=1)
151 users = [MockContainer(image) for image in self.mock_images]
152 images.add_user(users[0], 1)
153 images.add_user(users[1], 2)
154 images.end_user(users[0]['Id'])
155 self.assertEqual([self.mock_images[0]['Id']],
156 list(images.should_delete()))
158 def test_image_not_deletable_if_user_restarts(self):
159 images = self.setup_images(None, target_size=1)
160 user = MockContainer(self.mock_images[-1])
161 images.add_user(user, 1)
162 images.end_user(user['Id'])
163 images.add_user(user, 2)
164 self.assertEqual([], list(images.should_delete()))
166 def test_image_not_deletable_if_any_user_remains(self):
167 images = self.setup_images(None, target_size=1)
168 users = [MockContainer(self.mock_images[0]) for ii in range(2)]
169 images.add_user(users[0], 1)
170 images.add_user(users[1], 2)
171 images.end_user(users[0]['Id'])
172 self.assertEqual([], list(images.should_delete()))
174 def test_image_deletable_after_all_users_end(self):
175 images = self.setup_images(None, None, target_size=1)
176 users = [MockContainer(self.mock_images[ii]) for ii in [0, 1, 1]]
177 images.add_user(users[0], 1)
178 images.add_user(users[1], 2)
179 images.add_user(users[2], 3)
180 images.end_user(users[1]['Id'])
181 images.end_user(users[2]['Id'])
182 self.assertEqual([self.mock_images[-1]['Id']],
183 list(images.should_delete()))
185 def test_images_suggested_for_deletion_by_lru(self):
186 images = self.setup_images(10, 10, 10, target_size=1)
187 users = [MockContainer(image) for image in self.mock_images]
188 images.add_user(users[0], 3)
189 images.add_user(users[1], 1)
190 images.add_user(users[2], 2)
192 images.end_user(user['Id'])
193 self.assertEqual([self.mock_images[ii]['Id'] for ii in [1, 2]],
194 list(images.should_delete()))
196 def test_adding_user_without_image_does_not_implicitly_add_image(self):
197 images = self.setup_images(10)
198 images.add_user(MockContainer(MockImage()), 1)
199 self.assertEqual([], list(images.should_delete()))
201 def test_nonexistent_user_removed(self):
202 images = self.setup_images()
203 images.end_user('nonexistent')
204 # No exception should be raised.
206 def test_del_image_effective_with_users_present(self):
207 images = self.setup_images(None, target_size=1)
208 user = MockContainer(self.mock_images[0])
209 images.add_user(user, 1)
210 images.del_image(self.mock_images[0]['Id'])
211 images.end_user(user['Id'])
212 self.assertEqual([], list(images.should_delete()))
214 def setup_from_daemon(self, *vsizes, target_size=1500000):
215 self.setup_mock_images(*vsizes)
216 docker_client = mock.MagicMock(name='docker_client')
217 docker_client.images.return_value = iter(self.mock_images)
218 return cleaner.DockerImages.from_daemon(target_size, docker_client)
220 def test_images_loaded_from_daemon(self):
221 images = self.setup_from_daemon(None, None)
222 for image in self.mock_images:
223 self.assertTrue(images.has_image(image['Id']))
225 def test_target_size_set_from_daemon(self):
226 images = self.setup_from_daemon(20, 10, 5, target_size=15)
227 user = MockContainer(self.mock_images[-1])
228 images.add_user(user, 1)
229 self.assertEqual([self.mock_images[0]['Id']],
230 list(images.should_delete()))
233 class DockerImageUseRecorderTestCase(unittest.TestCase):
234 TEST_CLASS = cleaner.DockerImageUseRecorder
235 TEST_CLASS_INIT_KWARGS = {}
238 self.images = mock.MagicMock(name='images')
239 self.docker_client = mock.MagicMock(name='docker_client')
241 self.recorder = self.TEST_CLASS(self.images, self.docker_client,
242 self.encoded_events, **self.TEST_CLASS_INIT_KWARGS)
245 def encoded_events(self):
246 return (event.encoded() for event in self.events)
248 def test_unknown_events_ignored(self):
249 self.events.append(MockEvent('mock!event'))
251 # No exception should be raised.
253 def test_fetches_container_on_create(self):
254 self.events.append(MockEvent('create'))
256 self.docker_client.inspect_container.assert_called_with(
257 self.events[0]['id'])
259 def test_adds_user_on_container_create(self):
260 self.events.append(MockEvent('create'))
262 self.images.add_user.assert_called_with(
263 self.docker_client.inspect_container(), self.events[0]['time'])
265 def test_unknown_image_handling(self):
266 # The use recorder should not fetch any images.
267 self.events.append(MockEvent('create'))
269 self.assertFalse(self.docker_client.inspect_image.called)
271 def test_unfetchable_containers_ignored(self):
272 self.events.append(MockEvent('create'))
273 self.docker_client.inspect_container.side_effect = MockException(404)
275 self.assertFalse(self.images.add_user.called)
277 def test_ends_user_on_container_destroy(self):
278 self.events.append(MockEvent('destroy'))
280 self.images.end_user.assert_called_with(self.events[0]['id'])
283 class DockerImageCleanerTestCase(DockerImageUseRecorderTestCase):
284 TEST_CLASS = cleaner.DockerImageCleaner
286 def test_unknown_image_handling(self):
287 # The image cleaner should fetch and record new images.
288 self.images.has_image.return_value = False
289 self.events.append(MockEvent('create'))
291 self.docker_client.inspect_image.assert_called_with(
292 self.docker_client.inspect_container()['Image'])
293 self.images.add_image.assert_called_with(
294 self.docker_client.inspect_image())
296 def test_unfetchable_images_ignored(self):
297 self.images.has_image.return_value = False
298 self.docker_client.inspect_image.side_effect = MockException(404)
299 self.events.append(MockEvent('create'))
301 self.docker_client.inspect_image.assert_called_with(
302 self.docker_client.inspect_container()['Image'])
303 self.assertFalse(self.images.add_image.called)
305 def test_deletions_after_destroy(self):
306 delete_id = MockDockerId()
307 self.images.should_delete.return_value = [delete_id]
308 self.events.append(MockEvent('destroy'))
310 self.docker_client.remove_image.assert_called_with(delete_id)
311 self.images.del_image.assert_called_with(delete_id)
313 def test_failed_deletion_handling(self):
314 delete_id = MockDockerId()
315 self.images.should_delete.return_value = [delete_id]
316 self.docker_client.remove_image.side_effect = MockException(500)
317 self.events.append(MockEvent('destroy'))
319 self.docker_client.remove_image.assert_called_with(delete_id)
320 self.assertFalse(self.images.del_image.called)
323 class DockerContainerCleanerTestCase(DockerImageUseRecorderTestCase):
324 TEST_CLASS = cleaner.DockerImageCleaner
325 TEST_CLASS_INIT_KWARGS = {'remove_containers_onexit': True}
327 def test_container_deletion_deletes_volumes(self):
329 self.events.append(MockEvent('die', docker_id=cid))
331 self.docker_client.remove_container.assert_called_with(cid, v=True)
333 @mock.patch('arvados_docker.cleaner.logger')
334 def test_failed_container_deletion_handling(self, mockLogger):
336 self.docker_client.remove_container.side_effect = MockException(500)
337 self.events.append(MockEvent('die', docker_id=cid))
339 self.docker_client.remove_container.assert_called_with(cid, v=True)
340 self.assertEqual("Failed to remove container %s: %s",
341 mockLogger.warning.call_args[0][0])
342 self.assertEqual(cid,
343 mockLogger.warning.call_args[0][1])
346 class HumanSizeTestCase(unittest.TestCase):
348 def check(self, human_str, count, exp):
349 self.assertEqual(count * (1024 ** exp),
350 cleaner.human_size(human_str))
352 def test_bytes(self):
353 self.check('1', 1, 0)
354 self.check('82', 82, 0)
356 def test_kibibytes(self):
357 self.check('2K', 2, 1)
358 self.check('3k', 3, 1)
360 def test_mebibytes(self):
361 self.check('4M', 4, 2)
362 self.check('5m', 5, 2)
364 def test_gibibytes(self):
365 self.check('6G', 6, 3)
366 self.check('7g', 7, 3)
368 def test_tebibytes(self):
369 self.check('8T', 8, 4)
370 self.check('9t', 9, 4)
373 class RunTestCase(unittest.TestCase):
376 self.config = cleaner.default_config()
377 self.config['Quota'] = 1000000
378 self.docker_client = mock.MagicMock(name='docker_client')
381 test_start_time = int(time.time())
382 self.docker_client.events.return_value = []
383 cleaner.run(self.config, self.docker_client)
384 self.assertEqual(2, self.docker_client.events.call_count)
385 event_kwargs = [args[1] for args in
386 self.docker_client.events.call_args_list]
387 self.assertIn('since', event_kwargs[0])
388 self.assertIn('until', event_kwargs[0])
389 self.assertLessEqual(test_start_time, event_kwargs[0]['until'])
390 self.assertIn('since', event_kwargs[1])
391 self.assertEqual(event_kwargs[0]['until'], event_kwargs[1]['since'])
394 @mock.patch('docker.Client', name='docker_client')
395 @mock.patch('arvados_docker.cleaner.run', name='cleaner_run')
396 class MainTestCase(unittest.TestCase):
398 def test_client_api_version(self, run_mock, docker_client):
399 with tempfile.NamedTemporaryFile(mode='wt') as cf:
400 cf.write('{"Quota":"1000T"}')
402 cleaner.main(['--config', cf.name])
403 self.assertEqual(1, docker_client.call_count)
404 # 1.14 is the first version that's well defined, going back to
405 # Docker 1.2, and still supported up to at least Docker 1.9.
407 # <https://docs.docker.com/engine/reference/api/docker_remote_api/>.
408 self.assertEqual('1.14',
409 docker_client.call_args[1].get('version'))
410 self.assertEqual(1, run_mock.call_count)
411 self.assertIs(run_mock.call_args[0][1], docker_client())
414 class ConfigTestCase(unittest.TestCase):
416 def test_load_config(self):
417 with tempfile.NamedTemporaryFile(mode='wt') as cf:
419 '{"Quota":"1000T", "RemoveStoppedContainers":"always", "Verbose":2}')
421 config = cleaner.load_config(['--config', cf.name])
422 self.assertEqual(1000 << 40, config['Quota'])
423 self.assertEqual("always", config['RemoveStoppedContainers'])
424 self.assertEqual(2, config['Verbose'])
426 def test_args_override_config(self):
427 with tempfile.NamedTemporaryFile(mode='wt') as cf:
429 '{"Quota":"1000T", "RemoveStoppedContainers":"always", "Verbose":2}')
431 config = cleaner.load_config([
434 '--remove-stopped-containers', 'never',
437 self.assertEqual(1 << 30, config['Quota'])
438 self.assertEqual('never', config['RemoveStoppedContainers'])
439 self.assertEqual(1, config['Verbose'])
441 def test_args_no_config(self):
442 self.assertEqual(False, os.path.exists(cleaner.DEFAULT_CONFIG_FILE))
443 config = cleaner.load_config(['--quota', '1G'])
444 self.assertEqual(1 << 30, config['Quota'])
447 class ContainerRemovalTestCase(unittest.TestCase):
448 LIFECYCLE = ['create', 'attach', 'start', 'resize', 'die', 'destroy']
451 self.config = cleaner.default_config()
452 self.docker_client = mock.MagicMock(name='docker_client')
453 self.existingCID = MockDockerId()
454 self.docker_client.containers.return_value = [{
455 'Id': self.existingCID,
456 'Status': 'Exited (0) 6 weeks ago',
458 # If docker_client.containers() returns non-exited
459 # containers for some reason, do not remove them.
460 'Id': MockDockerId(),
463 self.newCID = MockDockerId()
464 self.docker_client.events.return_value = [
465 MockEvent(e, docker_id=self.newCID).encoded()
466 for e in self.LIFECYCLE]
468 def test_remove_onexit(self):
469 self.config['RemoveStoppedContainers'] = 'onexit'
470 cleaner.run(self.config, self.docker_client)
471 self.docker_client.remove_container.assert_called_once_with(
474 def test_remove_always(self):
475 self.config['RemoveStoppedContainers'] = 'always'
476 cleaner.run(self.config, self.docker_client)
477 self.docker_client.remove_container.assert_any_call(
478 self.existingCID, v=True)
479 self.docker_client.remove_container.assert_any_call(
481 self.assertEqual(2, self.docker_client.remove_container.call_count)
483 def test_remove_never(self):
484 self.config['RemoveStoppedContainers'] = 'never'
485 cleaner.run(self.config, self.docker_client)
486 self.assertEqual(0, self.docker_client.remove_container.call_count)
488 def test_container_exited_between_subscribe_events_and_check_existing(self):
489 self.config['RemoveStoppedContainers'] = 'always'
490 self.docker_client.events.return_value = [
491 MockEvent(e, docker_id=self.existingCID).encoded()
492 for e in ['die', 'destroy']]
493 cleaner.run(self.config, self.docker_client)
494 # Subscribed to events before getting the list of existing
496 self.docker_client.assert_has_calls([
497 mock.call.events(since=mock.ANY),
498 mock.call.containers(filters={'status': 'exited'})])
499 # Asked to delete the container twice?
500 self.docker_client.remove_container.assert_has_calls(
501 [mock.call(self.existingCID, v=True)] * 2)
502 self.assertEqual(2, self.docker_client.remove_container.call_count)