Merge branch '19954-permission-dedup-doc'

[arvados.git] / services / dockercleaner / arvados_docker / cleaner.py

diff --git a/services/dockercleaner/arvados_docker/cleaner.py b/services/dockercleaner/arvados_docker/cleaner.py
index 5531bf5ec42f7e95d26ed3da394f095b7346f76f..2a0e8b9108608df05dd3fe38e55f8311d82747f7 100755 (executable)
--- a/services/dockercleaner/arvados_docker/cleaner.py
+++ b/services/dockercleaner/arvados_docker/cleaner.py
@@ -1,4 +1,8 @@
 #!/usr/bin/env python3
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
 """arvados_docker.cleaner - Remove unused Docker images from compute nodes
 
 Usage:
@@ -17,10 +21,13 @@ import time
 import docker
 import json
 
+DEFAULT_CONFIG_FILE = '/etc/arvados/docker-cleaner/docker-cleaner.json'
+
 SUFFIX_SIZES = {suffix: 1024 ** exp for exp, suffix in enumerate('kmgt', 1)}
 
 logger = logging.getLogger('arvados_docker.cleaner')
 
+
 def return_when_docker_not_found(result=None):
     # If the decorated function raises a 404 error from Docker, return
     # `result` instead.
@@ -36,7 +43,9 @@ def return_when_docker_not_found(result=None):
         return docker_not_found_wrapper
     return docker_not_found_decorator
 
+
 class DockerImage:
+
     def __init__(self, image_hash):
         self.docker_id = image_hash['Id']
         self.size = image_hash['VirtualSize']
@@ -47,6 +56,7 @@ class DockerImage:
 
 
 class DockerImages:
+
     def __init__(self, target_size):
         self.target_size = target_size
         self.images = {}
@@ -118,6 +128,7 @@ class DockerImages:
 class DockerEventHandlers:
     # This class maps Docker event types to the names of methods that should
     # receive those events.
+
     def __init__(self):
         self.handler_names = collections.defaultdict(list)
 
@@ -221,7 +232,8 @@ class DockerImageCleaner(DockerImageUseRecorder):
             try:
                 self.docker_client.remove_image(image_id)
             except docker.errors.APIError as error:
-                logger.warning("Failed to remove image %s: %s", image_id, error)
+                logger.warning(
+                    "Failed to remove image %s: %s", image_id, error)
             else:
                 logger.info("Removed image %s", image_id)
                 self.images.del_image(image_id)
@@ -231,8 +243,9 @@ class DockerImageCleaner(DockerImageUseRecorder):
         unknown_ids = {image['Id'] for image in self.docker_client.images()
                        if not self.images.has_image(image['Id'])}
         for image_id in (unknown_ids - self.logged_unknown):
-            logger.info("Image %s is loaded but unused, so it won't be cleaned",
-                        image_id)
+            logger.info(
+                "Image %s is loaded but unused, so it won't be cleaned",
+                image_id)
         self.logged_unknown = unknown_ids
 
 
@@ -245,12 +258,24 @@ def human_size(size_str):
         size_str = size_str[:-1]
     return int(size_str) * multiplier
 
+
 def load_config(arguments):
     args = parse_arguments(arguments)
 
     config = default_config()
-    with open(args.config, 'r') as f:
-        config.update(json.load(f))
+    try:
+        with open(args.config, 'r') as f:
+            c = json.load(f)
+            config.update(c)
+    except (FileNotFoundError, IOError, ValueError) as error:
+        if (isinstance(error, FileNotFoundError) and
+            args.config == DEFAULT_CONFIG_FILE):
+            logger.warning("DEPRECATED: default config file %s not found; "
+                           "relying on command line configuration",
+                           repr(DEFAULT_CONFIG_FILE))
+        else:
+            sys.exit('error reading config file {}: {}'.format(
+                args.config, error))
 
     configargs = vars(args).copy()
     configargs.pop('config')
@@ -261,6 +286,7 @@ def load_config(arguments):
 
     return config
 
+
 def default_config():
     return {
         'Quota': '1G',
@@ -268,6 +294,7 @@ def default_config():
         'Verbose': 0,
     }
 
+
 def parse_arguments(arguments):
     class Formatter(argparse.ArgumentDefaultsHelpFormatter,
                     argparse.RawDescriptionHelpFormatter):
@@ -280,7 +307,7 @@ def parse_arguments(arguments):
         formatter_class=Formatter,
     )
     parser.add_argument(
-        '--config', action='store', type=str, default='/etc/arvados/dockercleaner/config.json',
+        '--config', action='store', type=str, default=DEFAULT_CONFIG_FILE,
         help="configuration file")
 
     deprecated = " (DEPRECATED -- use config file instead)"
@@ -299,14 +326,19 @@ def parse_arguments(arguments):
 
     return parser.parse_args(arguments)
 
-def setup_logging(config):
+
+def setup_logging():
     log_handler = logging.StreamHandler()
     log_handler.setFormatter(logging.Formatter(
-            '%(asctime)s %(name)s[%(process)d] %(levelname)s: %(message)s',
-            '%Y-%m-%d %H:%M:%S'))
+        '%(asctime)s %(name)s[%(process)d] %(levelname)s: %(message)s',
+        '%Y-%m-%d %H:%M:%S'))
     logger.addHandler(log_handler)
+
+
+def configure_logging(config):
     logger.setLevel(logging.ERROR - (10 * config['Verbose']))
 
+
 def run(config, docker_client):
     start_time = int(time.time())
     logger.debug("Loading Docker activity through present")
@@ -324,9 +356,11 @@ def run(config, docker_client):
     logger.info("Listening for docker events")
     cleaner.run()
 
+
 def main(arguments=sys.argv[1:]):
+    setup_logging()
     config = load_config(arguments)
-    setup_logging(config)
+    configure_logging(config)
     try:
         run(config, docker.Client(version='1.14'))
     except KeyboardInterrupt: