Merge branch 'master' into 3177-collection-choose-files
[arvados.git] / services / nodemanager / arvnodeman / config.py
1 #!/usr/bin/env python
2
3 from __future__ import absolute_import, print_function
4
5 import ConfigParser
6 import importlib
7 import logging
8 import ssl
9
10 import arvados
11 import httplib2
12 import libcloud.common.types as cloud_types
13 import pykka
14 from apiclient import errors as apierror
15
16 # IOError is the base class for socket.error and friends.
17 # It seems like it hits the sweet spot for operations we want to retry:
18 # it's low-level, but unlikely to catch code bugs.
19 NETWORK_ERRORS = (IOError, ssl.SSLError)
20 ARVADOS_ERRORS = NETWORK_ERRORS + (apierror.Error,)
21 CLOUD_ERRORS = NETWORK_ERRORS + (cloud_types.LibcloudError,)
22
23 actor_class = pykka.ThreadingActor
24
25 class NodeManagerConfig(ConfigParser.SafeConfigParser):
26     """Node Manager Configuration class.
27
28     This a standard Python ConfigParser, with additional helper methods to
29     create objects instantiated with configuration information.
30     """
31
32     LOGGING_NONLEVELS = frozenset(['file'])
33
34     def __init__(self, *args, **kwargs):
35         # Can't use super() because SafeConfigParser is an old-style class.
36         ConfigParser.SafeConfigParser.__init__(self, *args, **kwargs)
37         for sec_name, settings in {
38             'Arvados': {'insecure': 'no',
39                         'timeout': '15'},
40             'Daemon': {'min_nodes': '0',
41                        'max_nodes': '1',
42                        'poll_time': '60',
43                        'max_poll_time': '300',
44                        'poll_stale_after': '600',
45                        'node_stale_after': str(60 * 60 * 2)},
46             'Logging': {'file': '/dev/stderr',
47                         'level': 'WARNING'},
48         }.iteritems():
49             if not self.has_section(sec_name):
50                 self.add_section(sec_name)
51             for opt_name, value in settings.iteritems():
52                 if not self.has_option(sec_name, opt_name):
53                     self.set(sec_name, opt_name, value)
54
55     def get_section(self, section, transformer=None):
56         result = self._dict()
57         for key, value in self.items(section):
58             if transformer is not None:
59                 try:
60                     value = transformer(value)
61                 except (TypeError, ValueError):
62                     pass
63             result[key] = value
64         return result
65
66     def log_levels(self):
67         return {key: getattr(logging, self.get('Logging', key).upper())
68                 for key in self.options('Logging')
69                 if key not in self.LOGGING_NONLEVELS}
70
71     def new_arvados_client(self):
72         if self.has_option('Daemon', 'certs_file'):
73             certs_file = self.get('Daemon', 'certs_file')
74         else:
75             certs_file = None
76         insecure = self.getboolean('Arvados', 'insecure')
77         http = httplib2.Http(timeout=self.getint('Arvados', 'timeout'),
78                              ca_certs=certs_file,
79                              disable_ssl_certificate_validation=insecure)
80         return arvados.api('v1',
81                            cache=False,  # Don't reuse an existing client.
82                            host=self.get('Arvados', 'host'),
83                            token=self.get('Arvados', 'token'),
84                            insecure=insecure,
85                            http=http)
86
87     def new_cloud_client(self):
88         module = importlib.import_module('arvnodeman.computenode.driver.' +
89                                          self.get('Cloud', 'provider'))
90         auth_kwargs = self.get_section('Cloud Credentials')
91         if 'timeout' in auth_kwargs:
92             auth_kwargs['timeout'] = int(auth_kwargs['timeout'])
93         return module.ComputeNodeDriver(auth_kwargs,
94                                         self.get_section('Cloud List'),
95                                         self.get_section('Cloud Create'))
96
97     def node_sizes(self, all_sizes):
98         size_kwargs = {}
99         for sec_name in self.sections():
100             sec_words = sec_name.split(None, 2)
101             if sec_words[0] != 'Size':
102                 continue
103             size_kwargs[sec_words[1]] = self.get_section(sec_name, int)
104         return [(size, size_kwargs[size.id]) for size in all_sizes
105                 if size.id in size_kwargs]
106
107     def shutdown_windows(self):
108         return [int(n)
109                 for n in self.get('Cloud', 'shutdown_windows').split(',')]