3 from __future__ import absolute_import, print_function
13 from apiclient import errors as apierror
15 # IOError is the base class for socket.error, ssl.SSLError, and friends.
16 # It seems like it hits the sweet spot for operations we want to retry:
17 # it's low-level, but unlikely to catch code bugs.
18 NETWORK_ERRORS = (IOError,)
19 ARVADOS_ERRORS = NETWORK_ERRORS + (apierror.Error,)
21 actor_class = pykka.ThreadingActor
23 class NodeManagerConfig(ConfigParser.SafeConfigParser):
24 """Node Manager Configuration class.
26 This a standard Python ConfigParser, with additional helper methods to
27 create objects instantiated with configuration information.
30 LOGGING_NONLEVELS = frozenset(['file'])
32 def __init__(self, *args, **kwargs):
33 # Can't use super() because SafeConfigParser is an old-style class.
34 ConfigParser.SafeConfigParser.__init__(self, *args, **kwargs)
35 for sec_name, settings in {
36 'Arvados': {'insecure': 'no',
38 'Daemon': {'min_nodes': '0',
41 'max_poll_time': '300',
42 'poll_stale_after': '600',
43 'max_total_price': '0',
44 'boot_fail_after': str(sys.maxint),
45 'node_stale_after': str(60 * 60 * 2)},
46 'Logging': {'file': '/dev/stderr',
49 if not self.has_section(sec_name):
50 self.add_section(sec_name)
51 for opt_name, value in settings.iteritems():
52 if not self.has_option(sec_name, opt_name):
53 self.set(sec_name, opt_name, value)
55 def get_section(self, section, transformer=None):
57 for key, value in self.items(section):
58 if transformer is not None:
60 value = transformer(value)
61 except (TypeError, ValueError):
67 return {key: getattr(logging, self.get('Logging', key).upper())
68 for key in self.options('Logging')
69 if key not in self.LOGGING_NONLEVELS}
71 def dispatch_classes(self):
72 mod_name = 'arvnodeman.computenode.dispatch'
73 if self.has_option('Daemon', 'dispatcher'):
74 mod_name = '{}.{}'.format(mod_name,
75 self.get('Daemon', 'dispatcher'))
76 module = importlib.import_module(mod_name)
77 return (module.ComputeNodeSetupActor,
78 module.ComputeNodeShutdownActor,
79 module.ComputeNodeUpdateActor,
80 module.ComputeNodeMonitorActor)
82 def new_arvados_client(self):
83 if self.has_option('Daemon', 'certs_file'):
84 certs_file = self.get('Daemon', 'certs_file')
87 insecure = self.getboolean('Arvados', 'insecure')
88 http = httplib2.Http(timeout=self.getint('Arvados', 'timeout'),
90 disable_ssl_certificate_validation=insecure)
91 return arvados.api(version='v1',
92 host=self.get('Arvados', 'host'),
93 token=self.get('Arvados', 'token'),
97 def new_cloud_client(self):
98 module = importlib.import_module('arvnodeman.computenode.driver.' +
99 self.get('Cloud', 'provider'))
100 auth_kwargs = self.get_section('Cloud Credentials')
101 if 'timeout' in auth_kwargs:
102 auth_kwargs['timeout'] = int(auth_kwargs['timeout'])
103 return module.ComputeNodeDriver(auth_kwargs,
104 self.get_section('Cloud List'),
105 self.get_section('Cloud Create'))
107 def node_sizes(self, all_sizes):
108 """Finds all acceptable NodeSizes for our installation.
110 Returns a list of (NodeSize, kwargs) pairs for each NodeSize object
111 returned by libcloud that matches a size listed in our config file.
115 for sec_name in self.sections():
116 sec_words = sec_name.split(None, 2)
117 if sec_words[0] != 'Size':
119 size_spec = self.get_section(sec_name, int)
120 if 'price' in size_spec:
121 size_spec['price'] = float(size_spec['price'])
122 size_kwargs[sec_words[1]] = size_spec
123 # EC2 node sizes are identified by id. GCE sizes are identified by name.
125 for size in all_sizes:
126 if size.id in size_kwargs:
127 matching_sizes.append((size, size_kwargs[size.id]))
128 elif size.name in size_kwargs:
129 matching_sizes.append((size, size_kwargs[size.name]))
130 return matching_sizes
132 def shutdown_windows(self):
134 for n in self.get('Cloud', 'shutdown_windows').split(',')]