3 from __future__ import absolute_import, print_function
13 from apiclient import errors as apierror
15 from .baseactor import BaseNodeManagerActor
17 from libcloud.common.types import LibcloudError
18 from libcloud.common.exceptions import BaseHTTPError
20 # IOError is the base class for socket.error, ssl.SSLError, and friends.
21 # It seems like it hits the sweet spot for operations we want to retry:
22 # it's low-level, but unlikely to catch code bugs.
23 NETWORK_ERRORS = (IOError,)
24 ARVADOS_ERRORS = NETWORK_ERRORS + (apierror.Error,)
25 CLOUD_ERRORS = NETWORK_ERRORS + (LibcloudError, BaseHTTPError)
27 actor_class = BaseNodeManagerActor
29 class NodeManagerConfig(ConfigParser.SafeConfigParser):
30 """Node Manager Configuration class.
32 This a standard Python ConfigParser, with additional helper methods to
33 create objects instantiated with configuration information.
36 LOGGING_NONLEVELS = frozenset(['file'])
38 def __init__(self, *args, **kwargs):
39 # Can't use super() because SafeConfigParser is an old-style class.
40 ConfigParser.SafeConfigParser.__init__(self, *args, **kwargs)
41 for sec_name, settings in {
42 'Arvados': {'insecure': 'no',
47 'Daemon': {'min_nodes': '0',
50 'max_poll_time': '300',
51 'poll_stale_after': '600',
52 'max_total_price': '0',
53 'boot_fail_after': str(sys.maxint),
54 'node_stale_after': str(60 * 60 * 2),
56 'node_mem_scaling': '0.95'},
57 'Manage': {'address': '127.0.0.1',
59 'Logging': {'file': '/dev/stderr',
62 if not self.has_section(sec_name):
63 self.add_section(sec_name)
64 for opt_name, value in settings.iteritems():
65 if not self.has_option(sec_name, opt_name):
66 self.set(sec_name, opt_name, value)
68 def get_section(self, section, transformer=None):
70 for key, value in self.items(section):
71 if transformer is not None:
73 value = transformer(value)
74 except (TypeError, ValueError):
80 return {key: getattr(logging, self.get('Logging', key).upper())
81 for key in self.options('Logging')
82 if key not in self.LOGGING_NONLEVELS}
84 def dispatch_classes(self):
85 mod_name = 'arvnodeman.computenode.dispatch'
86 if self.has_option('Daemon', 'dispatcher'):
87 mod_name = '{}.{}'.format(mod_name,
88 self.get('Daemon', 'dispatcher'))
89 module = importlib.import_module(mod_name)
90 return (module.ComputeNodeSetupActor,
91 module.ComputeNodeShutdownActor,
92 module.ComputeNodeUpdateActor,
93 module.ComputeNodeMonitorActor)
95 def new_arvados_client(self):
96 if self.has_option('Daemon', 'certs_file'):
97 certs_file = self.get('Daemon', 'certs_file')
100 insecure = self.getboolean('Arvados', 'insecure')
101 http = httplib2.Http(timeout=self.getint('Arvados', 'timeout'),
103 disable_ssl_certificate_validation=insecure)
104 return arvados.api(version='v1',
105 host=self.get('Arvados', 'host'),
106 token=self.get('Arvados', 'token'),
110 def new_cloud_client(self):
111 module = importlib.import_module('arvnodeman.computenode.driver.' +
112 self.get('Cloud', 'provider'))
113 driver_class = module.ComputeNodeDriver.DEFAULT_DRIVER
114 if self.has_option('Cloud', 'driver_class'):
115 d = self.get('Cloud', 'driver_class').split('.')
116 mod = '.'.join(d[:-1])
118 driver_class = importlib.import_module(mod).__dict__[cls]
119 auth_kwargs = self.get_section('Cloud Credentials')
120 if 'timeout' in auth_kwargs:
121 auth_kwargs['timeout'] = int(auth_kwargs['timeout'])
122 return module.ComputeNodeDriver(auth_kwargs,
123 self.get_section('Cloud List'),
124 self.get_section('Cloud Create'),
125 driver_class=driver_class)
127 def node_sizes(self, all_sizes):
128 """Finds all acceptable NodeSizes for our installation.
130 Returns a list of (NodeSize, kwargs) pairs for each NodeSize object
131 returned by libcloud that matches a size listed in our config file.
135 for sec_name in self.sections():
136 sec_words = sec_name.split(None, 2)
137 if sec_words[0] != 'Size':
139 size_spec = self.get_section(sec_name, int)
140 if 'price' in size_spec:
141 size_spec['price'] = float(size_spec['price'])
142 size_kwargs[sec_words[1]] = size_spec
143 # EC2 node sizes are identified by id. GCE sizes are identified by name.
145 for size in all_sizes:
146 if size.id in size_kwargs:
147 matching_sizes.append((size, size_kwargs[size.id]))
148 elif size.name in size_kwargs:
149 matching_sizes.append((size, size_kwargs[size.name]))
150 return matching_sizes
152 def shutdown_windows(self):
154 for n in self.get('Cloud', 'shutdown_windows').split(',')]