return driver_class(**auth_kwargs)
@RetryMixin._retry()
- def _set_sizes(self):
- self.sizes = {sz.id: sz for sz in self.real.list_sizes()}
+ def sizes(self):
+ if self._sizes is None:
+ self._sizes = {sz.id: sz for sz in self.real.list_sizes()}
+ return self._sizes
def __init__(self, auth_kwargs, list_kwargs, create_kwargs,
driver_class, retry_wait=1, max_retry_wait=180):
if new_pair is not None:
self.create_kwargs[new_pair[0]] = new_pair[1]
- self._set_sizes()
+ self._sizes = None
def _init_ping_host(self, ping_host):
self.ping_host = ping_host
try:
kwargs = self.create_kwargs.copy()
kwargs.update(self.arvados_create_kwargs(size, arvados_node))
- kwargs['size'] = size
+ kwargs['size'] = size.real
return self.real.create_node(**kwargs)
except CLOUD_ERRORS as create_error:
# Workaround for bug #6702: sometimes the create node request
def arvados_create_kwargs(self, size, arvados_node):
tags = {
+ # Set up tag indicating the Arvados assigned Cloud Size id.
+ 'arvados_node_size': size.id,
'booted_at': time.strftime(ARVADOS_TIMEFMT, time.gmtime()),
'arv-ping-url': self._make_ping_url(arvados_node)
}
# Do our own filtering based on tag.
nodes = [node for node in
super(ComputeNodeDriver, self).list_nodes(ex_fetch_nic=False, ex_fetch_power_state=False)
- if node.extra["tags"].get("arvados-class") == self.tags["arvados-class"]]
+ if node.extra.get("tags", {}).get("arvados-class") == self.tags["arvados-class"]]
for n in nodes:
# Need to populate Node.size
if not n.size:
- n.size = self.sizes[n.extra["properties"]["hardwareProfile"]["vmSize"]]
+ n.size = self.sizes()[n.extra["properties"]["hardwareProfile"]["vmSize"]]
+ n.extra['arvados_node_size'] = n.extra.get('tags', {}).get('arvados_node_size')
return nodes
def broken(self, cloud_node):
"VolumeSize": volsize,
"VolumeType": "gp2"
}}]
+ if size.preemptible:
+ # Request a Spot instance for this node
+ kw['ex_spot_market'] = True
return kw
def sync_node(self, cloud_node, arvados_node):
self.real.ex_create_tags(cloud_node,
{'Name': arvados_node_fqdn(arvados_node)})
+ def create_node(self, size, arvados_node):
+ # Set up tag indicating the Arvados assigned Cloud Size id.
+ self.create_kwargs['ex_metadata'].update({'arvados_node_size': size.id})
+ return super(ComputeNodeDriver, self).create_node(size, arvados_node)
+
def list_nodes(self):
# Need to populate Node.size
nodes = super(ComputeNodeDriver, self).list_nodes()
for n in nodes:
if not n.size:
- n.size = self.sizes[n.extra["instance_type"]]
+ n.size = self.sizes()[n.extra["instance_type"]]
+ n.extra['arvados_node_size'] = n.extra.get('tags', {}).get('arvados_node_size')
return nodes
@classmethod
super(ComputeNodeDriver, self).__init__(
auth_kwargs, list_kwargs, create_kwargs,
driver_class)
- self._sizes_by_id = {sz.id: sz for sz in self.sizes.itervalues()}
self._disktype_links = {dt.name: self._object_link(dt)
for dt in self.real.ex_list_disktypes()}
'ex_disks_gce_struct': disks,
}
result['ex_metadata'].update({
- 'arv-ping-url': self._make_ping_url(arvados_node),
- 'booted_at': time.strftime(ARVADOS_TIMEFMT, time.gmtime()),
- 'hostname': arvados_node_fqdn(arvados_node),
- })
+ 'arvados_node_size': size.id,
+ 'arv-ping-url': self._make_ping_url(arvados_node),
+ 'booted_at': time.strftime(ARVADOS_TIMEFMT, time.gmtime()),
+ 'hostname': arvados_node_fqdn(arvados_node),
+ })
return result
-
def list_nodes(self):
# The GCE libcloud driver only supports filtering node lists by zone.
# Do our own filtering based on tag list.
nodelist = [node for node in
super(ComputeNodeDriver, self).list_nodes()
if self.node_tags.issubset(node.extra.get('tags', []))]
- # As of 0.18, the libcloud GCE driver sets node.size to the size's name.
- # It's supposed to be the actual size object. Check that it's not,
- # and monkeypatch the results when that's the case.
- if nodelist and not hasattr(nodelist[0].size, 'id'):
- for node in nodelist:
+ for node in nodelist:
+ # As of 0.18, the libcloud GCE driver sets node.size to the size's name.
+ # It's supposed to be the actual size object. Check that it's not,
+ # and monkeypatch the results when that's the case.
+ if not hasattr(node.size, 'id'):
- node.size = self._sizes_by_id[node.size]
+ node.size = self.sizes()[node.size]
+ # Get arvados-assigned cloud size id
+ node.extra['arvados_node_size'] = node.extra.get('metadata', {}).get('arvados_node_size')
return nodelist
@classmethod
busy_count = counts["busy"]
wishlist_count = self._size_wishlist(size)
- self._logger.info("%s: wishlist %i, up %i (booting %i, unpaired %i, idle %i, busy %i), down %i, shutdown %i", size.name,
+ self._logger.info("%s: wishlist %i, up %i (booting %i, unpaired %i, idle %i, busy %i), down %i, shutdown %i", size.id,
wishlist_count,
up_count,
counts["booting"],
can_boot = int((self.max_total_price - total_price) / size.price)
if can_boot == 0:
self._logger.info("Not booting %s (price %s) because with it would exceed max_total_price of %s (current total_price is %s)",
- size.name, size.price, self.max_total_price, total_price)
+ size.id, size.price, self.max_total_price, total_price)
return can_boot
else:
return wanted
return None
arvados_node = self.arvados_nodes.find_stale_node(self.node_stale_after)
self._logger.info("Want %i more %s nodes. Booting a node.",
- nodes_wanted, cloud_size.name)
+ nodes_wanted, cloud_size.id)
new_setup = self._node_setup.start(
timer_actor=self._timer,
arvados_client=self._new_arvados(),
arvados_node=arvados_node,
cloud_client=self._new_cloud(),
- cloud_size=self.server_calculator.find_size(cloud_size.id)).proxy()
- self.booting[new_setup.actor_ref.actor_urn] = new_setup
- self.sizes_booting[new_setup.actor_ref.actor_urn] = cloud_size
+ cloud_size=self.server_calculator.find_size(cloud_size.id))
+ self.booting[new_setup.actor_urn] = new_setup.proxy()
+ self.sizes_booting[new_setup.actor_urn] = cloud_size
if arvados_node is not None:
self.arvados_nodes[arvados_node['uuid']].assignment_time = (
time.time())
- new_setup.subscribe(self._later.node_setup_finished)
+ new_setup.tell_proxy().subscribe(self._later.node_setup_finished)
if nodes_wanted > 1:
self._later.start_node(cloud_size)
projects / arvados.git / commitdiff