def _update_poll_time(self, poll_key):
self.last_polls[poll_key] = time.time()
- def _resume_node(self, node_record):
- node_record.actor.resume_node()
-
def _pair_nodes(self, node_record, arvados_node):
self._logger.info("Cloud node %s is now paired with Arvados node %s",
node_record.cloud_node.name, arvados_node['uuid'])
if cloud_rec.actor.offer_arvados_pair(arv_node).get():
self._pair_nodes(cloud_rec, arv_node)
break
- for rec in self.cloud_nodes.nodes.itervalues():
- # crunch-dispatch turns all slurm states that are not either "idle"
- # or "alloc" into "down", but in case that behavior changes, assume
- # any state that is not "idle" or "alloc" could be a state we want
- # to try to resume from.
- if (rec.arvados_node is not None and
- rec.arvados_node["info"].get("slurm_state") not in ("idle", "alloc") and
- rec.cloud_node.id not in self.shutdowns):
- self._resume_node(rec)
def _nodes_booting(self, size):
s = sum(1
booting_count = self._nodes_booting(size) + self._nodes_unpaired(size)
shutdown_count = self._size_shutdowns(size)
busy_count = self._nodes_busy(size)
- up_count = self._nodes_up(size) - (shutdown_count + busy_count + self._nodes_missing(size))
+ idle_count = self._nodes_up(size) - (busy_count + self._nodes_missing(size))
self._logger.info("%s: wishlist %i, up %i (booting %i, idle %i, busy %i), shutting down %i", size.name,
self._size_wishlist(size),
- up_count + busy_count,
+ idle_count + busy_count,
booting_count,
- up_count - booting_count,
+ idle_count - booting_count,
busy_count,
shutdown_count)
- wanted = self._size_wishlist(size) - up_count
+ wanted = self._size_wishlist(size) - idle_count
if wanted > 0 and self.max_total_price and ((total_price + (size.price*wanted)) > self.max_total_price):
can_boot = int((self.max_total_price - total_price) / size.price)
if can_boot == 0:
return wanted
def _nodes_excess(self, size):
- up_count = self._nodes_up(size) - self._size_shutdowns(size)
+ up_count = (self._nodes_booting(size) + self._nodes_booted(size)) - self._size_shutdowns(size)
if size.id == self.min_cloud_size.id:
up_count -= self.min_nodes
return up_count - self._nodes_busy(size) - self._size_wishlist(size)