X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/80d30b6a1662e03d56d33f7b29a211d0b3413e2c..e94f86dc97746ef21c641aadb56112481d1e66f5:/services/nodemanager/arvnodeman/daemon.py

diff --git a/services/nodemanager/arvnodeman/daemon.py b/services/nodemanager/arvnodeman/daemon.py
index 9502542cff..243d3bfaa4 100644
--- a/services/nodemanager/arvnodeman/daemon.py
+++ b/services/nodemanager/arvnodeman/daemon.py
@@ -110,7 +110,8 @@ class NodeManagerDaemonActor(actor_class):
                  node_stale_after=7200,
                  node_setup_class=dispatch.ComputeNodeSetupActor,
                  node_shutdown_class=dispatch.ComputeNodeShutdownActor,
-                 node_actor_class=dispatch.ComputeNodeMonitorActor):
+                 node_actor_class=dispatch.ComputeNodeMonitorActor,
+                 max_total_price=0):
         super(NodeManagerDaemonActor, self).__init__()
         self._node_setup = node_setup_class
         self._node_shutdown = node_shutdown_class
@@ -127,6 +128,7 @@ class NodeManagerDaemonActor(actor_class):
         self.min_cloud_size = self.server_calculator.cheapest_size()
         self.min_nodes = min_nodes
         self.max_nodes = max_nodes
+        self.max_total_price = max_total_price
         self.poll_stale_after = poll_stale_after
         self.boot_fail_after = boot_fail_after
         self.node_stale_after = node_stale_after
@@ -216,13 +218,22 @@ class NodeManagerDaemonActor(actor_class):
         up = 0
         up += sum(1
                   for c in self.booting.itervalues()
-                  if size is None or c.cloud_node.get().size.id == size.id)
+                  if size is None or c.cloud_size.get().id == size.id)
         up += sum(1
                   for i in (self.booted, self.cloud_nodes.nodes)
                   for c in i.itervalues()
                   if size is None or c.cloud_node.size.id == size.id)
         return up
 
+    def _total_price(self):
+        cost = 0
+        cost += sum(self.server_calculator.find_size(c.cloud_size.get().id).price
+                  for c in self.booting.itervalues())
+        cost += sum(self.server_calculator.find_size(c.cloud_node.size.id).price
+                    for i in (self.booted, self.cloud_nodes.nodes)
+                    for c in i.itervalues())
+        return cost
+
     def _nodes_busy(self, size):
         return sum(1 for busy in
                    pykka.get_all(rec.actor.in_state('busy') for rec in
@@ -241,24 +252,41 @@ class NodeManagerDaemonActor(actor_class):
         return sum(1 for c in self.last_wishlist if c.id == size.id)
 
     def _size_shutdowns(self, size):
-        return sum(1 for c in self.shutdowns.itervalues()
-                   if c.cloud_node.get().size.id == size.id)
+        sh = 0
+        for c in self.shutdowns.itervalues():
+            try:
+                if c.cloud_node.get().size.id == size.id:
+                    sh += 1
+            except pykka.ActorDeadError:
+                pass
+        return sh
 
     def _nodes_wanted(self, size):
         total_up_count = self._nodes_up(None)
         under_min = self.min_nodes - total_up_count
         over_max = total_up_count - self.max_nodes
+        total_price = self._total_price()
+
         if over_max >= 0:
             return -over_max
         elif under_min > 0 and size.id == self.min_cloud_size.id:
             return under_min
+
+        up_count = self._nodes_up(size) - (self._size_shutdowns(size) +
+                                           self._nodes_busy(size) +
+                                           self._nodes_missing(size))
+
+        self._logger.debug("%s: idle nodes %i, wishlist size %i", size.name, up_count, self._size_wishlist(size))
+
+        wanted = self._size_wishlist(size) - up_count
+        if wanted > 0 and self.max_total_price and ((total_price + (size.price*wanted)) > self.max_total_price):
+            can_boot = int((self.max_total_price - total_price) / size.price)
+            if can_boot == 0:
+                self._logger.info("Not booting %s (price %s) because with it would exceed max_total_price of %s (current total_price is %s)",
+                                  size.name, size.price, self.max_total_price, total_price)
+            return can_boot
         else:
-            up_count = self._nodes_up(size) - (self._size_shutdowns(size) +
-                                               self._nodes_busy(size) +
-                                               self._nodes_missing(size))
-            #self._logger.info("_nodes_up for %s is %s", size.id, self._nodes_up(size))
-            #self._logger.info("counts %s %s %s", len(self.booting), len(self.booted), len(self.cloud_nodes))
-            return self._size_wishlist(size) - up_count
+            return wanted
 
     def _nodes_excess(self, size):
         up_count = self._nodes_up(size) - self._size_shutdowns(size)
@@ -269,8 +297,7 @@ class NodeManagerDaemonActor(actor_class):
     def update_server_wishlist(self, wishlist):
         self._update_poll_time('server_wishlist')
         self.last_wishlist = wishlist
-        for sz in reversed(self.server_calculator.cloud_sizes):
-            size = sz.real
+        for size in reversed(self.server_calculator.cloud_sizes):
             nodes_wanted = self._nodes_wanted(size)
             if nodes_wanted > 0:
                 self._later.start_node(size)