projects
/
arvados.git
/ blobdiff
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
Try to make logging identify the actor consistently
[arvados.git]
/
services
/
nodemanager
/
arvnodeman
/
jobqueue.py
diff --git
a/services/nodemanager/arvnodeman/jobqueue.py
b/services/nodemanager/arvnodeman/jobqueue.py
index 239934f52911782730840d3d1aa5042866345451..87cf738311730feed045d23c63af6481c0731e06 100644
(file)
--- a/
services/nodemanager/arvnodeman/jobqueue.py
+++ b/
services/nodemanager/arvnodeman/jobqueue.py
@@
-30,6
+30,9
@@
class ServerCalculator(object):
raise ValueError("unrecognized size field '%s'" % (name,))
setattr(self, name, override)
raise ValueError("unrecognized size field '%s'" % (name,))
setattr(self, name, override)
+ if self.price is None:
+ raise ValueError("Required field 'price' is None")
+
def meets_constraints(self, **kwargs):
for name, want_value in kwargs.iteritems():
have_value = getattr(self, name)
def meets_constraints(self, **kwargs):
for name, want_value in kwargs.iteritems():
have_value = getattr(self, name)
@@
-38,12
+41,12
@@
class ServerCalculator(object):
return True
return True
- def __init__(self, server_list, m
in_nodes=0, max_nodes
=None):
+ def __init__(self, server_list, m
ax_nodes=None, max_price
=None):
self.cloud_sizes = [self.CloudSizeWrapper(s, **kws)
for s, kws in server_list]
self.cloud_sizes.sort(key=lambda s: s.price)
self.cloud_sizes = [self.CloudSizeWrapper(s, **kws)
for s, kws in server_list]
self.cloud_sizes.sort(key=lambda s: s.price)
- self.min_nodes = min_nodes
self.max_nodes = max_nodes or float('inf')
self.max_nodes = max_nodes or float('inf')
+ self.max_price = max_price or float('inf')
self.logger = logging.getLogger('arvnodeman.jobqueue')
self.logged_jobs = set()
self.logger = logging.getLogger('arvnodeman.jobqueue')
self.logged_jobs = set()
@@
-70,24
+73,25
@@
class ServerCalculator(object):
for job in queue:
seen_jobs.add(job['uuid'])
constraints = job['runtime_constraints']
for job in queue:
seen_jobs.add(job['uuid'])
constraints = job['runtime_constraints']
- want_count =
self.coerce_int(constraints.get('min_nodes'), 1
)
+ want_count =
max(1, self.coerce_int(constraints.get('min_nodes'), 1)
)
cloud_size = self.cloud_size_for_constraints(constraints)
if cloud_size is None:
if job['uuid'] not in self.logged_jobs:
self.logged_jobs.add(job['uuid'])
self.logger.debug("job %s not satisfiable", job['uuid'])
cloud_size = self.cloud_size_for_constraints(constraints)
if cloud_size is None:
if job['uuid'] not in self.logged_jobs:
self.logged_jobs.add(job['uuid'])
self.logger.debug("job %s not satisfiable", job['uuid'])
- elif (want_count <= self.max_nodes):
- servers.extend([cloud_size.real] *
max(1, want_count)
)
+ elif (want_count <= self.max_nodes)
and (want_count*cloud_size.price <= self.max_price)
:
+ servers.extend([cloud_size.real] *
want_count
)
self.logged_jobs.intersection_update(seen_jobs)
self.logged_jobs.intersection_update(seen_jobs)
-
- # Make sure the server queue has at least enough entries to
- # satisfy min_nodes.
- node_shortfall = self.min_nodes - len(servers)
- if node_shortfall > 0:
- basic_node = self.cloud_size_for_constraints({})
- servers.extend([basic_node.real] * node_shortfall)
return servers
return servers
+ def cheapest_size(self):
+ return self.cloud_sizes[0]
+
+ def find_size(self, sizeid):
+ for s in self.cloud_sizes:
+ if s.id == sizeid:
+ return s
+ return None
class JobQueueMonitorActor(clientactor.RemotePollLoopActor):
"""Actor to generate server wishlists from the job queue.
class JobQueueMonitorActor(clientactor.RemotePollLoopActor):
"""Actor to generate server wishlists from the job queue.
@@
-98,7
+102,6
@@
class JobQueueMonitorActor(clientactor.RemotePollLoopActor):
"""
CLIENT_ERRORS = ARVADOS_ERRORS
"""
CLIENT_ERRORS = ARVADOS_ERRORS
- LOGGER_NAME = 'arvnodeman.jobqueue'
def __init__(self, client, timer_actor, server_calc, *args, **kwargs):
super(JobQueueMonitorActor, self).__init__(
def __init__(self, client, timer_actor, server_calc, *args, **kwargs):
super(JobQueueMonitorActor, self).__init__(
@@
-110,6
+113,6
@@
class JobQueueMonitorActor(clientactor.RemotePollLoopActor):
def _got_response(self, queue):
server_list = self._calculator.servers_for_queue(queue)
def _got_response(self, queue):
server_list = self._calculator.servers_for_queue(queue)
- self._logger.debug("
Sending server
wishlist: %s",
+ self._logger.debug("
Calculated
wishlist: %s",
', '.join(s.name for s in server_list) or "(empty)")
return super(JobQueueMonitorActor, self)._got_response(server_list)
', '.join(s.name for s in server_list) or "(empty)")
return super(JobQueueMonitorActor, self)._got_response(server_list)