X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/b6423b5e5261f1ddaa88db2548e2190f82de21e8..6fe8e52020d421797306e5c6536afbcee761510a:/services/nodemanager/arvnodeman/launcher.py diff --git a/services/nodemanager/arvnodeman/launcher.py b/services/nodemanager/arvnodeman/launcher.py index 93f6cbdbe3..f65e0806ec 100644 --- a/services/nodemanager/arvnodeman/launcher.py +++ b/services/nodemanager/arvnodeman/launcher.py @@ -1,4 +1,7 @@ #!/usr/bin/env python +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 from __future__ import absolute_import, print_function @@ -13,6 +16,7 @@ import pykka import libcloud from . import config as nmconfig +from . import status from .baseactor import WatchdogActor from .daemon import NodeManagerDaemonActor from .jobqueue import JobQueueMonitorActor, ServerCalculator @@ -21,6 +25,7 @@ from .timedcallback import TimedCallBackActor from ._version import __version__ node_daemon = None +watchdog = None def abort(msg, code=1): print("arvados-node-manager: " + msg) @@ -66,7 +71,7 @@ def setup_logging(path, level, **sublevels): return root_logger def build_server_calculator(config): - cloud_size_list = config.node_sizes(config.new_cloud_client().list_sizes()) + cloud_size_list = config.node_sizes() if not cloud_size_list: abort("No valid node sizes configured") return ServerCalculator(cloud_size_list, @@ -75,7 +80,7 @@ def build_server_calculator(config): config.getfloat('Daemon', 'node_mem_scaling')) def launch_pollers(config, server_calculator): - poll_time = config.getint('Daemon', 'poll_time') + poll_time = config.getfloat('Daemon', 'poll_time') max_poll_time = config.getint('Daemon', 'max_poll_time') timer = TimedCallBackActor.start(poll_time / 10.0).tell_proxy() @@ -85,7 +90,10 @@ def launch_pollers(config, server_calculator): config.new_arvados_client(), timer, poll_time, max_poll_time).tell_proxy() job_queue_poller = JobQueueMonitorActor.start( config.new_arvados_client(), timer, server_calculator, - poll_time, max_poll_time).tell_proxy() + config.getboolean('Arvados', 'jobs_queue'), + config.getboolean('Arvados', 'slurm_queue'), + poll_time, max_poll_time + ).tell_proxy() return timer, cloud_node_poller, arvados_node_poller, job_queue_poller _caught_signals = {} @@ -96,6 +104,7 @@ def shutdown_signal(signal_code, frame): pykka.ActorRegistry.stop_all() sys.exit(-signal_code) elif current_count == 0: + watchdog.stop() node_daemon.shutdown() elif current_count == 1: pykka.ActorRegistry.stop_all() @@ -103,7 +112,7 @@ def shutdown_signal(signal_code, frame): sys.exit(-signal_code) def main(args=None): - global node_daemon + global node_daemon, watchdog args = parse_cli(args) config = load_config(args.config) @@ -112,15 +121,17 @@ def main(args=None): for sigcode in [signal.SIGINT, signal.SIGQUIT, signal.SIGTERM]: signal.signal(sigcode, shutdown_signal) + status.Server(config).start() + try: root_logger = setup_logging(config.get('Logging', 'file'), **config.log_levels()) - root_logger.info("%s %s, libcloud %s", sys.argv[0], __version__, libcloud.__version__) + root_logger.info("%s %s started, libcloud %s", sys.argv[0], __version__, libcloud.__version__) node_setup, node_shutdown, node_update, node_monitor = \ config.dispatch_classes() server_calculator = build_server_calculator(config) timer, cloud_node_poller, arvados_node_poller, job_queue_poller = \ launch_pollers(config, server_calculator) - cloud_node_updater = node_update.start(config.new_cloud_client).tell_proxy() + cloud_node_updater = node_update.start(config.new_cloud_client, timer).tell_proxy() node_daemon = NodeManagerDaemonActor.start( job_queue_poller, arvados_node_poller, cloud_node_poller, cloud_node_updater, timer, @@ -135,7 +146,7 @@ def main(args=None): node_setup, node_shutdown, node_monitor, max_total_price=config.getfloat('Daemon', 'max_total_price')).tell_proxy() - WatchdogActor.start(config.getint('Daemon', 'watchdog'), + watchdog = WatchdogActor.start(config.getint('Daemon', 'watchdog'), cloud_node_poller.actor_ref, arvados_node_poller.actor_ref, job_queue_poller.actor_ref,