# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: AGPL-3.0

# EC2 configuration for Arvados Node Manager.
# All times are in seconds unless specified otherwise.

[Manage]
# The management server responds to http://addr:port/status.json with
# a snapshot of internal state.

# Management server listening address (default 127.0.0.1)
#address = 0.0.0.0

# Management server port number (default -1, server is disabled)
#port = 8989

[Daemon]
# The dispatcher can customize the start and stop procedure for
# cloud nodes.  For example, the SLURM dispatcher drains nodes
# through SLURM before shutting them down.
#dispatcher = slurm

# Node Manager will ensure that there are at least this many nodes running at
# all times.  If node manager needs to start new idle nodes for the purpose of
# satisfying min_nodes, it will use the cheapest node type.  However, depending
# on usage patterns, it may also satisfy min_nodes by keeping alive some
# more-expensive nodes
min_nodes = 0

# Node Manager will not start any compute nodes when at least this
# many are running.
max_nodes = 8

# Upper limit on rate of spending (in $/hr), will not boot additional nodes
# if total price of already running nodes meets or exceeds this threshold.
# default 0 means no limit.
max_total_price = 0

# Poll EC2 nodes and Arvados for new information every N seconds.
poll_time = 60

# Polls have exponential backoff when services fail to respond.
# This is the longest time to wait between polls.
max_poll_time = 300

# If Node Manager can't succesfully poll a service for this long,
# it will never start or stop compute nodes, on the assumption that its
# information is too outdated.
poll_stale_after = 600

# If Node Manager boots a cloud node, and it does not pair with an Arvados
# node before this long, assume that there was a cloud bootstrap failure and
# shut it down.  Note that normal shutdown windows apply (see the Cloud
# section), so this should be shorter than the first shutdown window value.
boot_fail_after = 1800

# "Node stale time" affects two related behaviors.
# 1. If a compute node has been running for at least this long, but it
# isn't paired with an Arvados node, do not shut it down, but leave it alone.
# This prevents the node manager from shutting down a node that might
# actually be doing work, but is having temporary trouble contacting the
# API server.
# 2. When the Node Manager starts a new compute node, it will try to reuse
# an Arvados node that hasn't been updated for this long.
node_stale_after = 14400

# Number of consecutive times a node must report as "idle" before it
# will be considered eligible for shutdown.  Node status is checked
# each poll period, and node can go idle at any point during a poll
# period (meaning a node could be reported as idle that has only been
# idle for 1 second).  With a 60 second poll period, three consecutive
# status updates of "idle" suggests the node has been idle at least
# 121 seconds.
consecutive_idle_count = 3

# Scaling factor to be applied to nodes' available RAM size. Usually there's a
# variable discrepancy between the advertised RAM value on cloud nodes and the
# actual amount available.
# If not set, this value will be set to 0.95
node_mem_scaling = 0.95

# File path for Certificate Authorities
certs_file = /etc/ssl/certs/ca-certificates.crt

[Logging]
# Log file path
file = /var/log/arvados/node-manager.log

# Log level for most Node Manager messages.
# Choose one of DEBUG, INFO, WARNING, ERROR, or CRITICAL.
# WARNING lets you know when polling a service fails.
# INFO additionally lets you know when a compute node is started or stopped.
level = INFO

# You can also set different log levels for specific libraries.
# Pykka is the Node Manager's actor library.
# Setting this to DEBUG will display tracebacks for uncaught
# exceptions in the actors, but it's also very chatty.
pykka = WARNING

# Setting apiclient to INFO will log the URL of every Arvados API request.
apiclient = WARNING

[Arvados]
host = zyxwv.arvadosapi.com
token = ARVADOS_TOKEN
timeout = 15
jobs_queue = yes   # Get work request from Arvados jobs queue (jobs API)
slurm_queue = yes  # Get work request from squeue (containers API)

# Accept an untrusted SSL certificate from the API server?
insecure = no

[Cloud]
provider = ec2

# It's usually most cost-effective to shut down compute nodes during narrow
# windows of time.  For example, EC2 bills each node by the hour, so the best
# time to shut down a node is right before a new hour of uptime starts.
# Shutdown windows define these periods of time.  These are windows in
# full minutes, separated by commas.  Counting from the time the node is
# booted, the node WILL NOT shut down for N1 minutes; then it MAY shut down
# for N2 minutes; then it WILL NOT shut down for N3 minutes; and so on.
# For example, "54, 5, 1" means the node may shut down from the 54th to the
# 59th minute of each hour of uptime.
# Specify at least two windows.  You can add as many as you need beyond that.
shutdown_windows = 54, 5, 1

[Cloud Credentials]
key = KEY
secret = SECRET_KEY
region = us-east-1
timeout = 60

[Cloud List]
# This section defines filters that find compute nodes.
# Tags that you specify here will automatically be added to nodes you create.
# Replace colons in Amazon filters with underscores
# (e.g., write "tag:mytag" as "tag_mytag").
instance-state-name = running
tag_arvados-class = dynamic-compute
tag_cluster = zyxwv

[Cloud Create]
# New compute nodes will send pings to Arvados at this host.
# You may specify a port, and use brackets to disambiguate IPv6 addresses.
ping_host = hostname:port

# Give the name of an SSH key on AWS...
ex_keyname = string

# ... or a file path for an SSH key that can log in to the compute node.
# (One or the other, not both.)
# ssh_key = path

# The EC2 IDs of the image and subnet compute nodes should use.
image_id = idstring
subnet_id = idstring

# Comma-separated EC2 IDs for the security group(s) assigned to each
# compute node.
security_groups = idstring1, idstring2

# Apply an Instance Profile ARN to the newly created compute nodes
# For more info, see:
# https://aws.amazon.com/premiumsupport/knowledge-center/iam-policy-restrict-vpc/
# ex_iamprofile = arn:aws:iam::ACCOUNTNUMBER:instance-profile/ROLENAME


# You can define any number of Size sections to list EC2 sizes you're
# willing to use.  The Node Manager should boot the cheapest size(s) that
# can run jobs in the queue.
#
# Each size section MUST define the number of cores are available in this
# size class (since libcloud does not provide any consistent API for exposing
# this setting).
# You may also want to define the amount of scratch space (expressed
# in MB) for Crunch jobs.  You can also override Amazon's provided
# data fields (such as price per hour) by setting them here.
#
# Additionally, you can ask for a preemptible instance (AWS's spot instance)
# by adding the appropriate boolean configuration flag. If you want to have
# both spot & reserved versions of the same size, you can do so by renaming
# the Size section and specifying the instance type inside it.

# 100 GB scratch space
[Size m4.large]
cores = 2
price = 0.126
scratch = 100000

# 10 GB scratch space
[Size m4.large.spot]
instance_type = m4.large
preemptible = true
cores = 2
price = 0.126
scratch = 10000

# 200 GB scratch space
[Size m4.xlarge]
cores = 4
price = 0.252
scratch = 200000