+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+require 'tempfile'
+
class Node < ArvadosModel
include HasUuid
include KindAndEtag
include CommonApiTemplate
- serialize :info, Hash
- serialize :properties, Hash
+
+ # Posgresql JSONB columns should NOT be declared as serialized, Rails 5
+ # already know how to properly treat them.
+ attribute :properties, :jsonbHash, default: {}
+ attribute :info, :jsonbHash, default: {}
+
before_validation :ensure_ping_secret
after_update :dns_server_update
belongs_to(:job, foreign_key: :job_uuid, primary_key: :uuid)
attr_accessor :job_readable
+ UNUSED_NODE_IP = '127.40.4.0'
+
api_accessible :user, :extend => :common do |t|
t.add :hostname
t.add :domain
api_accessible :superuser, :extend => :user do |t|
t.add :first_ping_at
t.add :info
- t.add lambda { |x| Rails.configuration.compute_node_nameservers }, :as => :nameservers
+ t.add lambda { |x| Rails.configuration.Containers.SLURM.Managed.ComputeNodeNameservers.keys }, :as => :nameservers
end
- def initialize(*args)
- super
+ after_initialize do
@bypass_arvados_authorization = false
end
def domain
- super || Rails.configuration.compute_node_domain
+ super || Rails.configuration.Containers.SLURM.Managed.ComputeNodeDomain
end
def api_job_uuid
def crunch_worker_state
return 'down' if slot_number.nil?
case self.info.andand['slurm_state']
- when 'alloc', 'comp'
+ when 'alloc', 'comp', 'mix', 'drng'
'busy'
when 'idle'
'idle'
end
end
- # Assign slot_number
- if self.slot_number.nil?
- try_slot = 0
- begin
- self.slot_number = try_slot
- begin
- self.save!
- break
- rescue ActiveRecord::RecordNotUnique
- try_slot += 1
- end
- raise "No available node slots" if try_slot == Rails.configuration.max_compute_nodes
- end while true
- end
-
- # Assign hostname
- if self.hostname.nil? and Rails.configuration.assign_node_hostname
- self.hostname = self.class.hostname_for_slot(self.slot_number)
- end
+ assign_slot
# Record other basic stats
['total_cpu_cores', 'total_ram_mb', 'total_scratch_mb'].each do |key|
save!
end
+ def assign_slot
+ return if self.slot_number.andand > 0
+ while true
+ self.slot_number = self.class.available_slot_number
+ if self.slot_number.nil?
+ raise "No available node slots"
+ end
+ begin
+ save!
+ return assign_hostname
+ rescue ActiveRecord::RecordNotUnique
+ # try again
+ end
+ end
+ end
+
protected
+ def assign_hostname
+ if self.hostname.nil? and Rails.configuration.Containers.SLURM.Managed.AssignNodeHostname
+ self.hostname = self.class.hostname_for_slot(self.slot_number)
+ end
+ end
+
+ def self.available_slot_number
+ # Join the sequence 1..max with the nodes table. Return the first
+ # (i.e., smallest) value that doesn't match the slot_number of any
+ # existing node.
+ connection.exec_query('SELECT n FROM generate_series(1, $1) AS slot(n)
+ LEFT JOIN nodes ON n=slot_number
+ WHERE slot_number IS NULL
+ LIMIT 1',
+ # query label:
+ 'Node.available_slot_number',
+ # [col_id, val] for $1 vars:
+ [[nil, Rails.configuration.Containers.MaxComputeVMs]],
+ ).rows.first.andand.first
+ end
+
def ensure_ping_secret
self.info['ping_secret'] ||= rand(2**256).to_s(36)
end
def dns_server_update
- if self.hostname_changed? or self.ip_address_changed?
- if not self.ip_address.nil?
- stale_conflicting_nodes = Node.where('id != ? and ip_address = ? and last_ping_at < ?',self.id,self.ip_address,10.minutes.ago)
- if not stale_conflicting_nodes.empty?
- # One or more stale compute node records have the same IP address as the new node.
- # Clear the ip_address field on the stale nodes.
- stale_conflicting_nodes.each do |stale_node|
- stale_node.ip_address = nil
- stale_node.save!
- end
- end
- end
- if self.hostname and self.ip_address
- self.class.dns_server_update(self.hostname, self.ip_address)
+ if saved_change_to_ip_address? && ip_address
+ Node.where('id != ? and ip_address = ?',
+ id, ip_address).each do |stale_node|
+ # One or more(!) stale node records have the same IP address
+ # as the new node. Clear the ip_address field on the stale
+ # nodes. Otherwise, we (via SLURM) might inadvertently connect
+ # to the new node using the old node's hostname.
+ stale_node.update_attributes!(ip_address: nil)
end
end
+ if hostname_before_last_save && saved_change_to_hostname?
+ self.class.dns_server_update(hostname_before_last_save, UNUSED_NODE_IP)
+ end
+ if hostname && (saved_change_to_hostname? || saved_change_to_ip_address?)
+ self.class.dns_server_update(hostname, ip_address || UNUSED_NODE_IP)
+ end
end
def self.dns_server_update hostname, ip_address
template_vars = {
hostname: hostname,
- uuid_prefix: Rails.configuration.uuid_prefix,
+ uuid_prefix: Rails.configuration.ClusterID,
ip_address: ip_address,
ptr_domain: ptr_domain,
}
- if Rails.configuration.dns_server_conf_dir and Rails.configuration.dns_server_conf_template
+ if (!Rails.configuration.Containers.SLURM.Managed.DNSServerConfDir.to_s.empty? and
+ !Rails.configuration.Containers.SLURM.Managed.DNSServerConfTemplate.to_s.empty?)
+ tmpfile = nil
begin
begin
- template = IO.read(Rails.configuration.dns_server_conf_template)
- rescue => e
- logger.error "Reading #{Rails.configuration.dns_server_conf_template}: #{e.message}"
+ template = IO.read(Rails.configuration.Containers.SLURM.Managed.DNSServerConfTemplate)
+ rescue IOError, SystemCallError => e
+ logger.error "Reading #{Rails.configuration.Containers.SLURM.Managed.DNSServerConfTemplate}: #{e.message}"
raise
end
- hostfile = File.join Rails.configuration.dns_server_conf_dir, "#{hostname}.conf"
- File.open hostfile+'.tmp', 'w' do |f|
+ hostfile = File.join Rails.configuration.Containers.SLURM.Managed.DNSServerConfDir, "#{hostname}.conf"
+ Tempfile.open(["#{hostname}-", ".conf.tmp"],
+ Rails.configuration.Containers.SLURM.Managed.DNSServerConfDir) do |f|
+ tmpfile = f.path
f.puts template % template_vars
end
- File.rename hostfile+'.tmp', hostfile
- rescue => e
+ File.rename tmpfile, hostfile
+ rescue IOError, SystemCallError => e
logger.error "Writing #{hostfile}: #{e.message}"
ok = false
+ ensure
+ if tmpfile and File.file? tmpfile
+ # Cleanup remaining temporary file.
+ File.unlink tmpfile
+ end
end
end
- if Rails.configuration.dns_server_update_command
- cmd = Rails.configuration.dns_server_update_command % template_vars
+ if !Rails.configuration.Containers.SLURM.Managed.DNSServerUpdateCommand.empty?
+ cmd = Rails.configuration.Containers.SLURM.Managed.DNSServerUpdateCommand % template_vars
if not system cmd
logger.error "dns_server_update_command #{cmd.inspect} failed: #{$?}"
ok = false
end
end
- if Rails.configuration.dns_server_conf_dir and Rails.configuration.dns_server_reload_command
- restartfile = File.join(Rails.configuration.dns_server_conf_dir, 'restart.txt')
+ if (!Rails.configuration.Containers.SLURM.Managed.DNSServerConfDir.to_s.empty? and
+ !Rails.configuration.Containers.SLURM.Managed.DNSServerReloadCommand.to_s.empty?)
+ restartfile = File.join(Rails.configuration.Containers.SLURM.Managed.DNSServerConfDir, 'restart.txt')
begin
File.open(restartfile, 'w') do |f|
# Typically, this is used to trigger a dns server restart
- f.puts Rails.configuration.dns_server_reload_command
+ f.puts Rails.configuration.Containers.SLURM.Managed.DNSServerReloadCommand
end
- rescue => e
+ rescue IOError, SystemCallError => e
logger.error "Unable to write #{restartfile}: #{e.message}"
ok = false
end
end
def self.hostname_for_slot(slot_number)
- config = Rails.configuration.assign_node_hostname
+ config = Rails.configuration.Containers.SLURM.Managed.AssignNodeHostname
return nil if !config
# At startup, make sure all DNS entries exist. Otherwise, slurmctld
# will refuse to start.
- if Rails.configuration.dns_server_conf_dir and Rails.configuration.dns_server_conf_template and Rails.configuration.assign_node_hostname
- (0..Rails.configuration.max_compute_nodes-1).each do |slot_number|
+ if (!Rails.configuration.Containers.SLURM.Managed.DNSServerConfDir.to_s.empty? and
+ !Rails.configuration.Containers.SLURM.Managed.DNSServerConfTemplate.to_s.empty? and
+ !Rails.configuration.Containers.SLURM.Managed.AssignNodeHostname.empty?)
+
+ (0..Rails.configuration.Containers.MaxComputeVMs-1).each do |slot_number|
hostname = hostname_for_slot(slot_number)
- hostfile = File.join Rails.configuration.dns_server_conf_dir, "#{hostname}.conf"
- if !File.exists? hostfile
+ hostfile = File.join Rails.configuration.Containers.SLURM.Managed.DNSServerConfDir, "#{hostname}.conf"
+ if !File.exist? hostfile
n = Node.where(:slot_number => slot_number).first
if n.nil? or n.ip_address.nil?
- dns_server_update(hostname, '127.40.4.0')
+ dns_server_update(hostname, UNUSED_NODE_IP)
else
dns_server_update(hostname, n.ip_address)
end
projects / arvados.git / blobdiff