1 class Node < ArvadosModel
4 include CommonApiTemplate
6 serialize :properties, Hash
7 before_validation :ensure_ping_secret
8 after_update :dnsmasq_update
12 @@confdir = Rails.configuration.dnsmasq_conf_dir
13 @@domain = Rails.configuration.compute_node_domain rescue `hostname --domain`.strip
14 @@nameservers = Rails.configuration.compute_node_nameservers
16 api_accessible :user, :extend => :common do |t|
23 t.add :crunch_worker_state
26 api_accessible :superuser, :extend => :user do |t|
29 t.add lambda { |x| @@nameservers }, :as => :nameservers
36 def crunch_worker_state
37 case self.info.andand['slurm_state']
49 if Time.now - self.created_at > 5.minutes
54 elsif Time.now - self.last_ping_at > 1.hours
62 raise "must have :ip and :ping_secret" unless o[:ip] and o[:ping_secret]
64 if o[:ping_secret] != self.info['ping_secret']
65 logger.info "Ping: secret mismatch: received \"#{o[:ping_secret]}\" != \"#{self.info['ping_secret']}\""
66 raise ArvadosModel::UnauthorizedError.new("Incorrect ping_secret")
68 self.last_ping_at = Time.now
70 @bypass_arvados_authorization = true
73 if self.ip_address.nil?
74 logger.info "#{self.uuid} ip_address= #{o[:ip]}"
75 self.ip_address = o[:ip]
76 self.first_ping_at = Time.now
79 # Record instance ID if not already known
80 if o[:ec2_instance_id]
81 if !self.info['ec2_instance_id']
82 self.info['ec2_instance_id'] = o[:ec2_instance_id]
83 if (Rails.configuration.compute_node_ec2_tag_enable rescue true)
84 tag_cmd = ("ec2-create-tags #{o[:ec2_instance_id]} " +
85 "--tag 'Name=#{self.uuid}'")
88 elsif self.info['ec2_instance_id'] != o[:ec2_instance_id]
89 logger.debug "Multiple nodes have credentials for #{self.uuid}"
90 raise "#{self.uuid} is already running at #{self.info['ec2_instance_id']} so rejecting ping from #{o[:ec2_instance_id]}"
95 if self.slot_number.nil?
98 self.slot_number = try_slot
102 rescue ActiveRecord::RecordNotUnique
105 raise "No available node slots" if try_slot == MAX_SLOTS
107 self.hostname = self.class.hostname_for_slot(self.slot_number)
108 if info['ec2_instance_id']
109 if (Rails.configuration.compute_node_ec2_tag_enable rescue true)
110 `ec2-create-tags #{self.info['ec2_instance_id']} --tag 'hostname=#{self.hostname}'`
115 # Record other basic stats
116 ['total_cpu_cores', 'total_ram_mb', 'total_scratch_mb'].each do |key|
117 if value = (o[key] or o[key.to_sym])
118 self.properties[key] = value.to_i
120 self.properties.delete(key)
127 def start!(ping_url_method)
128 ensure_permission_to_save
129 ping_url = ping_url_method.call({ id: self.uuid, ping_secret: self.info['ping_secret'] })
130 if (Rails.configuration.compute_node_ec2run_args and
131 Rails.configuration.compute_node_ami)
132 ec2_args = ["--user-data '#{ping_url}'",
134 Rails.configuration.compute_node_ec2run_args,
135 Rails.configuration.compute_node_ami
137 ec2run_cmd = ["ec2-run-instances",
138 "--client-token", self.uuid,
139 ec2_args].flatten.join(' ')
140 ec2spot_cmd = ["ec2-request-spot-instances",
141 "-p #{Rails.configuration.compute_node_spot_bid} --type one-time",
142 ec2_args].flatten.join(' ')
147 self.info['ec2_run_command'] = ec2run_cmd
148 self.info['ec2_spot_command'] = ec2spot_cmd
149 self.info['ec2_start_command'] = ec2spot_cmd
150 logger.info "#{self.uuid} ec2_start_command= #{ec2spot_cmd.inspect}"
151 result = `#{ec2spot_cmd} 2>&1`
152 self.info['ec2_start_result'] = result
153 logger.info "#{self.uuid} ec2_start_result= #{result.inspect}"
154 result.match(/INSTANCE\s*(i-[0-9a-f]+)/) do |m|
156 self.info['ec2_instance_id'] = instance_id
157 if (Rails.configuration.compute_node_ec2_tag_enable rescue true)
158 `ec2-create-tags #{instance_id} --tag 'Name=#{self.uuid}'`
161 result.match(/SPOTINSTANCEREQUEST\s*(sir-[0-9a-f]+)/) do |m|
163 self.info['ec2_sir_id'] = sir_id
164 if (Rails.configuration.compute_node_ec2_tag_enable rescue true)
165 `ec2-create-tags #{sir_id} --tag 'Name=#{self.uuid}'`
173 def ensure_ping_secret
174 self.info['ping_secret'] ||= rand(2**256).to_s(36)
178 if self.hostname_changed? or self.ip_address_changed?
179 if self.hostname and self.ip_address
180 self.class.dnsmasq_update(self.hostname, self.ip_address)
185 def self.dnsmasq_update(hostname, ip_address)
186 return unless @@confdir
187 ptr_domain = ip_address.
188 split('.').reverse.join('.').concat('.in-addr.arpa')
189 hostfile = File.join @@confdir, hostname
190 File.open hostfile, 'w' do |f|
191 f.puts "address=/#{hostname}/#{ip_address}"
192 f.puts "address=/#{hostname}.#{@@domain}/#{ip_address}" if @@domain
193 f.puts "ptr-record=#{ptr_domain},#{hostname}"
195 File.open(File.join(@@confdir, 'restart.txt'), 'w') do |f|
196 # this should trigger a dnsmasq restart
200 def self.hostname_for_slot(slot_number)
201 "compute#{slot_number}"
204 # At startup, make sure all DNS entries exist. Otherwise, slurmctld
205 # will refuse to start.
207 !File.exists? (File.join(@@confdir, hostname_for_slot(MAX_SLOTS-1)))
208 (0..MAX_SLOTS-1).each do |slot_number|
209 hostname = hostname_for_slot(slot_number)
210 hostfile = File.join @@confdir, hostname
211 if !File.exists? hostfile
212 dnsmasq_update(hostname, '127.40.4.0')
217 def permission_to_update
218 @bypass_arvados_authorization or super
221 def permission_to_create
222 current_user and current_user.is_admin