1 class Node < ArvadosModel
4 include CommonApiTemplate
6 before_validation :ensure_ping_secret
7 after_update :dnsmasq_update
11 @@confdir = Rails.configuration.dnsmasq_conf_dir
12 @@domain = Rails.configuration.compute_node_domain rescue `hostname --domain`.strip
13 @@nameservers = Rails.configuration.compute_node_nameservers
15 api_accessible :user, :extend => :common do |t|
22 t.add :crunch_worker_state
25 api_accessible :superuser, :extend => :user do |t|
27 t.add lambda { |x| @@nameservers }, :as => :nameservers
31 if current_user.andand.current_user.is_admin
34 super.select { |k| not k.to_s.include? "secret" }
42 def crunch_worker_state
43 case self.info.andand['slurm_state']
55 if Time.now - self.created_at > 5.minutes
60 elsif Time.now - self.last_ping_at > 1.hours
68 raise "must have :ip and :ping_secret" unless o[:ip] and o[:ping_secret]
70 if o[:ping_secret] != self.info['ping_secret']
71 logger.info "Ping: secret mismatch: received \"#{o[:ping_secret]}\" != \"#{self.info['ping_secret']}\""
72 raise ArvadosModel::UnauthorizedError.new("Incorrect ping_secret")
74 self.last_ping_at = Time.now
76 @bypass_arvados_authorization = true
79 if self.ip_address.nil?
80 logger.info "#{self.uuid} ip_address= #{o[:ip]}"
81 self.ip_address = o[:ip]
82 self.first_ping_at = Time.now
85 # Record instance ID if not already known
86 if o[:ec2_instance_id]
87 if !self.info['ec2_instance_id']
88 self.info['ec2_instance_id'] = o[:ec2_instance_id]
89 if (Rails.configuration.compute_node_ec2_tag_enable rescue true)
90 tag_cmd = ("ec2-create-tags #{o[:ec2_instance_id]} " +
91 "--tag 'Name=#{self.uuid}'")
94 elsif self.info['ec2_instance_id'] != o[:ec2_instance_id]
95 logger.debug "Multiple nodes have credentials for #{self.uuid}"
96 raise "#{self.uuid} is already running at #{self.info['ec2_instance_id']} so rejecting ping from #{o[:ec2_instance_id]}"
101 if self.slot_number.nil?
104 self.slot_number = try_slot
108 rescue ActiveRecord::RecordNotUnique
111 raise "No available node slots" if try_slot == MAX_SLOTS
113 self.hostname = self.class.hostname_for_slot(self.slot_number)
114 if info['ec2_instance_id']
115 if (Rails.configuration.compute_node_ec2_tag_enable rescue true)
116 `ec2-create-tags #{self.info['ec2_instance_id']} --tag 'hostname=#{self.hostname}'`
121 # Record other basic stats
122 ['total_cpu_cores', 'total_ram_mb', 'total_scratch_mb'].each do |key|
123 if value = (o[key] or o[key.to_sym])
124 self.info[key] = value
126 self.info.delete(key)
133 def start!(ping_url_method)
134 ensure_permission_to_save
135 ping_url = ping_url_method.call({ id: self.uuid, ping_secret: self.info['ping_secret'] })
136 if (Rails.configuration.compute_node_ec2run_args and
137 Rails.configuration.compute_node_ami)
138 ec2_args = ["--user-data '#{ping_url}'",
140 Rails.configuration.compute_node_ec2run_args,
141 Rails.configuration.compute_node_ami
143 ec2run_cmd = ["ec2-run-instances",
144 "--client-token", self.uuid,
145 ec2_args].flatten.join(' ')
146 ec2spot_cmd = ["ec2-request-spot-instances",
147 "-p #{Rails.configuration.compute_node_spot_bid} --type one-time",
148 ec2_args].flatten.join(' ')
153 self.info['ec2_run_command'] = ec2run_cmd
154 self.info['ec2_spot_command'] = ec2spot_cmd
155 self.info['ec2_start_command'] = ec2spot_cmd
156 logger.info "#{self.uuid} ec2_start_command= #{ec2spot_cmd.inspect}"
157 result = `#{ec2spot_cmd} 2>&1`
158 self.info['ec2_start_result'] = result
159 logger.info "#{self.uuid} ec2_start_result= #{result.inspect}"
160 result.match(/INSTANCE\s*(i-[0-9a-f]+)/) do |m|
162 self.info['ec2_instance_id'] = instance_id
163 if (Rails.configuration.compute_node_ec2_tag_enable rescue true)
164 `ec2-create-tags #{instance_id} --tag 'Name=#{self.uuid}'`
167 result.match(/SPOTINSTANCEREQUEST\s*(sir-[0-9a-f]+)/) do |m|
169 self.info['ec2_sir_id'] = sir_id
170 if (Rails.configuration.compute_node_ec2_tag_enable rescue true)
171 `ec2-create-tags #{sir_id} --tag 'Name=#{self.uuid}'`
179 def ensure_ping_secret
180 self.info['ping_secret'] ||= rand(2**256).to_s(36)
184 if self.hostname_changed? or self.ip_address_changed?
185 if self.hostname and self.ip_address
186 self.class.dnsmasq_update(self.hostname, self.ip_address)
191 def self.dnsmasq_update(hostname, ip_address)
192 return unless @@confdir
193 ptr_domain = ip_address.
194 split('.').reverse.join('.').concat('.in-addr.arpa')
195 hostfile = File.join @@confdir, hostname
196 File.open hostfile, 'w' do |f|
197 f.puts "address=/#{hostname}/#{ip_address}"
198 f.puts "address=/#{hostname}.#{@@domain}/#{ip_address}" if @@domain
199 f.puts "ptr-record=#{ptr_domain},#{hostname}"
201 File.open(File.join(@@confdir, 'restart.txt'), 'w') do |f|
202 # this should trigger a dnsmasq restart
206 def self.hostname_for_slot(slot_number)
207 "compute#{slot_number}"
210 # At startup, make sure all DNS entries exist. Otherwise, slurmctld
211 # will refuse to start.
213 !File.exists? (File.join(@@confdir, hostname_for_slot(MAX_SLOTS-1)))
214 (0..MAX_SLOTS-1).each do |slot_number|
215 hostname = hostname_for_slot(slot_number)
216 hostfile = File.join @@confdir, hostname
217 if !File.exists? hostfile
218 dnsmasq_update(hostname, '127.40.4.0')
223 def permission_to_update
224 @bypass_arvados_authorization or super
227 def permission_to_create
228 current_user and current_user.is_admin