1 class Node < ArvadosModel
4 include CommonApiTemplate
6 before_validation :ensure_ping_secret
7 after_update :dnsmasq_update
11 @@confdir = if Rails.configuration.respond_to? :dnsmasq_conf_dir
12 Rails.configuration.dnsmasq_conf_dir
13 elsif File.exists? '/etc/dnsmasq.d/.'
18 @@domain = Rails.configuration.compute_node_domain rescue `hostname --domain`.strip
19 @@nameservers = Rails.configuration.compute_node_nameservers
21 api_accessible :user, :extend => :common do |t|
28 t.add :crunch_worker_state
30 api_accessible :superuser, :extend => :user do |t|
33 t.add lambda { |x| @@nameservers }, :as => :nameservers
45 def crunch_worker_state
46 case self.info.andand[:slurm_state]
58 if Time.now - self.created_at > 5.minutes
63 elsif Time.now - self.last_ping_at > 1.hours
71 raise "must have :ip and :ping_secret" unless o[:ip] and o[:ping_secret]
73 if o[:ping_secret] != self.info[:ping_secret]
74 logger.info "Ping: secret mismatch: received \"#{o[:ping_secret]}\" != \"#{self.info[:ping_secret]}\""
77 self.last_ping_at = Time.now
79 @bypass_arvados_authorization = true
82 if self.ip_address.nil?
83 logger.info "#{self.uuid} ip_address= #{o[:ip]}"
84 self.ip_address = o[:ip]
85 self.first_ping_at = Time.now
88 # Record instance ID if not already known
89 if o[:ec2_instance_id]
90 if !self.info[:ec2_instance_id]
91 self.info[:ec2_instance_id] = o[:ec2_instance_id]
92 `ec2-create-tags #{o[:ec2_instance_id]} --tag 'Name=#{self.uuid}'`
93 elsif self.info[:ec2_instance_id] != o[:ec2_instance_id]
94 logger.debug "Multiple nodes have credentials for #{self.uuid}"
95 raise "#{self.uuid} is already running at #{self.info[:ec2_instance_id]} so rejecting ping from #{o[:ec2_instance_id]}"
100 if self.slot_number.nil?
103 self.slot_number = try_slot
107 rescue ActiveRecord::RecordNotUnique
110 raise "No available node slots" if try_slot == MAX_SLOTS
112 self.hostname = self.class.hostname_for_slot(self.slot_number)
113 if info[:ec2_instance_id]
114 `ec2-create-tags #{self.info[:ec2_instance_id]} --tag 'hostname=#{self.hostname}'`
121 def start!(ping_url_method)
122 ensure_permission_to_update
123 ping_url = ping_url_method.call({ uuid: self.uuid, ping_secret: self.info[:ping_secret] })
124 ec2_args = ["--user-data '#{ping_url}'",
126 Rails.configuration.compute_node_ec2run_args,
127 Rails.configuration.compute_node_ami
129 ec2run_cmd = ["ec2-run-instances",
130 "--client-token", self.uuid,
131 ec2_args].flatten.join(' ')
132 ec2spot_cmd = ["ec2-request-spot-instances",
133 "-p #{Rails.configuration.compute_node_spot_bid} --type one-time",
134 ec2_args].flatten.join(' ')
135 self.info[:ec2_run_command] = ec2run_cmd
136 self.info[:ec2_spot_command] = ec2spot_cmd
137 self.info[:ec2_start_command] = ec2spot_cmd
138 logger.info "#{self.uuid} ec2_start_command= #{ec2spot_cmd.inspect}"
139 result = `#{ec2spot_cmd} 2>&1`
140 self.info[:ec2_start_result] = result
141 logger.info "#{self.uuid} ec2_start_result= #{result.inspect}"
142 result.match(/INSTANCE\s*(i-[0-9a-f]+)/) do |m|
144 self.info[:ec2_instance_id] = instance_id
145 `ec2-create-tags #{instance_id} --tag 'Name=#{self.uuid}'`
147 result.match(/SPOTINSTANCEREQUEST\s*(sir-[0-9a-f]+)/) do |m|
149 self.info[:ec2_sir_id] = sir_id
150 `ec2-create-tags #{sir_id} --tag 'Name=#{self.uuid}'`
157 def ensure_ping_secret
158 self.info[:ping_secret] ||= rand(2**256).to_s(36)
162 if self.hostname_changed? or self.ip_address_changed?
163 if self.hostname and self.ip_address
164 self.class.dnsmasq_update(self.hostname, self.ip_address)
169 def self.dnsmasq_update(hostname, ip_address)
170 return unless @@confdir
171 ptr_domain = ip_address.
172 split('.').reverse.join('.').concat('.in-addr.arpa')
173 hostfile = File.join @@confdir, hostname
174 File.open hostfile, 'w' do |f|
175 f.puts "address=/#{hostname}/#{ip_address}"
176 f.puts "address=/#{hostname}.#{@@domain}/#{ip_address}" if @@domain
177 f.puts "ptr-record=#{ptr_domain},#{hostname}"
179 File.open(File.join(@@confdir, 'restart.txt'), 'w') do |f|
180 # this should trigger a dnsmasq restart
184 def self.hostname_for_slot(slot_number)
185 "compute#{slot_number}"
188 # At startup, make sure all DNS entries exist. Otherwise, slurmctld
189 # will refuse to start.
191 !File.exists? (File.join(@@confdir, hostname_for_slot(MAX_SLOTS-1)))
192 (0..MAX_SLOTS-1).each do |slot_number|
193 hostname = hostname_for_slot(slot_number)
194 hostfile = File.join @@confdir, hostname
195 if !File.exists? hostfile
196 dnsmasq_update(hostname, '127.40.4.0')
201 def permission_to_update
202 @bypass_arvados_authorization or super
205 def permission_to_create
206 current_user and current_user.is_admin