X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/788ecdf8085f5e69cd3dc960f510b49f11432cb3..8595030d0314e6f88a245e66f90cce0a306b6867:/services/api/app/models/node.rb

diff --git a/services/api/app/models/node.rb b/services/api/app/models/node.rb
index db39ab658d..e470e4c2bd 100644
--- a/services/api/app/models/node.rb
+++ b/services/api/app/models/node.rb
@@ -5,7 +5,7 @@ class Node < ArvadosModel
   serialize :info, Hash
   serialize :properties, Hash
   before_validation :ensure_ping_secret
-  after_update :dnsmasq_update
+  after_update :dns_server_update
 
   # Only a controller can figure out whether or not the current API tokens
   # have access to the associated Job.  They're expected to set
@@ -13,11 +13,7 @@ class Node < ArvadosModel
   belongs_to(:job, foreign_key: :job_uuid, primary_key: :uuid)
   attr_accessor :job_readable
 
-  MAX_SLOTS = 64
-
-  @@confdir = Rails.configuration.dnsmasq_conf_dir
-  @@domain = Rails.configuration.compute_node_domain rescue `hostname --domain`.strip
-  @@nameservers = Rails.configuration.compute_node_nameservers
+  UNUSED_NODE_IP = '127.40.4.0'
 
   api_accessible :user, :extend => :common do |t|
     t.add :hostname
@@ -33,11 +29,11 @@ class Node < ArvadosModel
   api_accessible :superuser, :extend => :user do |t|
     t.add :first_ping_at
     t.add :info
-    t.add lambda { |x| @@nameservers }, :as => :nameservers
+    t.add lambda { |x| Rails.configuration.compute_node_nameservers }, :as => :nameservers
   end
 
   def domain
-    super || @@domain
+    super || Rails.configuration.compute_node_domain
   end
 
   def api_job_uuid
@@ -58,12 +54,12 @@ class Node < ArvadosModel
 
   def status
     if !self.last_ping_at
-      if Time.now - self.created_at > 5.minutes
+      if db_current_time - self.created_at > 5.minutes
         'startup-fail'
       else
         'pending'
       end
-    elsif Time.now - self.last_ping_at > 1.hours
+    elsif db_current_time - self.last_ping_at > 1.hours
       'missing'
     else
       'running'
@@ -77,7 +73,9 @@ class Node < ArvadosModel
       logger.info "Ping: secret mismatch: received \"#{o[:ping_secret]}\" != \"#{self.info['ping_secret']}\""
       raise ArvadosModel::UnauthorizedError.new("Incorrect ping_secret")
     end
-    self.last_ping_at = Time.now
+
+    current_time = db_current_time
+    self.last_ping_at = current_time
 
     @bypass_arvados_authorization = true
 
@@ -85,7 +83,7 @@ class Node < ArvadosModel
     if self.ip_address.nil?
       logger.info "#{self.uuid} ip_address= #{o[:ip]}"
       self.ip_address = o[:ip]
-      self.first_ping_at = Time.now
+      self.first_ping_at = current_time
     end
 
     # Record instance ID if not already known
@@ -98,7 +96,7 @@ class Node < ArvadosModel
       end
     end
 
-    # Assign hostname
+    # Assign slot_number
     if self.slot_number.nil?
       try_slot = 0
       begin
@@ -109,8 +107,12 @@ class Node < ArvadosModel
         rescue ActiveRecord::RecordNotUnique
           try_slot += 1
         end
-        raise "No available node slots" if try_slot == MAX_SLOTS
+        raise "No available node slots" if try_slot == Rails.configuration.max_compute_nodes
       end while true
+    end
+
+    # Assign hostname
+    if self.hostname.nil? and Rails.configuration.assign_node_hostname
       self.hostname = self.class.hostname_for_slot(self.slot_number)
     end
 
@@ -132,42 +134,105 @@ class Node < ArvadosModel
     self.info['ping_secret'] ||= rand(2**256).to_s(36)
   end
 
-  def dnsmasq_update
-    if self.hostname_changed? or self.ip_address_changed?
-      if self.hostname and self.ip_address
-        self.class.dnsmasq_update(self.hostname, self.ip_address)
+  def dns_server_update
+    if hostname_changed? && hostname_was
+      self.class.dns_server_update(hostname_was, UNUSED_NODE_IP)
+    end
+    if hostname_changed? or ip_address_changed?
+      if ip_address
+        Node.where('id != ? and ip_address = ? and last_ping_at < ?',
+                   id, ip_address, 10.minutes.ago).each do |stale_node|
+          # One or more stale compute node records have the same IP
+          # address as the new node.  Clear the ip_address field on
+          # the stale nodes.
+          stale_node.ip_address = nil
+          stale_node.save!
+        end
+      end
+      if hostname
+        self.class.dns_server_update(hostname, ip_address || UNUSED_NODE_IP)
       end
     end
   end
 
-  def self.dnsmasq_update(hostname, ip_address)
-    return unless @@confdir
+  def self.dns_server_update hostname, ip_address
+    ok = true
+
     ptr_domain = ip_address.
       split('.').reverse.join('.').concat('.in-addr.arpa')
-    hostfile = File.join @@confdir, hostname
-    File.open hostfile, 'w' do |f|
-      f.puts "address=/#{hostname}/#{ip_address}"
-      f.puts "address=/#{hostname}.#{@@domain}/#{ip_address}" if @@domain
-      f.puts "ptr-record=#{ptr_domain},#{hostname}"
+
+    template_vars = {
+      hostname: hostname,
+      uuid_prefix: Rails.configuration.uuid_prefix,
+      ip_address: ip_address,
+      ptr_domain: ptr_domain,
+    }
+
+    if Rails.configuration.dns_server_conf_dir and Rails.configuration.dns_server_conf_template
+      begin
+        begin
+          template = IO.read(Rails.configuration.dns_server_conf_template)
+        rescue => e
+          logger.error "Reading #{Rails.configuration.dns_server_conf_template}: #{e.message}"
+          raise
+        end
+
+        hostfile = File.join Rails.configuration.dns_server_conf_dir, "#{hostname}.conf"
+        File.open hostfile+'.tmp', 'w' do |f|
+          f.puts template % template_vars
+        end
+        File.rename hostfile+'.tmp', hostfile
+      rescue => e
+        logger.error "Writing #{hostfile}: #{e.message}"
+        ok = false
+      end
     end
-    File.open(File.join(@@confdir, 'restart.txt'), 'w') do |f|
-      # this should trigger a dnsmasq restart
+
+    if Rails.configuration.dns_server_update_command
+      cmd = Rails.configuration.dns_server_update_command % template_vars
+      if not system cmd
+        logger.error "dns_server_update_command #{cmd.inspect} failed: #{$?}"
+        ok = false
+      end
+    end
+
+    if Rails.configuration.dns_server_conf_dir and Rails.configuration.dns_server_reload_command
+      restartfile = File.join(Rails.configuration.dns_server_conf_dir, 'restart.txt')
+      begin
+        File.open(restartfile, 'w') do |f|
+          # Typically, this is used to trigger a dns server restart
+          f.puts Rails.configuration.dns_server_reload_command
+        end
+      rescue => e
+        logger.error "Unable to write #{restartfile}: #{e.message}"
+        ok = false
+      end
     end
+
+    ok
   end
 
   def self.hostname_for_slot(slot_number)
-    "compute#{slot_number}"
+    config = Rails.configuration.assign_node_hostname
+
+    return nil if !config
+
+    sprintf(config, {:slot_number => slot_number})
   end
 
   # At startup, make sure all DNS entries exist.  Otherwise, slurmctld
   # will refuse to start.
-  if @@confdir and
-      !File.exists? (File.join(@@confdir, hostname_for_slot(MAX_SLOTS-1)))
-    (0..MAX_SLOTS-1).each do |slot_number|
+  if Rails.configuration.dns_server_conf_dir and Rails.configuration.dns_server_conf_template and Rails.configuration.assign_node_hostname
+    (0..Rails.configuration.max_compute_nodes-1).each do |slot_number|
       hostname = hostname_for_slot(slot_number)
-      hostfile = File.join @@confdir, hostname
+      hostfile = File.join Rails.configuration.dns_server_conf_dir, "#{hostname}.conf"
       if !File.exists? hostfile
-        dnsmasq_update(hostname, '127.40.4.0')
+        n = Node.where(:slot_number => slot_number).first
+        if n.nil? or n.ip_address.nil?
+          dns_server_update(hostname, UNUSED_NODE_IP)
+        else
+          dns_server_update(hostname, n.ip_address)
+        end
       end
     end
   end