10868: Remove old DNS entry immediately when a new node re-uses an old node's IP...

[arvados.git] / services / api / app / models / node.rb

diff --git a/services/api/app/models/node.rb b/services/api/app/models/node.rb
index 1cd6387cff37da82f23eeaf0d040ca64c57bf14e..e0cdda15628fb78514beaa32269f22ed8793b6e3 100644 (file)
--- a/services/api/app/models/node.rb
+++ b/services/api/app/models/node.rb
@@ -13,6 +13,8 @@ class Node < ArvadosModel
   belongs_to(:job, foreign_key: :job_uuid, primary_key: :uuid)
   attr_accessor :job_readable
 
+  UNUSED_NODE_IP = '127.40.4.0'
+
   api_accessible :user, :extend => :common do |t|
     t.add :hostname
     t.add :domain
@@ -30,6 +32,10 @@ class Node < ArvadosModel
     t.add lambda { |x| Rails.configuration.compute_node_nameservers }, :as => :nameservers
   end
 
+  after_initialize do
+    @bypass_arvados_authorization = false
+  end
+
   def domain
     super || Rails.configuration.compute_node_domain
   end
@@ -94,7 +100,7 @@ class Node < ArvadosModel
       end
     end
 
-    # Assign hostname
+    # Assign slot_number
     if self.slot_number.nil?
       try_slot = 0
       begin
@@ -107,6 +113,10 @@ class Node < ArvadosModel
         end
         raise "No available node slots" if try_slot == Rails.configuration.max_compute_nodes
       end while true
+    end
+
+    # Assign hostname
+    if self.hostname.nil? and Rails.configuration.assign_node_hostname
       self.hostname = self.class.hostname_for_slot(self.slot_number)
     end
 
@@ -129,22 +139,22 @@ class Node < ArvadosModel
   end
 
   def dns_server_update
-    if self.hostname_changed? or self.ip_address_changed?
-      if not self.ip_address.nil?
-        stale_conflicting_nodes = Node.where('id != ? and ip_address = ? and last_ping_at < ?',self.id,self.ip_address,10.minutes.ago)
-        if not stale_conflicting_nodes.empty?
-          # One or more stale compute node records have the same IP address as the new node.
-          # Clear the ip_address field on the stale nodes.
-          stale_conflicting_nodes.each do |stale_node|
-            stale_node.ip_address = nil
-            stale_node.save!
-          end
-        end
-      end
-      if self.hostname and self.ip_address
-        self.class.dns_server_update(self.hostname, self.ip_address)
+    if ip_address_changed? && ip_address
+      Node.where('id != ? and ip_address = ?',
+                 id, ip_address).each do |stale_node|
+        # One or more(!) stale node records have the same IP address
+        # as the new node. Clear the ip_address field on the stale
+        # nodes. Otherwise, we (via SLURM) might inadvertently connect
+        # to the new node using the old node's hostname.
+        stale_node.update_attributes!(ip_address: nil)
       end
     end
+    if hostname_was && hostname_changed?
+      self.class.dns_server_update(hostname_was, UNUSED_NODE_IP)
+    end
+    if hostname && (hostname_changed? || ip_address_changed?)
+      self.class.dns_server_update(hostname, ip_address || UNUSED_NODE_IP)
+    end
   end
 
   def self.dns_server_update hostname, ip_address
@@ -205,19 +215,23 @@ class Node < ArvadosModel
   end
 
   def self.hostname_for_slot(slot_number)
-    "compute#{slot_number}"
+    config = Rails.configuration.assign_node_hostname
+
+    return nil if !config
+
+    sprintf(config, {:slot_number => slot_number})
   end
 
   # At startup, make sure all DNS entries exist.  Otherwise, slurmctld
   # will refuse to start.
-  if Rails.configuration.dns_server_conf_dir and Rails.configuration.dns_server_conf_template
+  if Rails.configuration.dns_server_conf_dir and Rails.configuration.dns_server_conf_template and Rails.configuration.assign_node_hostname
     (0..Rails.configuration.max_compute_nodes-1).each do |slot_number|
       hostname = hostname_for_slot(slot_number)
       hostfile = File.join Rails.configuration.dns_server_conf_dir, "#{hostname}.conf"
-      if !File.exists? hostfile
+      if !File.exist? hostfile
         n = Node.where(:slot_number => slot_number).first
         if n.nil? or n.ip_address.nil?
-          dns_server_update(hostname, '127.40.4.0')
+          dns_server_update(hostname, UNUSED_NODE_IP)
         else
           dns_server_update(hostname, n.ip_address)
         end