Arvados-DCO-1.1-Signed-off-by: Radhika Chippada <radhika@curoverse.com>

[arvados.git] / services / api / app / models / node.rb

diff --git a/services/api/app/models/node.rb b/services/api/app/models/node.rb
index 18550204669c7cc6353d87cfc863bcbf3c4d876a..bf1b636c52836bd54c75373c29fde51897e4b766 100644 (file)
--- a/services/api/app/models/node.rb
+++ b/services/api/app/models/node.rb
@@ -1,3 +1,9 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+require 'tempfile'
+
 class Node < ArvadosModel
   include HasUuid
   include KindAndEtag
@@ -47,7 +53,7 @@ class Node < ArvadosModel
   def crunch_worker_state
     return 'down' if slot_number.nil?
     case self.info.andand['slurm_state']
-    when 'alloc', 'comp'
+    when 'alloc', 'comp', 'mix', 'drng'
       'busy'
     when 'idle'
       'idle'
@@ -102,17 +108,19 @@ class Node < ArvadosModel
 
     # Assign slot_number
     if self.slot_number.nil?
-      try_slot = 0
-      begin
-        self.slot_number = try_slot
+      while true
+        n = self.class.available_slot_number
+        if n.nil?
+          raise "No available node slots"
+        end
+        self.slot_number = n
         begin
           self.save!
           break
         rescue ActiveRecord::RecordNotUnique
-          try_slot += 1
+          # try again
         end
-        raise "No available node slots" if try_slot == Rails.configuration.max_compute_nodes
-      end while true
+      end
     end
 
     # Assign hostname
@@ -134,28 +142,41 @@ class Node < ArvadosModel
 
   protected
 
+  def self.available_slot_number
+    # Join the sequence 1..max with the nodes table. Return the first
+    # (i.e., smallest) value that doesn't match the slot_number of any
+    # existing node.
+    connection.exec_query('SELECT n FROM generate_series(1, $1) AS slot(n)
+                          LEFT JOIN nodes ON n=slot_number
+                          WHERE slot_number IS NULL
+                          LIMIT 1',
+                          # query label:
+                          'Node.available_slot_number',
+                          # [col_id, val] for $1 vars:
+                          [[nil, Rails.configuration.max_compute_nodes]],
+                         ).rows.first.andand.first
+  end
+
   def ensure_ping_secret
     self.info['ping_secret'] ||= rand(2**256).to_s(36)
   end
 
   def dns_server_update
-    if hostname_changed? && hostname_was
+    if ip_address_changed? && ip_address
+      Node.where('id != ? and ip_address = ?',
+                 id, ip_address).each do |stale_node|
+        # One or more(!) stale node records have the same IP address
+        # as the new node. Clear the ip_address field on the stale
+        # nodes. Otherwise, we (via SLURM) might inadvertently connect
+        # to the new node using the old node's hostname.
+        stale_node.update_attributes!(ip_address: nil)
+      end
+    end
+    if hostname_was && hostname_changed?
       self.class.dns_server_update(hostname_was, UNUSED_NODE_IP)
     end
-    if hostname_changed? or ip_address_changed?
-      if ip_address
-        Node.where('id != ? and ip_address = ? and last_ping_at < ?',
-                   id, ip_address, 10.minutes.ago).each do |stale_node|
-          # One or more stale compute node records have the same IP
-          # address as the new node.  Clear the ip_address field on
-          # the stale nodes.
-          stale_node.ip_address = nil
-          stale_node.save!
-        end
-      end
-      if hostname
-        self.class.dns_server_update(hostname, ip_address || UNUSED_NODE_IP)
-      end
+    if hostname && (hostname_changed? || ip_address_changed?)
+      self.class.dns_server_update(hostname, ip_address || UNUSED_NODE_IP)
     end
   end
 
@@ -173,22 +194,30 @@ class Node < ArvadosModel
     }
 
     if Rails.configuration.dns_server_conf_dir and Rails.configuration.dns_server_conf_template
+      tmpfile = nil
       begin
         begin
           template = IO.read(Rails.configuration.dns_server_conf_template)
-        rescue => e
+        rescue IOError, SystemCallError => e
           logger.error "Reading #{Rails.configuration.dns_server_conf_template}: #{e.message}"
           raise
         end
 
         hostfile = File.join Rails.configuration.dns_server_conf_dir, "#{hostname}.conf"
-        File.open hostfile+'.tmp', 'w' do |f|
+        Tempfile.open(["#{hostname}-", ".conf.tmp"],
+                                 Rails.configuration.dns_server_conf_dir) do |f|
+          tmpfile = f.path
           f.puts template % template_vars
         end
-        File.rename hostfile+'.tmp', hostfile
-      rescue => e
+        File.rename tmpfile, hostfile
+      rescue IOError, SystemCallError => e
         logger.error "Writing #{hostfile}: #{e.message}"
         ok = false
+      ensure
+        if tmpfile and File.file? tmpfile
+          # Cleanup remaining temporary file.
+          File.unlink tmpfile
+        end
       end
     end
 
@@ -207,7 +236,7 @@ class Node < ArvadosModel
           # Typically, this is used to trigger a dns server restart
           f.puts Rails.configuration.dns_server_reload_command
         end
-      rescue => e
+      rescue IOError, SystemCallError => e
         logger.error "Unable to write #{restartfile}: #{e.message}"
         ok = false
       end