Allow users to cancel a running crunch job by updating cancelled_at

[arvados.git] / services / api / script / crunch-dispatch.rb

diff --git a/services/api/script/crunch-dispatch.rb b/services/api/script/crunch-dispatch.rb
index 88840d53b69a4358e6686b7e3f99b154dd5ddc31..dff68d867e668dbee9cd3f088d8a88319360016f 100755 (executable)
--- a/services/api/script/crunch-dispatch.rb
+++ b/services/api/script/crunch-dispatch.rb
@@ -2,6 +2,7 @@
 
 include Process
 
+$warned = {}
 $signal = {}
 %w{TERM INT}.each do |sig|
   signame = sig
@@ -10,6 +11,10 @@ $signal = {}
     $signal[:term] = true
   end
 end
+Signal.trap('HUP') do
+  $stderr.puts "Received HUP signal"
+  $signal[:hup] = true
+end
 
 if ENV["CRUNCH_DISPATCH_LOCKFILE"]
   lockfilename = ENV.delete "CRUNCH_DISPATCH_LOCKFILE"
@@ -309,6 +314,30 @@ class Dispatcher
           end
         end
       else
+        if File.exists?(Rails.configuration.crunch_dispatch_hup_trigger)
+          begin
+            File.unlink(Rails.configuration.crunch_dispatch_hup_trigger)
+            $signal[:hup] = true
+          rescue Errno::ENOENT
+            $stderr.puts "Weird, hup_trigger file was deleted by someone else."
+          rescue Errno::EPERM
+            if not $warned[:hup_trigger_perm]
+              $warned[:hup_trigger_perm] = true
+              $stderr.puts "Install problem: I see the hup_trigger file but cannot delete it."
+            end
+          end
+        end
+        if $signal[:hup]
+          # Pass HUP through to all crunch-job processes.
+          @running.each do |uuid, j|
+            begin
+              Process.kill 'HUP', j[:wait_thr].pid
+            rescue Errno::ESRCH
+              # Process ended but hasn't been reaped. Nothing to do.
+            end
+          end
+          $signal.delete :hup
+        end
         refresh_todo unless did_recently(:refresh_todo, 1.0)
         update_node_status
         start_jobs unless @todo.empty? or did_recently(:start_jobs, 1.0)