Merge branch 'master' into 3140-project-content-tabs

[arvados.git] / services / api / script / crunch-dispatch.rb

diff --git a/services/api/script/crunch-dispatch.rb b/services/api/script/crunch-dispatch.rb
index 6f3aeda413bac1b5e8a2ce29bf6a842d8467a871..5a990f0cb41ad30bd8832399a0d40c1839f2460e 100755 (executable)
--- a/services/api/script/crunch-dispatch.rb
+++ b/services/api/script/crunch-dispatch.rb
@@ -78,7 +78,7 @@ class Dispatcher
           # update our database (and cache) when a node's state changes
           if @node_state[re[1]] != re[2]
             @node_state[re[1]] = re[2]
-            node = Node.where('hostname=?', re[1]).first
+            node = Node.where('hostname=?', re[1]).order(:last_ping_at).last
             if node
               $stderr.puts "dispatch: update #{re[1]} state to #{re[2]}"
               node.info['slurm_state'] = re[2]
@@ -166,6 +166,10 @@ class Dispatcher
       cmd_args = nil
       case Server::Application.config.crunch_job_wrapper
       when :none
+        if @running.size > 0
+            # Don't run more than one at a time.
+            return
+        end
         cmd_args = []
       when :slurm_immediate
         nodelist = nodes_available_for_job(job)
@@ -226,7 +230,7 @@ class Dispatcher
         next
       end
 
-      $stderr.puts `cd #{arvados_internal.shellescape} && git fetch --no-tags #{src_repo.shellescape} && git tag #{job.uuid.shellescape} #{job.script_version.shellescape}`
+      $stderr.puts `cd #{arvados_internal.shellescape} && git fetch-pack --all #{src_repo.shellescape} && git tag #{job.uuid.shellescape} #{job.script_version.shellescape}`
 
       cmd_args << crunch_job_bin
       cmd_args << '--job-api-token'
@@ -263,6 +267,8 @@ class Dispatcher
         job_auth: job_auth,
         stderr_buf_to_flush: '',
         stderr_flushed_at: 0,
+        bytes_logged: 0,
+        events_logged: 0,
         log_truncated: false
       }
       i.close
@@ -448,17 +454,13 @@ class Dispatcher
 
   protected
 
-  def too_many_bytes_logged_for_job(running_job)
-    bytes_logged = Log.where(uuid: running_job[:uuid]).map { |event|
-      (event.properties["text"] || "").length
-    }.sum
-    return (bytes_logged + j[:stderr_buf_to_flush].size >
+  def too_many_bytes_logged_for_job(j)
+    return (j[:bytes_logged] + j[:stderr_buf_to_flush].size >
             Rails.configuration.crunch_limit_log_event_bytes_per_job)
   end
 
-  def too_many_events_logged_for_job(running_job)
-    log_count = Log.where(uuid: running_job[:uuid]).count
-    return (log_count >= Rails.configuration.crunch_limit_log_events_per_job)
+  def too_many_events_logged_for_job(j)
+    return (j[:events_logged] >= Rails.configuration.crunch_limit_log_events_per_job)
   end
 
   def did_recently(thing, min_interval)
@@ -493,6 +495,8 @@ class Dispatcher
                       owner_uuid: running_job[:job].owner_uuid,
                       properties: {"text" => running_job[:stderr_buf_to_flush]})
         log.save!
+        running_job[:bytes_logged] += running_job[:stderr_buf_to_flush].size
+        running_job[:events_logged] += 1
         running_job[:stderr_buf_to_flush] = ''
         running_job[:stderr_flushed_at] = Time.now.to_i
       end