2934: add limits for crunch job log generation.

* crunch_limit_log_events_per_job
* crunch_limit_log_event_bytes_per_job

  Limit the number of events a job may log, and the total number of
  bytes a job may log.

* crunch_log_bytes_per_event
* crunch_log_seconds_between_events

  Set minimum logging thresholds: event logs will only be flushed to the
  database when at least crunch_log_bytes_per_event have been generated,
  or crunch_log_seconds_between_events have passed.

Integration test CrunchDispatchTest exercised creating a job.

Refs #2934.

diff --git a/services/api/script/crunch-dispatch.rb b/services/api/script/crunch-dispatch.rb
index 59e3aff31e692c4003479615017f768624e49f9e..6f3aeda413bac1b5e8a2ce29bf6a842d8467a871 100755 (executable)
--- a/services/api/script/crunch-dispatch.rb
+++ b/services/api/script/crunch-dispatch.rb
@@ -26,8 +26,6 @@ require File.dirname(__FILE__) + '/../config/boot'
 require File.dirname(__FILE__) + '/../config/environment'
 require 'open3'
 
-LOG_BUFFER_SIZE = 4096
-
 class Dispatcher
   include ApplicationHelper
 
@@ -264,7 +262,8 @@ class Dispatcher
         sent_int: 0,
         job_auth: job_auth,
         stderr_buf_to_flush: '',
-        stderr_flushed_at: 0
+        stderr_flushed_at: 0,
+        log_truncated: false
       }
       i.close
       update_node_status
@@ -314,7 +313,8 @@ class Dispatcher
             j[:stderr_buf_to_flush] << pub_msg
           end
 
-          if (LOG_BUFFER_SIZE < j[:stderr_buf_to_flush].size) || ((j[:stderr_flushed_at]+1) < Time.now.to_i)
+          if (Rails.configuration.crunch_log_bytes_per_event < j[:stderr_buf_to_flush].size or
+              (j[:stderr_flushed_at] + Rails.configuration.crunch_log_seconds_between_events < Time.now.to_i))
             write_log j
           end
         end
@@ -448,6 +448,19 @@ class Dispatcher
 
   protected
 
+  def too_many_bytes_logged_for_job(running_job)
+    bytes_logged = Log.where(uuid: running_job[:uuid]).map { |event|
+      (event.properties["text"] || "").length
+    }.sum
+    return (bytes_logged + j[:stderr_buf_to_flush].size >
+            Rails.configuration.crunch_limit_log_event_bytes_per_job)
+  end
+
+  def too_many_events_logged_for_job(running_job)
+    log_count = Log.where(uuid: running_job[:uuid]).count
+    return (log_count >= Rails.configuration.crunch_limit_log_events_per_job)
+  end
+
   def did_recently(thing, min_interval)
     @did_recently ||= {}
     if !@did_recently[thing] or @did_recently[thing] < Time.now - min_interval
@@ -462,6 +475,19 @@ class Dispatcher
   def write_log running_job
     begin
       if (running_job && running_job[:stderr_buf_to_flush] != '')
+        # Truncate logs if they exceed crunch_limit_log_event_bytes_per_job
+        # or crunch_limit_log_events_per_job.
+        if (too_many_bytes_logged_for_job(running_job))
+          return if running_job[:log_truncated]
+          running_job[:log_truncated] = true
+          running_job[:stderr_buf_to_flush] =
+              "Server configured limit reached (crunch_limit_log_event_bytes_per_job: #{Rails.configuration.crunch_limit_log_event_bytes_per_job}). Subsequent logs truncated"
+        elsif (too_many_events_logged_for_job(running_job))
+          return if running_job[:log_truncated]
+          running_job[:log_truncated] = true
+          running_job[:stderr_buf_to_flush] =
+              "Server configured limit reached (crunch_limit_log_events_per_job: #{Rails.configuration.crunch_limit_log_events_per_job}). Subsequent logs truncated"
+        end
         log = Log.new(object_uuid: running_job[:job].uuid,
                       event_type: 'stderr',
                       owner_uuid: running_job[:job].owner_uuid,

diff --git a/services/api/test/integration/crunch_dispatch_test.rb b/services/api/test/integration/crunch_dispatch_test.rb
new file mode 100644 (file)
index 0000000..e968cf1
--- /dev/null
+++ b/services/api/test/integration/crunch_dispatch_test.rb
@@ -0,0 +1,27 @@
+require 'pp'
+require 'test_helper'
+load 'test/functional/arvados/v1/git_setup.rb'
+
+class CrunchDispatchTest < ActionDispatch::IntegrationTest
+  include CurrentApiClient
+  include GitSetup
+
+  fixtures :all
+
+  test "job runs" do
+    authorize_with :admin
+    set_user_from_auth :admin
+    post "/arvados/v1/jobs", {
+      format: "json",
+      job: {
+        script: "log",
+        repository: "bar",
+        script_version: "143fec09e988160673c63457fa12a0f70b5b8a26",
+        script_parameters: {}
+      }
+    }, {HTTP_AUTHORIZATION: "OAuth2 #{current_api_client_authorization}"}
+    p "response: #{@response.body}"
+    assert_response :success
+    resp = JSON.parse(@response.body)
+  end
+end

diff --git a/services/api/test/test.git.tar b/services/api/test/test.git.tar
index fdd7db6a2bc1f3cb25c517e28d8730e27aeaf474..589f18cc6eff58223c9c0caee021139e74472b35 100644 (file)
Binary files a/services/api/test/test.git.tar and b/services/api/test/test.git.tar differ