7399: use regexp to look for partial line segments.

diff --git a/services/api/lib/crunch_dispatch.rb b/services/api/lib/crunch_dispatch.rb
index 1017895b25218cea5927fd568851be8047554ffb..24d4d33f30d1fb1c543d1fb08a505f0e5addadd7 100644 (file)
--- a/services/api/lib/crunch_dispatch.rb
+++ b/services/api/lib/crunch_dispatch.rb
@@ -447,8 +447,10 @@ class CrunchDispatch
     if running_job[:log_throttle_is_open]
       partial_line = false
       skip_counts = false
-      line_splits = line.split('stderr ')
-      if line_splits[1].andand.start_with?('[...]') and line_splits[1].end_with?('[...]')
+      matches = line.match(/^\S+ \S+ \d+ \d+ stderr (.*)/)
+      if matches and matches[1] and
+         (matches[1].start_with?('[...]') or matches[1].start_with?('crunchstat [...]')) and
+         matches[1].end_with?('[...]')
         partial_line = true
         if Time.now > running_job[:log_throttle_partial_line_last_at] + Rails.configuration.crunch_log_partial_line_throttle_period
           running_job[:log_throttle_partial_line_last_at] = Time.now

diff --git a/services/api/test/unit/crunch_dispatch_test.rb b/services/api/test/unit/crunch_dispatch_test.rb
index 04b06d747e724a3594ca7c4ba356a01dbb417bdc..a4503a845ef784eb6cd96cf5c564d41a5c9658a0 100644 (file)
--- a/services/api/test/unit/crunch_dispatch_test.rb
+++ b/services/api/test/unit/crunch_dispatch_test.rb
@@ -139,39 +139,44 @@ class CrunchDispatchTest < ActiveSupport::TestCase
       assert_equal 1, job[:log_throttle_lines_so_far]
 
       # first partial line segment is skipped and counted towards skipped lines
-      now = Time.now
-      line = "#{Time.now} stderr [...] this is first partial line segment [...]"
+      now = Time.now.strftime('%Y-%m-%d-%H:%M:%S')
+      line = "#{now} localhost 100 0 stderr [...] this is first partial line segment [...]"
       limit = dispatch.rate_limit(job, line)
       assert_equal true, limit
       assert_includes line, "Rate-limiting partial segments of long lines", line
       assert_equal 2, job[:log_throttle_lines_so_far]
 
       # next partial line segment within throttle interval is skipped but not counted towards skipped lines
-      line = "#{Time.now} stderr [...] second partial line segment within the interval [...]"
+      line = "#{now} localhost 100 0 stderr [...] second partial line segment within the interval [...]"
+      limit = dispatch.rate_limit(job, line)
+      assert_equal false, limit
+      assert_equal 2, job[:log_throttle_lines_so_far]
+
+      # crunchstat partial line segment is also skipped
+      line = "#{now} localhost 100 0 stderr crunchstat [...] second partial line segment within the interval [...]"
       limit = dispatch.rate_limit(job, line)
       assert_equal false, limit
       assert_equal 2, job[:log_throttle_lines_so_far]
 
       # next partial line after interval is counted towards skipped lines
       sleep(1)
-      line = "#{Time.now} stderr [...] third partial line segment after the interval [...]"
+      line = "#{now} localhost 100 0 stderr [...] third partial line segment after the interval [...]"
       limit = dispatch.rate_limit(job, line)
       assert_equal false, limit
       assert_equal 3, job[:log_throttle_lines_so_far]
 
-      now = Time.now
       # this is not a valid line segment
-      line = "#{now} stderr [...] does not end with [...] and is not a partial segment"
+      line = "#{now} localhost 100 0 stderr [...] does not end with [...] and is not a partial segment"
       limit = dispatch.rate_limit(job, line)
       assert_equal true, limit
-      assert_equal "#{now} stderr [...] does not end with [...] and is not a partial segment", line
+      assert_equal "#{now} localhost 100 0 stderr [...] does not end with [...] and is not a partial segment", line
       assert_equal 4, job[:log_throttle_lines_so_far]
 
       # this also is not a valid line segment
-      line = "#{now} stderr does not start correctly but ends with [...]"
+      line = "#{now} localhost 100 0 stderr does not start correctly but ends with [...]"
       limit = dispatch.rate_limit(job, line)
       assert_equal true, limit
-      assert_equal "#{now} stderr does not start correctly but ends with [...]", line
+      assert_equal "#{now} localhost 100 0 stderr does not start correctly but ends with [...]", line
       assert_equal 5, job[:log_throttle_lines_so_far]
     end
   end