4266: Add word missing from comment.

[arvados.git] / services / api / script / crunch-dispatch.rb

diff --git a/services/api/script/crunch-dispatch.rb b/services/api/script/crunch-dispatch.rb
index 601ec47667d45d61eaf667d5e71c8730bd1a8389..d3147225d63d4c805320302c8a62eae1f2a2c2fc 100755 (executable)
--- a/services/api/script/crunch-dispatch.rb
+++ b/services/api/script/crunch-dispatch.rb
@@ -440,6 +440,7 @@ class Dispatcher
         if not j[:log_throttle_is_open]
           j[:log_throttle_bytes_skipped] += streambuf.size + buf.size
           streambuf.replace ''
+          next
         elsif buf == ''
           next
         end
@@ -448,10 +449,28 @@ class Dispatcher
         streambuf << buf
 
         bufend = ''
-        streambuf.lines("\n").each do |line|
-          if not line.end_with? "\n"
-            bufend = line
-            break
+        streambuf.each_line do |line|
+          if not line.end_with? $/
+            if line.size > Rails.configuration.crunch_log_throttle_bytes
+              # Without a limit here, we'll use 2x an arbitrary amount
+              # of memory, and waste a lot of time copying strings
+              # around, all without providing any feedback to anyone
+              # about what's going on _or_ hitting any of our throttle
+              # limits.
+              #
+              # Here we leave "line" alone, knowing it will never be
+              # sent anywhere: rate_limit() will reach
+              # crunch_log_throttle_bytes immediately. However, we'll
+              # leave [...] in bufend: if the trailing end of the long
+              # line does end up getting sent anywhere, it will have
+              # some indication that it is incomplete.
+              bufend = "[...]"
+            else
+              # If line length is sane, we'll wait for the rest of the
+              # line to appear in the next read_pipes() call.
+              bufend = line
+              break
+            end
           end
           # rate_limit returns true or false as to whether to actually log
           # the line or not.  It also modifies "line" in place to replace
@@ -463,6 +482,9 @@ class Dispatcher
             j[:stderr_buf_to_flush] << pub_msg
           end
         end
+
+        # Leave the trailing incomplete line (if any) in streambuf for
+        # next time.
         streambuf.replace bufend
       end
       # Flush buffered logs to the logs table, if appropriate. We have