From: Tom Clegg <tom@curoverse.com>
Date: Tue, 21 Oct 2014 17:46:00 +0000 (-0400)
Subject: 4266: Tweak long-line handling.
X-Git-Tag: 1.1.0~2085^2~4
X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/de79b73c55c5b1c469766ac8f930f335ac3fc246

4266: Tweak long-line handling.
---

diff --git a/services/api/script/crunch-dispatch.rb b/services/api/script/crunch-dispatch.rb
index bbb2be9a33..94311edea4 100755
--- a/services/api/script/crunch-dispatch.rb
+++ b/services/api/script/crunch-dispatch.rb
@@ -450,14 +450,20 @@ class Dispatcher
         bufend = ''
         streambuf.lines("\n").each do |line|
           if not line.end_with? "\n"
-            if line.size > 2**20
+            if line.size > Rails.configuration.crunch_log_throttle_bytes
               # Without a limit here, we'll use 2x an arbitrary amount
               # of memory, and waste a lot of time copying strings
               # around, all without providing any feedback to anyone
               # about what's going on _or_ hitting any of our throttle
-              # limits. Better to introduce a spurious line break once
-              # in a while:
-              line << "[...]\n"
+              # limits.
+              #
+              # Here we leave "line" alone, knowing it will never be
+              # sent anywhere: rate_limit() will reach
+              # crunch_log_throttle_bytes immediately. However, we'll
+              # leave [...] in bufend: if the trailing end of the long
+              # line does end up getting sent anywhere, it will have
+              # some indication that it incomplete.
+              bufend = "[...]"
             else
               # If line length is sane, we'll wait for the rest of the
               # line to appear in the next read_pipes() call.