From: radhika <radhika@curoverse.com>
Date: Wed, 21 Sep 2016 00:22:09 +0000 (-0400)
Subject: 9514: Refactor job log related bit out of log_viewer.js into job_log_viewer.js
X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/04d8e74e9b6476b2a87c5a66c07566f041c97b37

9514: Refactor job log related bit out of log_viewer.js into job_log_viewer.js
---

diff --git a/apps/workbench/app/assets/javascripts/job_log_viewer.js b/apps/workbench/app/assets/javascripts/job_log_viewer.js
new file mode 100644
index 0000000000..4fa038a204
--- /dev/null
+++ b/apps/workbench/app/assets/javascripts/job_log_viewer.js
@@ -0,0 +1,233 @@
+function newTaskState() {
+    return {"complete_count": 0,
+            "failure_count": 0,
+            "task_count": 0,
+            "incomplete_count": 0,
+            "nodes": []};
+}
+
+function addToJobLogViewer(logViewer, lines, taskState) {
+    var re = /((\d\d\d\d)-(\d\d)-(\d\d))_((\d\d):(\d\d):(\d\d)) ([a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15}) (\d+) (\d+)? (.*)/;
+
+    var items = [];
+    var count = logViewer.items.length;
+    for (var a in lines) {
+        var v = lines[a].match(re);
+        if (v != null) {
+
+            var ts = new Date(Date.UTC(v[2], v[3]-1, v[4], v[6], v[7], v[8]));
+
+            v11 = v[11];
+            if (typeof v[11] === 'undefined') {
+                v11 = "";
+            } else {
+                v11 = Number(v11);
+            }
+
+            var message = v[12];
+            var type = "";
+            var node = "";
+            var slot = "";
+            if (v11 !== "") {
+                if (!taskState.hasOwnProperty(v11)) {
+                    taskState[v11] = {};
+                    taskState.task_count += 1;
+                }
+
+                if (/^stderr /.test(message)) {
+                    message = message.substr(7);
+                    if (/^crunchstat: /.test(message)) {
+                        type = "crunchstat";
+                        message = message.substr(12);
+                    } else if (/^srun: /.test(message) || /^slurmd/.test(message)) {
+                        type = "task-dispatch";
+                    } else {
+                        type = "task-print";
+                    }
+                } else {
+                    var m;
+                    if (m = /^success in (\d+) second/.exec(message)) {
+                        taskState[v11].outcome = "success";
+                        taskState[v11].runtime = Number(m[1]);
+                        taskState.complete_count += 1;
+                    }
+                    else if (m = /^failure \(\#\d+, (temporary|permanent)\) after (\d+) second/.exec(message)) {
+                        taskState[v11].outcome = "failure";
+                        taskState[v11].runtime = Number(m[2]);
+                        taskState.failure_count += 1;
+                        if (m[1] == "permanent") {
+                            taskState.incomplete_count += 1;
+                        }
+                    }
+                    else if (m = /^child \d+ started on ([^.]*)\.(\d+)/.exec(message)) {
+                        taskState[v11].node = m[1];
+                        taskState[v11].slot = m[2];
+                        if (taskState.nodes.indexOf(m[1], 0) == -1) {
+                            taskState.nodes.push(m[1]);
+                        }
+                        for (var i in items) {
+                            if (i > 0) {
+                                if (items[i].taskid === v11) {
+                                    items[i].node = m[1];
+                                    items[i].slot = m[2];
+                                }
+                            }
+                        }
+                    }
+                    type = "task-dispatch";
+                }
+                node = taskState[v11].node;
+                slot = taskState[v11].slot;
+            } else {
+                type = "crunch";
+            }
+
+            items.push({
+                id: count,
+                ts: ts,
+                timestamp: ts.toLocaleDateString() + " " + ts.toLocaleTimeString(),
+                taskid: v11,
+                node: node,
+                slot: slot,
+                message: message.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;'),
+                type: type
+            });
+            count += 1;
+        } else {
+            console.log("Did not parse line " + a + ": " + lines[a]);
+        }
+    }
+    logViewer.add(items);
+}
+
+function sortById(a, b, opt) {
+    a = a.values();
+    b = b.values();
+
+    if (a["id"] > b["id"]) {
+        return 1;
+    }
+    if (a["id"] < b["id"]) {
+        return -1;
+    }
+    return 0;
+}
+
+function sortByTask(a, b, opt) {
+    var aa = a.values();
+    var bb = b.values();
+
+    if (aa["taskid"] === "" && bb["taskid"] !== "") {
+        return -1;
+    }
+    if (aa["taskid"] !== "" && bb["taskid"] === "") {
+        return 1;
+    }
+
+    if (aa["taskid"] !== "" && bb["taskid"] !== "") {
+        if (aa["taskid"] > bb["taskid"]) {
+            return 1;
+        }
+        if (aa["taskid"] < bb["taskid"]) {
+            return -1;
+        }
+    }
+
+    return sortById(a, b, opt);
+}
+
+function sortByNode(a, b, opt) {
+    var aa = a.values();
+    var bb = b.values();
+
+    if (aa["node"] === "" && bb["node"] !== "") {
+        return -1;
+    }
+    if (aa["node"] !== "" && bb["node"] === "") {
+        return 1;
+    }
+
+    if (aa["node"] !== "" && bb["node"] !== "") {
+        if (aa["node"] > bb["node"]) {
+            return 1;
+        }
+        if (aa["node"] < bb["node"]) {
+            return -1;
+        }
+    }
+
+    if (aa["slot"] !== "" && bb["slot"] !== "") {
+        if (aa["slot"] > bb["slot"]) {
+            return 1;
+        }
+        if (aa["slot"] < bb["slot"]) {
+            return -1;
+        }
+    }
+
+    return sortById(a, b, opt);
+}
+
+
+function dumbPluralize(n, s, p) {
+    if (typeof p === 'undefined') {
+        p = "s";
+    }
+    if (n == 0 || n > 1) {
+        return n + " " + (s + p);
+    } else {
+        return n + " " + s;
+    }
+}
+
+function generateJobOverview(id, logViewer, taskState) {
+    var html = "";
+
+    if (logViewer.items.length > 2) {
+        var first = logViewer.items[1];
+        var last = logViewer.items[logViewer.items.length-1];
+        var duration = (last.values().ts.getTime() - first.values().ts.getTime()) / 1000;
+
+        var hours = 0;
+        var minutes = 0;
+        var seconds;
+
+        if (duration >= 3600) {
+            hours = Math.floor(duration / 3600);
+            duration -= (hours * 3600);
+        }
+        if (duration >= 60) {
+            minutes = Math.floor(duration / 60);
+            duration -= (minutes * 60);
+        }
+        seconds = duration;
+
+        var tcount = taskState.task_count;
+
+        html += "<p>";
+        html += "Started at " + first.values().timestamp + ".  ";
+        html += "Ran " + dumbPluralize(tcount, " task") + " over ";
+        if (hours > 0) {
+            html += dumbPluralize(hours, " hour");
+        }
+        if (minutes > 0) {
+            html += " " + dumbPluralize(minutes, " minute");
+        }
+        if (seconds > 0) {
+            html += " " + dumbPluralize(seconds, " second");
+        }
+
+        html += " using " + dumbPluralize(taskState.nodes.length, " node");
+
+        html += ".  " + dumbPluralize(taskState.complete_count, "task") + " completed";
+        html += ",  " + dumbPluralize(taskState.incomplete_count, "task") +  " incomplete";
+        html += " (" + dumbPluralize(taskState.failure_count, " failure") + ")";
+
+        html += ".  Finished at " + last.values().timestamp + ".";
+        html += "</p>";
+    } else {
+       html = "<p>Job log is empty or failed to load.</p>";
+    }
+
+    $(id).html(html);
+}
diff --git a/apps/workbench/app/assets/javascripts/log_viewer.js b/apps/workbench/app/assets/javascripts/log_viewer.js
index 0e12f9cfd4..58bde9adec 100644
--- a/apps/workbench/app/assets/javascripts/log_viewer.js
+++ b/apps/workbench/app/assets/javascripts/log_viewer.js
@@ -1,237 +1,3 @@
-function newTaskState() {
-    return {"complete_count": 0,
-            "failure_count": 0,
-            "task_count": 0,
-            "incomplete_count": 0,
-            "nodes": []};
-}
-
-function addToLogViewer(logViewer, lines, taskState) {
-    var re = /((\d\d\d\d)-(\d\d)-(\d\d))_((\d\d):(\d\d):(\d\d)) ([a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15}) (\d+) (\d+)? (.*)/;
-
-    var items = [];
-    var count = logViewer.items.length;
-    for (var a in lines) {
-        var v = lines[a].match(re);
-        if (v != null) {
-
-            var ts = new Date(Date.UTC(v[2], v[3]-1, v[4], v[6], v[7], v[8]));
-
-            v11 = v[11];
-            if (typeof v[11] === 'undefined') {
-                v11 = "";
-            } else {
-                v11 = Number(v11);
-            }
-
-            var message = v[12];
-            var type = "";
-            var node = "";
-            var slot = "";
-            if (v11 !== "") {
-                if (!taskState.hasOwnProperty(v11)) {
-                    taskState[v11] = {};
-                    taskState.task_count += 1;
-                }
-
-                if (/^stderr /.test(message)) {
-                    message = message.substr(7);
-                    if (/^crunchstat: /.test(message)) {
-                        type = "crunchstat";
-                        message = message.substr(12);
-                    } else if (/^srun: /.test(message) || /^slurmd/.test(message)) {
-                        type = "task-dispatch";
-                    } else {
-                        type = "task-print";
-                    }
-                } else {
-                    var m;
-                    if (m = /^success in (\d+) second/.exec(message)) {
-                        taskState[v11].outcome = "success";
-                        taskState[v11].runtime = Number(m[1]);
-                        taskState.complete_count += 1;
-                    }
-                    else if (m = /^failure \(\#\d+, (temporary|permanent)\) after (\d+) second/.exec(message)) {
-                        taskState[v11].outcome = "failure";
-                        taskState[v11].runtime = Number(m[2]);
-                        taskState.failure_count += 1;
-                        if (m[1] == "permanent") {
-                            taskState.incomplete_count += 1;
-                        }
-                    }
-                    else if (m = /^child \d+ started on ([^.]*)\.(\d+)/.exec(message)) {
-                        taskState[v11].node = m[1];
-                        taskState[v11].slot = m[2];
-                        if (taskState.nodes.indexOf(m[1], 0) == -1) {
-                            taskState.nodes.push(m[1]);
-                        }
-                        for (var i in items) {
-                            if (i > 0) {
-                                if (items[i].taskid === v11) {
-                                    items[i].node = m[1];
-                                    items[i].slot = m[2];
-                                }
-                            }
-                        }
-                    }
-                    type = "task-dispatch";
-                }
-                node = taskState[v11].node;
-                slot = taskState[v11].slot;
-            } else {
-                type = "crunch";
-            }
-
-            items.push({
-                id: count,
-                ts: ts,
-                timestamp: ts.toLocaleDateString() + " " + ts.toLocaleTimeString(),
-                taskid: v11,
-                node: node,
-                slot: slot,
-                message: message.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;'),
-                type: type
-            });
-            count += 1;
-        } else {
-            console.log("Did not parse line " + a + ": " + lines[a]);
-        }
-    }
-    logViewer.add(items);
-}
-
-function sortById(a, b, opt) {
-    a = a.values();
-    b = b.values();
-
-    if (a["id"] > b["id"]) {
-        return 1;
-    }
-    if (a["id"] < b["id"]) {
-        return -1;
-    }
-    return 0;
-}
-
-function sortByTask(a, b, opt) {
-    var aa = a.values();
-    var bb = b.values();
-
-    if (aa["taskid"] === "" && bb["taskid"] !== "") {
-        return -1;
-    }
-    if (aa["taskid"] !== "" && bb["taskid"] === "") {
-        return 1;
-    }
-
-    if (aa["taskid"] !== "" && bb["taskid"] !== "") {
-        if (aa["taskid"] > bb["taskid"]) {
-            return 1;
-        }
-        if (aa["taskid"] < bb["taskid"]) {
-            return -1;
-        }
-    }
-
-    return sortById(a, b, opt);
-}
-
-function sortByNode(a, b, opt) {
-    var aa = a.values();
-    var bb = b.values();
-
-    if (aa["node"] === "" && bb["node"] !== "") {
-        return -1;
-    }
-    if (aa["node"] !== "" && bb["node"] === "") {
-        return 1;
-    }
-
-    if (aa["node"] !== "" && bb["node"] !== "") {
-        if (aa["node"] > bb["node"]) {
-            return 1;
-        }
-        if (aa["node"] < bb["node"]) {
-            return -1;
-        }
-    }
-
-    if (aa["slot"] !== "" && bb["slot"] !== "") {
-        if (aa["slot"] > bb["slot"]) {
-            return 1;
-        }
-        if (aa["slot"] < bb["slot"]) {
-            return -1;
-        }
-    }
-
-    return sortById(a, b, opt);
-}
-
-
-function dumbPluralize(n, s, p) {
-    if (typeof p === 'undefined') {
-        p = "s";
-    }
-    if (n == 0 || n > 1) {
-        return n + " " + (s + p);
-    } else {
-        return n + " " + s;
-    }
-}
-
-function generateJobOverview(id, logViewer, taskState) {
-    var html = "";
-
-    if (logViewer.items.length > 2) {
-        var first = logViewer.items[1];
-        var last = logViewer.items[logViewer.items.length-1];
-        var duration = (last.values().ts.getTime() - first.values().ts.getTime()) / 1000;
-
-        var hours = 0;
-        var minutes = 0;
-        var seconds;
-
-        if (duration >= 3600) {
-            hours = Math.floor(duration / 3600);
-            duration -= (hours * 3600);
-        }
-        if (duration >= 60) {
-            minutes = Math.floor(duration / 60);
-            duration -= (minutes * 60);
-        }
-        seconds = duration;
-
-        var tcount = taskState.task_count;
-
-        html += "<p>";
-        html += "Started at " + first.values().timestamp + ".  ";
-        html += "Ran " + dumbPluralize(tcount, " task") + " over ";
-        if (hours > 0) {
-            html += dumbPluralize(hours, " hour");
-        }
-        if (minutes > 0) {
-            html += " " + dumbPluralize(minutes, " minute");
-        }
-        if (seconds > 0) {
-            html += " " + dumbPluralize(seconds, " second");
-        }
-
-        html += " using " + dumbPluralize(taskState.nodes.length, " node");
-
-        html += ".  " + dumbPluralize(taskState.complete_count, "task") + " completed";
-        html += ",  " + dumbPluralize(taskState.incomplete_count, "task") +  " incomplete";
-        html += " (" + dumbPluralize(taskState.failure_count, " failure") + ")";
-
-        html += ".  Finished at " + last.values().timestamp + ".";
-        html += "</p>";
-    } else {
-       html = "<p>Job log is empty or failed to load.</p>";
-    }
-
-    $(id).html(html);
-}
-
 function gotoPage(n, logViewer, page, id) {
     if (n < 0) { return; }
     if (n*page > logViewer.matchingItems.length) { return; }
@@ -280,3 +46,13 @@ function nextPage(logViewer, page, id) {
 function prevPage(logViewer, page, id) {
     gotoPage(logViewer.page_offset-1, logViewer, page, id);
 }
+
+function addToLogViewer(logViewer, lines) {
+    var items = [];
+    for (var a in lines) {
+      items.push({
+        message: lines[a].replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;')
+      });
+    }
+    logViewer.add(items);
+}
diff --git a/apps/workbench/app/views/jobs/_show_log.html.erb b/apps/workbench/app/views/jobs/_show_log.html.erb
index 02ad2b7398..448e04c663 100644
--- a/apps/workbench/app/views/jobs/_show_log.html.erb
+++ b/apps/workbench/app/views/jobs/_show_log.html.erb
@@ -98,7 +98,7 @@ var makeFilter = function() {
             return load_log();
         }
         logViewer.filter();
-        addToLogViewer(logViewer, data.split("\n"), taskState);
+        addToJobLogViewer(logViewer, data.split("\n"), taskState);
         logViewer.filter(makeFilter());
         content_range_hdr = jqxhr.getResponseHeader('Content-Range');
         var v = content_range_hdr && content_range_hdr.match(/bytes \d+-(\d+)\/(.+)/);
diff --git a/apps/workbench/app/views/work_units/_show_log.html.erb b/apps/workbench/app/views/work_units/_show_log.html.erb
index 6a0916fe98..f9b6241ea3 100644
--- a/apps/workbench/app/views/work_units/_show_log.html.erb
+++ b/apps/workbench/app/views/work_units/_show_log.html.erb
@@ -9,16 +9,111 @@
   </div>
 <% end %>
 
-<%# Show log in terminal window %>
-<h4>Recent logs</h4>
-<div id="event_log_div"
-     class="arv-log-event-listener arv-log-event-handler-append-logs arv-job-log-window"
-     data-object-uuids="<%= wu.log_object_uuids.join(' ') %>"
-  ><%= wu.live_log_lines(Rails.configuration.running_job_log_records_to_fetch).join("\n") %>
-</div>
-
-<%# Applying a long throttle suppresses the auto-refresh of this
-    partial that would normally be triggered by arv-log-event. %>
-<div class="arv-log-refresh-control"
-     data-load-throttle="86486400000" <%# 1001 nights %>>
-</div>
+<% live_log_lines = wu.live_log_lines(Rails.configuration.running_job_log_records_to_fetch).join("\n") %>
+<% if !render_log or (live_log_lines.size > 0) %>
+  <%# Still running, or recently finished and logs are still available from logs table %>
+  <%# Show recent logs in terminal window %>
+  <h4>Recent logs</h4>
+  <div id="event_log_div"
+       class="arv-log-event-listener arv-log-event-handler-append-logs arv-job-log-window"
+       data-object-uuids="<%= wu.log_object_uuids.join(' ') %>"
+    ><%= live_log_lines %>
+  </div>
+
+  <%# Applying a long throttle suppresses the auto-refresh of this
+      partial that would normally be triggered by arv-log-event. %>
+  <div class="arv-log-refresh-control"
+       data-load-throttle="86486400000" <%# 1001 nights %>>
+  </div>
+<% elsif render_log[:log] %>
+  <%# Retrieve finished log from keep and show %>
+  <script>
+    (function() {
+      var pagesize = 1000;
+      var logViewer = new List('log-viewer', {
+        valueNames: [ 'message'],
+        page: pagesize
+      });
+
+      logViewer.page_offset = 0;
+      logViewer.on("updated", function() { updatePaging(".log-viewer-paging", logViewer, pagesize) } );
+      $(".log-viewer-page-up").on("click", function() { prevPage(logViewer, pagesize, ".log-viewer-paging"); return false; });
+      $(".log-viewer-page-down").on("click", function() { nextPage(logViewer, pagesize, ".log-viewer-paging"); return false; });
+
+      <% logcollection = render_log[:log] %>
+      var log_size = <%= logcollection.files[0][2] %>
+      var log_maxbytes = <%= Rails.configuration.log_viewer_max_bytes %>;
+      var logcollection_url = '<%=j url_for logcollection %>/<%=j logcollection.files[0][1] %>';
+      var headers = {};
+      if (log_size > log_maxbytes) {
+        headers['Range'] = 'bytes=0-' + log_maxbytes;
+      }
+      var ajax_opts = { dataType: 'text', headers: headers };
+      load_log();
+
+      function load_log() {
+        $.ajax(logcollection_url, ajax_opts).done(done).fail(fail);
+      }
+      function done(data, status, jqxhr) {
+        if (jqxhr.getResponseHeader('Content-Type').indexOf('application/json') === 0) {
+          // The browser won't allow a redirect-with-cookie response
+          // because keep-web isn't same-origin with us. Instead, we
+          // assure keep-web it's OK to respond with the content
+          // immediately by setting the token in the request body
+          // instead and adding disposition=attachment.
+          logcollection_url = JSON.parse(data).href;
+          var queryAt = logcollection_url.indexOf('?api_token=');
+          if (queryAt >= 0) {
+            ajax_opts.method = 'POST';
+            ajax_opts.data = {
+              api_token: logcollection_url.slice(queryAt+11),
+              disposition: 'attachment',
+            };
+            logcollection_url = logcollection_url.slice(0, queryAt);
+          }
+          return load_log();
+        }
+        addToLogViewer(logViewer, data.split("\n"));
+        content_range_hdr = jqxhr.getResponseHeader('Content-Range');
+        var v = content_range_hdr && content_range_hdr.match(/bytes \d+-(\d+)\/(.+)/);
+        short_log = v && (v[2] == '*' || parseInt(v[1]) + 1 < v[2]);
+        if (jqxhr.status == 206 && short_log) {
+          $("#log-viewer-overview").html(
+            '<p>Showing only ' + data.length + ' bytes of this log.</p>'
+          );
+        }
+
+        $("#log-viewer .spinner").detach();
+      }
+      function fail(jqxhr, status, error) {
+        // TODO: tell the user about the error
+        $("#log-viewer .spinner").detach();
+      }
+    })();
+  </script>
+
+  <div id="log-viewer">
+    <p id="log-viewer-overview"></p>
+
+    <div class="h3">Log</div>
+    <div class="smart-scroll" data-smart-scroll-padding-bottom="50" style="margin-bottom: 0px">
+      <table class="log-viewer-table">
+        <tbody class="list">
+          <tr>
+            <td class="message"></td>
+          </tr>
+        </tbody>
+      </table>
+
+      <% if logcollection %>
+        <div class="spinner spinner-32px"></div>
+      <% end %>
+    </div>
+
+    <div class="log-viewer-paging-div" style="margin-bottom: -15px">
+      <a href="#" class="log-viewer-page-up"><span class='glyphicon glyphicon-arrow-up'></span></a>
+      <span class="log-viewer-paging"></span>
+      <a href="#" class="log-viewer-page-down"><span class='glyphicon glyphicon-arrow-down'></span></a>
+    </div>
+  </div>
+<% end %>
diff --git a/services/api/test/fixtures/container_requests.yml b/services/api/test/fixtures/container_requests.yml
index 49a1faec3f..78d9fa7383 100644
--- a/services/api/test/fixtures/container_requests.yml
+++ b/services/api/test/fixtures/container_requests.yml
@@ -184,7 +184,7 @@ cr_for_failed:
   cwd: test
   output_path: test
   command: ["echo", "hello"]
-  container_uuid: zzzzz-dz642-failedcntnr
+  container_uuid: zzzzz-dz642-failedcontainr1
   runtime_constraints:
     vcpus: 1
     ram: 123