Merge branch 'master' into 3177-collection-choose-files
authorradhika <radhika@curoverse.com>
Mon, 10 Nov 2014 16:32:31 +0000 (11:32 -0500)
committerradhika <radhika@curoverse.com>
Mon, 10 Nov 2014 16:32:31 +0000 (11:32 -0500)
27 files changed:
apps/workbench/Gemfile.lock
apps/workbench/app/assets/javascripts/event_log.js
apps/workbench/app/assets/javascripts/pipeline_instances.js
apps/workbench/app/assets/javascripts/tab_panes.js
apps/workbench/app/assets/stylesheets/application.css.scss
apps/workbench/app/assets/stylesheets/jobs.css.scss
apps/workbench/app/helpers/pipeline_components_helper.rb
apps/workbench/app/views/application/_content.html.erb
apps/workbench/app/views/jobs/_show_job_buttons.html.erb [new file with mode: 0644]
apps/workbench/app/views/jobs/_show_log.html.erb
apps/workbench/app/views/jobs/_show_status.html.erb
apps/workbench/app/views/jobs/show.html.erb
apps/workbench/app/views/pipeline_instances/_running_component.html.erb
apps/workbench/app/views/pipeline_instances/_show_components.html.erb
apps/workbench/app/views/pipeline_instances/_show_components_running.html.erb
apps/workbench/app/views/pipeline_instances/_show_log.html.erb
apps/workbench/app/views/pipeline_instances/show.html.erb
apps/workbench/app/views/projects/index.html.erb
apps/workbench/test/integration/websockets_test.rb [new file with mode: 0644]
apps/workbench/test/test_helper.rb
doc/install/index.html.textile.liquid
sdk/python/arvados/collection.py
sdk/python/tests/run_test_server.py
sdk/python/tests/test_collections.py
services/api/config/application.default.yml
services/api/script/crunch-dispatch.rb
services/api/test/fixtures/jobs.yml

index 70ef7a0f101abff207922b23bdbf97b22c86948c..53d42cb856e59903704b39e157bb118de6c4320f 100644 (file)
@@ -66,7 +66,7 @@ GEM
       net-sftp (>= 2.0.0)
       net-ssh (>= 2.0.14)
       net-ssh-gateway (>= 1.1.0)
-    capybara (2.2.1)
+    capybara (2.4.4)
       mime-types (>= 1.16)
       nokogiri (>= 1.3.3)
       rack (>= 1.0.0)
@@ -127,7 +127,7 @@ GEM
       treetop (~> 1.4.8)
     metaclass (0.0.4)
     mime-types (1.25.1)
-    mini_portile (0.5.2)
+    mini_portile (0.6.0)
     minitest (5.3.3)
     mocha (1.1.0)
       metaclass (~> 0.0.1)
@@ -140,8 +140,8 @@ GEM
     net-ssh (2.7.0)
     net-ssh-gateway (1.2.0)
       net-ssh (>= 2.6.5)
-    nokogiri (1.6.1)
-      mini_portile (~> 0.5.0)
+    nokogiri (1.6.3.1)
+      mini_portile (= 0.6.0)
     oj (2.1.7)
     passenger (4.0.23)
       daemon_controller (>= 1.1.0)
index 8bfa1b067252086defb578adefa129c9a31998d0..36361a17d12e3f295910b87be2ff85a6e6077110 100644 (file)
@@ -4,51 +4,55 @@
 
 /* Subscribe to websockets event log.  Do nothing if already connected. */
 function subscribeToEventLog () {
-  // if websockets are not supported by browser, do not subscribe for events
-  websocketsSupported = ('WebSocket' in window);
-  if (websocketsSupported == false) {
-    return;
-  }
-
-  // check if websocket connection is already stored on the window
-  event_log_disp = $(window).data("arv-websocket");
-  if (event_log_disp == null) {
-    // need to create new websocket and event log dispatcher
-    websocket_url = $('meta[name=arv-websocket-url]').attr("content");
-    if (websocket_url == null)
-      return;
-
-    event_log_disp = new WebSocket(websocket_url);
-
-    event_log_disp.onopen = onEventLogDispatcherOpen;
-    event_log_disp.onmessage = onEventLogDispatcherMessage;
-
-    // store websocket in window to allow reuse when multiple divs subscribe for events
-    $(window).data("arv-websocket", event_log_disp);
-  }
+    // if websockets are not supported by browser, do not subscribe for events
+    websocketsSupported = ('WebSocket' in window);
+    if (websocketsSupported == false) {
+        return;
+    }
+
+    // check if websocket connection is already stored on the window
+    event_log_disp = $(window).data("arv-websocket");
+    if (event_log_disp == null) {
+        // need to create new websocket and event log dispatcher
+        websocket_url = $('meta[name=arv-websocket-url]').attr("content");
+        if (websocket_url == null)
+            return;
+
+        event_log_disp = new WebSocket(websocket_url);
+
+        event_log_disp.onopen = onEventLogDispatcherOpen;
+        event_log_disp.onmessage = onEventLogDispatcherMessage;
+
+        // store websocket in window to allow reuse when multiple divs subscribe for events
+        $(window).data("arv-websocket", event_log_disp);
+    }
 }
 
 /* Send subscribe message to the websockets server.  Without any filters
    arguments, this subscribes to all events */
 function onEventLogDispatcherOpen(event) {
-  this.send('{"method":"subscribe"}');
+    this.send('{"method":"subscribe"}');
 }
 
 /* Trigger event for all applicable elements waiting for this event */
 function onEventLogDispatcherMessage(event) {
-  parsedData = JSON.parse(event.data);
-  object_uuid = parsedData.object_uuid;
+    parsedData = JSON.parse(event.data);
+    object_uuid = parsedData.object_uuid;
 
-  // if there are any listeners for this object uuid or "all", trigger the event
-  matches = ".arv-log-event-listener[data-object-uuid=\"" + object_uuid + "\"],.arv-log-event-listener[data-object-uuids~=\"" + object_uuid + "\"],.arv-log-event-listener[data-object-uuid=\"all\"],.arv-log-event-listener[data-object-kind=\"" + parsedData.object_kind + "\"]";
-  $(matches).trigger('arv-log-event', event.data);
+    if (!object_uuid) {
+        return;
+    }
+
+    // if there are any listeners for this object uuid or "all", trigger the event
+    matches = ".arv-log-event-listener[data-object-uuid=\"" + object_uuid + "\"],.arv-log-event-listener[data-object-uuids~=\"" + object_uuid + "\"],.arv-log-event-listener[data-object-uuid=\"all\"],.arv-log-event-listener[data-object-kind=\"" + parsedData.object_kind + "\"]";
+    $(matches).trigger('arv-log-event', parsedData);
 }
 
 /* Automatically connect if there are any elements on the page that want to
-   received event log events. */
+   receive event log events. */
 $(document).on('ajax:complete ready', function() {
-  var a = $('.arv-log-event-listener');
-  if (a.length > 0) {
-    subscribeToEventLog();
-  }
+    var a = $('.arv-log-event-listener');
+    if (a.length > 0) {
+        subscribeToEventLog();
+    }
 });
index 3c949f4e834a27fe07885e69d6711ae040c85be6..761477e4653629851edfe58837c2c3db0cc233e1 100644 (file)
@@ -47,54 +47,65 @@ $(document).on('ready ajax:complete', function() {
     run_pipeline_button_state();
 });
 
-$(document).on('arv-log-event', '.arv-log-event-handler-append-logs', function(event, eventData){
-    var wasatbottom = ($(this).scrollTop() + $(this).height() >=
-                       this.scrollHeight);
-    var parsedData = JSON.parse(eventData);
-    var propertyText = undefined;
-    var properties = parsedData.properties;
+$(document).on('arv-log-event', '.arv-refresh-on-state-change', function(event, eventData) {
+    if (this != event.target) {
+        // Not interested in events sent to child nodes.
+        return;
+    }
+    if (eventData.event_type == "update" &&
+        eventData.properties.old_attributes.state != eventData.properties.new_attributes.state)
+    {
+        $(event.target).trigger('arv:pane:reload');
+    }
+});
 
-    if (properties !== null) {
-        propertyText = properties.text;
+$(document).on('arv-log-event', '.arv-log-event-subscribe-to-pipeline-job-uuids', function(event, eventData){
+    if (this != event.target) {
+        // Not interested in events sent to child nodes.
+        return;
     }
-    if (propertyText !== undefined) {
-        propertyText = propertyText.
-            replace(/\n$/, '').
-            replace(/\n/g, '<br/>');
-        $(this).append(propertyText + "<br/>");
-    } else if (parsedData.summary !== undefined) {
-        if (parsedData.summary.match(/^update of [-a-z0-9]{27}$/))
-            ; // Not helpful.
-        else
-            $(this).append(parsedData.summary + "<br/>");
+    if (!((eventData.object_kind == 'arvados#pipelineInstance') &&
+          (eventData.event_type == "create" ||
+           eventData.event_type == "update") &&
+         eventData.properties &&
+         eventData.properties.new_attributes &&
+         eventData.properties.new_attributes.components)) {
+        return;
     }
-    if (wasatbottom)
-        this.scrollTop = this.scrollHeight;
-}).on('arv:pane:loaded', '#Logs,#Log', function(){
-    $('.arv-log-event-handler-append-logs', this).each(function() {
-        this.scrollTop = this.scrollHeight;
-        $(this).closest('.tab-pane').on('arv:pane:reload', function(e) {
-            // Do not let this tab auto-refresh.
-            e.stopPropagation();
-        });
-    });
-}).on('ready ajax:complete', function(){
-    $(".arv-log-event-listener[data-object-uuids-live]").each(function() {
-        // Look at data-object-uuid attribute of elements matching
-        // given selector, so the event listener can listen for events
-        // that appeared on the page via ajax.
-        var $listener = $(this);
-        var have_uuids = '' + $listener.attr('data-object-uuids');
-        $($listener.attr('data-object-uuids-live')).each(function() {
-            var this_uuid = $(this).attr('data-object-uuid');
-            if (have_uuids.indexOf(this_uuid) == -1) {
-                have_uuids = have_uuids + ' ' + this_uuid;
-            }
-        });
-        $listener.attr('data-object-uuids', have_uuids);
+    var objs = "";
+    var components = eventData.properties.new_attributes.components;
+    for (a in components) {
+        if (components[a].job && components[a].job.uuid) {
+            objs += " " + components[a].job.uuid;
+        }
+    }
+    $(event.target).attr("data-object-uuids", eventData.object_uuid + objs);
+});
+
+$(document).on('ready ajax:success', function() {
+    $('.arv-log-refresh-control').each(function() {
+        var uuids = $(this).attr('data-object-uuids');
+        var $pane = $(this).closest('[data-pane-content-url]');
+        $pane.attr('data-object-uuids', uuids);
     });
 });
 
+$(document).on('arv-log-event', '.arv-log-event-handler-append-logs', function(event, eventData){
+    if (this != event.target) {
+        // Not interested in events sent to child nodes.
+        return;
+    }
+    var wasatbottom = ($(this).scrollTop() + $(this).height() >= this.scrollHeight);
+
+    if (eventData.event_type == "stderr" || eventData.event_type == "stdout") {
+        $(this).append(eventData.properties.text);
+    }
+
+    if (wasatbottom) {
+        this.scrollTop = this.scrollHeight;
+    }
+});
+
 var showhide_compare = function() {
     var form = $('form#compare')[0];
     $('input[type=hidden][name="uuids[]"]', form).remove();
@@ -111,9 +122,3 @@ var showhide_compare = function() {
 };
 $('[data-object-uuid*=-d1hrv-] input[name="uuids[]"]').on('click', showhide_compare);
 showhide_compare();
-
-setInterval(function(){
-    if ($('[data-pipeline-state=RunningOnServer],[data-pipeline-state=RunningOnClient]').length > 0) {
-        $('#Components-tab,#Graph-tab,#pipeline-instance-tab-buttons').trigger('arv:pane:reload');
-    }
-}, 15000);
index cca49b2799873211cfbfc636fea1be5ea13047e3..07e46fe65fc845328eb21c0c0bc7dd6042ba5d21 100644 (file)
 // Load tab panes on demand. See app/views/application/_content.html.erb
 
 // Fire when a tab is selected/clicked.
-$(document).on('shown.bs.tab', '[data-toggle="tab"]', function(e) {
-    $(this).trigger('arv:pane:reload');
+$(document).on('shown.bs.tab', '[data-toggle="tab"]', function(event) {
+    // reload the pane (unless it's already loaded)
+    $($(event.target).attr('href')).
+        not('.pane-loaded').
+        trigger('arv:pane:reload');
 });
 
-// Fire when the content in a pane becomes stale/dirty. If the pane is
-// 'active', reload it right away. Otherwise, just replace the current content
-// with a spinner for now, don't load the new content unless/until the pane
-// becomes active.
-$(document).on('arv:pane:reload', function(e) {
-    // Unload a single pane. Reload it if it's active.
-    $(e.target).removeClass('loaded');
-    var $pane = $($(e.target).attr('href'));
-    if ($pane.hasClass('active')) {
-        var content_url = $(e.target).attr('data-pane-content-url');
-        $.ajax(content_url, {dataType: 'html', type: 'GET', context: $pane}).
-            done(function(data, status, jqxhr) {
-                // Preserve collapsed state
-                var collapsable = {};
-                $(".collapse", $pane).each(function(i, c) {
-                    collapsable[c.id] = $(c).hasClass('in');
-                });
-                var tmp = $(data);
-                $(".collapse", tmp).each(function(i, c) {
-                    if (collapsable[c.id]) {
-                        $(c).addClass('in');
-                    } else {
-                        $(c).removeClass('in');
-                    }
-                });
-                $pane.html(tmp);
-                $(e.target).addClass('loaded');
-                $pane.trigger('arv:pane:loaded');
-            }).fail(function(jqxhr, status, error) {
-                var errhtml;
-                if (jqxhr.getResponseHeader('Content-Type').match(/\btext\/html\b/)) {
-                    var $response = $(jqxhr.responseText);
-                    var $wrapper = $('div#page-wrapper', $response);
-                    if ($wrapper.length) {
-                        errhtml = $wrapper.html();
-                    } else {
-                        errhtml = jqxhr.responseText;
-                    }
+// Ask a refreshable pane to reload via ajax.
+//
+// Target of this event is the DOM element to be updated. A reload
+// consists of an AJAX call to load the "data-pane-content-url" and
+// replace the content of the target element with the retrieved HTML.
+//
+// There are four CSS classes set on the element to indicate its state:
+// pane-loading, pane-stale, pane-loaded, pane-reload-pending
+//
+// There are five states based on the presence or absence of css classes:
+//
+// 1. Absence of any pane-* states means the pane is empty, and should
+// be loaded as soon as it becomes visible.
+//
+// 2. "pane-loading" means an AJAX call has been made to reload the
+// pane and we are waiting on a result.
+//
+// 3. "pane-loading pane-stale" means the pane is loading, but has
+// already been invalidated and should schedule a reload as soon as
+// possible after the current load completes. (This happens when there
+// is a cluster of events, where the reload is triggered by the first
+// event, but we want ensure that we eventually load the final
+// quiescent state).
+//
+// 4. "pane-loaded" means the pane is up to date.
+//
+// 5. "pane-loaded pane-reload-pending" means a reload is needed, and
+// has been scheduled, but has not started because the pane's
+// minimum-time-between-reloads throttle has not yet been reached.
+//
+$(document).on('arv:pane:reload', '[data-pane-content-url]', function(e) {
+    if (this != e.target) {
+        // An arv:pane:reload event was sent to an element (e.target)
+        // which happens to have an ancestor (this) matching the above
+        // '[data-pane-content-url]' selector. This happens because
+        // events bubble up the DOM on their way to document. However,
+        // here we only care about events delivered directly to _this_
+        // selected element (i.e., this==e.target), not ones delivered
+        // to its children. The event "e" is uninteresting here.
+        return;
+    }
+
+    // $pane, the event target, is an element whose content is to be
+    // replaced. Pseudoclasses on $pane (pane-loading, etc) encode the
+    // current loading state.
+    var $pane = $(this);
+
+    if ($pane.hasClass('pane-loading')) {
+        // Already loading, mark stale to schedule a reload after this one.
+        $pane.addClass('pane-stale');
+        return;
+    }
+
+    // The default throttle (mininum milliseconds between refreshes)
+    // can be overridden by an .arv-log-refresh-control element inside
+    // the pane -- or, failing that, the pane element itself -- with a
+    // data-load-throttle attribute. This allows the server to adjust
+    // the throttle depending on the pane content.
+    var throttle =
+        $pane.find('.arv-log-refresh-control').attr('data-load-throttle') ||
+        $pane.attr('data-load-throttle') ||
+        15000;
+    var now = (new Date()).getTime();
+    var loaded_at = $pane.attr('data-loaded-at');
+    var since_last_load = now - loaded_at;
+    if (loaded_at && (since_last_load < throttle)) {
+        if (!$pane.hasClass('pane-reload-pending')) {
+            $pane.addClass('pane-reload-pending');
+            setTimeout((function() {
+                $pane.trigger('arv:pane:reload');
+            }), throttle - since_last_load);
+        }
+        return;
+    }
+
+    // We know this doesn't have 'pane-loading' because we tested for it above
+    $pane.removeClass('pane-reload-pending');
+    $pane.removeClass('pane-loaded');
+    $pane.removeClass('pane-stale');
+
+    if (!$pane.hasClass('active') &&
+        $pane.parent().hasClass('tab-content')) {
+        // $pane is one of the content areas in a bootstrap tabs
+        // widget, and it isn't the currently selected tab. If and
+        // when the user does select the corresponding tab, it will
+        // get a shown.bs.tab event, which will invoke this reload
+        // function again (see handler above). For now, we just insert
+        // a spinner, which will be displayed while the new content is
+        // loading.
+        $pane.html('<div class="spinner spinner-32px spinner-h-center"></div>');
+        return;
+    }
+
+    $pane.addClass('pane-loading');
+
+    var content_url = $pane.attr('data-pane-content-url');
+    $.ajax(content_url, {dataType: 'html', type: 'GET', context: $pane}).
+        done(function(data, status, jqxhr) {
+            // Preserve collapsed state
+            var $pane = this;
+            var collapsable = {};
+            $(".collapse", this).each(function(i, c) {
+                collapsable[c.id] = $(c).hasClass('in');
+            });
+            var tmp = $(data);
+            $(".collapse", tmp).each(function(i, c) {
+                if (collapsable[c.id]) {
+                    $(c).addClass('in');
                 } else {
-                    errhtml = ("An error occurred: " +
-                               (jqxhr.responseText || status)).
-                        replace(/&/g, '&amp;').
-                        replace(/</g, '&lt;').
-                        replace(/>/g, '&gt;');
+                    $(c).removeClass('in');
                 }
-                $pane.html('<div><p>' +
-                        '<a href="#" class="btn btn-primary tab_reload">' +
-                        '<i class="fa fa-fw fa-refresh"></i> ' +
-                        'Reload tab</a></p><iframe></iframe></div>');
-                $('.tab_reload', $pane).click(function() {
-                    $pane.html('<div class="spinner spinner-32px spinner-h-center"></div>');
-                    $(e.target).trigger('arv:pane:reload');
-                });
-                // We want to render the error in an iframe, in order to
-                // avoid conflicts with the main page's element ids, etc.
-                // In order to do that dynamically, we have to set a
-                // timeout on the iframe window to load our HTML *after*
-                // the default source (e.g., about:blank) has loaded.
-                var iframe = $('iframe', e.target)[0];
-                iframe.contentWindow.setTimeout(function() {
-                    $('body', iframe.contentDocument).html(errhtml);
-                    iframe.height = iframe.contentDocument.body.scrollHeight + "px";
-                }, 1);
-                $(e.target).addClass('loaded');
             });
-    } else {
-        // When the user selects e.target tab, show a spinner instead of
-        // old content while loading.
-        $pane.html('<div class="spinner spinner-32px spinner-h-center"></div>');
-    }
+            $pane.html(tmp);
+            $pane.removeClass('pane-loading');
+            $pane.addClass('pane-loaded');
+            $pane.attr('data-loaded-at', (new Date()).getTime());
+            $pane.trigger('arv:pane:loaded');
+
+            if ($pane.hasClass('pane-stale')) {
+                $pane.trigger('arv:pane:reload');
+            }
+        }).fail(function(jqxhr, status, error) {
+            var $pane = this;
+            var errhtml;
+            var contentType = jqxhr.getResponseHeader('Content-Type');
+            if (contentType && contentType.match(/\btext\/html\b/)) {
+                var $response = $(jqxhr.responseText);
+                var $wrapper = $('div#page-wrapper', $response);
+                if ($wrapper.length) {
+                    errhtml = $wrapper.html();
+                } else {
+                    errhtml = jqxhr.responseText;
+                }
+            } else {
+                errhtml = ("An error occurred: " +
+                           (jqxhr.responseText || status)).
+                    replace(/&/g, '&amp;').
+                    replace(/</g, '&lt;').
+                    replace(/>/g, '&gt;');
+            }
+            $pane.html('<div><p>' +
+                      '<a href="#" class="btn btn-primary tab_reload">' +
+                      '<i class="fa fa-fw fa-refresh"></i> ' +
+                      'Reload tab</a></p><iframe style="width: 100%"></iframe></div>');
+            $('.tab_reload', $pane).click(function() {
+                $(this).
+                    html('<div class="spinner spinner-32px spinner-h-center"></div>').
+                    closest('.pane-loaded').
+                    attr('data-loaded-at', 0).
+                    trigger('arv:pane:reload');
+            });
+            // We want to render the error in an iframe, in order to
+            // avoid conflicts with the main page's element ids, etc.
+            // In order to do that dynamically, we have to set a
+            // timeout on the iframe window to load our HTML *after*
+            // the default source (e.g., about:blank) has loaded.
+            var iframe = $('iframe', $pane)[0];
+            iframe.contentWindow.setTimeout(function() {
+                $('body', iframe.contentDocument).html(errhtml);
+                iframe.height = iframe.contentDocument.body.scrollHeight + "px";
+            }, 1);
+            $pane.removeClass('pane-loading');
+            $pane.addClass('pane-loaded');
+        });
 });
 
-// Mark all panes as stale/dirty. Refresh the active pane.
-$(document).on('arv-log-event arv:pane:reload:all', function() {
-    $('.pane-anchor.loaded').trigger('arv:pane:reload');
+// Mark all panes as stale/dirty. Refresh any 'active' panes.
+$(document).on('arv:pane:reload:all', function() {
+    $('[data-pane-content-url]').trigger('arv:pane:reload');
+});
+
+$(document).on('arv-log-event', '.arv-refresh-on-log-event', function(event) {
+    if (this != event.target) {
+        // Not interested in events sent to child nodes.
+        return;
+    }
+    // Panes marked arv-refresh-on-log-event should be refreshed
+    $(event.target).trigger('arv:pane:reload');
 });
 
 // If there is a 'tab counts url' in the nav-tabs element then use it to get some javascript that will update them
index 7007d8c348657c11dc08b19d8c051db360b603d5..7dbeac9d4ee6b59773ad842d60c572090196f898 100644 (file)
@@ -271,3 +271,7 @@ span.editable-textile {
 .compute-summary-numbers td {
   font-size: 150%;
 }
+
+.arv-log-refresh-control {
+  display: none;
+}
index b25c04a4a8a5aab43aaabc2cf14571eaf6bee0db..f76c70bdc9a203d009618d42d742106bf84991e2 100644 (file)
@@ -1,6 +1,6 @@
 .arv-job-log-window {
     height: 40em;
-    white-space: nowrap;
+    white-space: pre;
     overflow: scroll;
     background: black;
     color: white;
index 9fead2c8f0ed0a5c167984ab50cd260b1b0d665b..8f5dba1a87dea86b72615f32a0e87cb917fd684c 100644 (file)
@@ -3,7 +3,7 @@ module PipelineComponentsHelper
     begin
       render(partial: "pipeline_instances/show_components_#{template_suffix}",
              locals: locals)
-    rescue Exception => e
+    rescue => e
       logger.error "#{e.inspect}"
       logger.error "#{e.backtrace.join("\n\t")}"
       case fallback
index e9fec776de337ec588584c5715c99ef4c9bfa8d5..782a6af07996efe888489b33bf04c0145d76d9d3 100644 (file)
@@ -7,12 +7,11 @@
       <% pane_name = (pane.is_a?(Hash) ? pane[:name] : pane) %>
       <li class="<%= 'active' if i==0 %>">
         <a href="#<%= pane_name %>"
-           class="pane-anchor"
            id="<%= pane_name %>-tab"
            data-toggle="tab"
            data-tab-history=true
            data-tab-history-update-url=true
-           data-pane-content-url="<%= url_for(params.merge(tab_pane: pane_name)) %>">
+           >
           <%= pane_name.gsub('_', ' ') %> <span id="<%= pane_name %>-count"></span>
         </a>
       </li>
     <% pane_list.each_with_index do |pane, i| %>
       <% pane_name = (pane.is_a?(Hash) ? pane[:name] : pane) %>
       <div id="<%= pane_name %>"
-           class="tab-pane fade <%= 'in active loaded' if i==0 %> arv-log-event-listener"
+           class="tab-pane fade <%= 'in active pane-loaded' if i==0 %> arv-log-event-listener arv-refresh-on-log-event arv-log-event-subscribe-to-pipeline-job-uuids"
            <% if controller.action_name == "index" %>
              data-object-kind="arvados#<%= ArvadosApiClient.class_kind controller.model_class %>"
            <% else %>
              data-object-uuid="<%= @object.uuid %>"
            <% end %>
-      >
-        <div id="<%= pane_name %>-scroll" style="margin-top:0.5em;">
-          <div class="pane-content">
-            <% if i == 0 %>
-              <%= render_pane pane_name, to_string: true %>
-            <% else %>
-              <div class="spinner spinner-32px spinner-h-center"></div>
-            <% end %>
-          </div>
+           data-pane-content-url="<%= url_for(params.merge(tab_pane: pane_name)) %>"
+           style="margin-top:0.5em;"
+           >
+        <div class="pane-content">
+          <% if i == 0 %>
+            <%= render_pane pane_name, to_string: true %>
+          <% else %>
+            <div class="spinner spinner-32px spinner-h-center"></div>
+          <% end %>
         </div>
       </div>
     <% end %>
diff --git a/apps/workbench/app/views/jobs/_show_job_buttons.html.erb b/apps/workbench/app/views/jobs/_show_job_buttons.html.erb
new file mode 100644 (file)
index 0000000..644da77
--- /dev/null
@@ -0,0 +1,29 @@
+<% if @object.state != "Running" %>
+    <%= form_tag '/jobs', style: "display:inline; padding-left: 1em" do |f| %>
+      <% [:script, :script_version, :repository, :supplied_script_version, :nondeterministic].each do |d| %>
+        <%= hidden_field :job, d, :value => @object[d] %>
+      <% end %>
+      <% [:script_parameters, :runtime_constraints].each do |d| %>
+        <%= hidden_field :job, d, :value => JSON.dump(@object[d]) %>
+      <% end %>
+      <%= button_tag ({class: 'btn btn-sm btn-primary', id: "re-run-same-job-button",
+                       title: 'Re-run job using the same script version as this run'}) do %>
+        <i class="fa fa-fw fa-gear"></i> Re-run same version
+      <% end %>
+    <% end %>
+  <% if @object.respond_to? :supplied_script_version and !@object.supplied_script_version.nil? and !@object.supplied_script_version.empty? and @object.script_version != @object.supplied_script_version%>
+      <%= form_tag '/jobs', style: "display:inline" do |f| %>
+      <% [:script, :repository, :supplied_script_version, :nondeterministic].each do |d| %>
+        <%= hidden_field :job, d, :value => @object[d] %>
+      <% end %>
+      <%= hidden_field :job, :script_version, :value => @object[:supplied_script_version] %>
+      <% [:script_parameters, :runtime_constraints].each do |d| %>
+        <%= hidden_field :job, d, :value => JSON.dump(@object[d]) %>
+      <% end %>
+      <%= button_tag ({class: 'btn btn-sm btn-primary', id: "re-run-latest-job-button",
+                       title: 'Re-run job using the latest script version'}) do%>
+        <i class="fa fa-fw fa-gear"></i> Re-run latest version
+      <% end %>
+    <% end %>
+  <% end %>
+<% end %>
index 7324104a55875903c0712e05b522a897050432ad..8082d6f5a44f42d21fe07830c2978df45c5e3961 100644 (file)
@@ -1,11 +1,17 @@
 <% if !@object.log %>
 
 <% log_history = stderr_log_history([@object.uuid]) %>
-<div class="arv-log-event-listener arv-log-event-handler-append-logs arv-job-log-window" id="pipeline_event_log_div" data-object-uuids="<%= @object.uuid %>">
-  <% log_history.each do |entry| %>
-    <%=entry%><br/>
-  <% end %>
-</div>
+
+<div id="event_log_div"
+     class="arv-log-event-listener arv-log-event-handler-append-logs arv-job-log-window"
+     data-object-uuid="<%= @object.uuid %>"
+     ><%= log_history.join("\n") %></div>
+
+<%# Applying a long throttle suppresses the auto-refresh of this
+    partial that would normally be triggered by arv-log-event. %>
+<div class="arv-log-refresh-control"
+     data-load-throttle="86486400000" <%# 1001 nights %>
+     ></div>
 
 <% else %>
 
index cec63403ef1063cb652c642ab7238ddf931c27da..807520940c9218473f01935f732556afa6cd373a 100644 (file)
@@ -1,27 +1,36 @@
-<div style="margin-top: 10px">
-<% pj = {} %>
-<% pj[:job] = @object %>
-<% pj[:name] = @object[:name] || "this job" %>
-<% tasks = JobTask.filter([['job_uuid', '=', @object.uuid]]).results %>
-<%= render partial: 'pipeline_instances/running_component', locals: {tasks: tasks, pj: pj, i: 0, expanded: true} %>
-</div>
+<div class="arv-log-refresh-control"
+     data-load-throttle="15000"
+     ></div>
+<%=
+   pj = {}
+   pj[:job] = @object
+   pj[:name] = @object[:name] || "this job"
+   pj[:progress_bar] = render(partial: "job_progress",
+                              locals: {:j => @object })
+   tasks = JobTask.filter([['job_uuid', '=', @object.uuid]]).results
+   render(partial: 'pipeline_instances/running_component',
+          locals: { tasks: tasks, pj: pj, i: 0, expanded: true})
+%>
 
-  <div class="panel panel-default">
-    <div class="panel-heading">
-      <span class="panel-title">Used in pipelines</span>
-    </div>
-    <div class="panel-body">
-<% pi = PipelineInstance.order("created_at desc").filter([["components", "like", "%#{@object.uuid}%"]]) %>
+<div class="panel panel-default">
+  <div class="panel-heading">
+    <span class="panel-title">Used in pipelines</span>
+  </div>
+  <div class="panel-body">
+    <% pi = PipelineInstance.order("created_at desc").filter([["components", "like", "%#{@object.uuid}%"]]) %>
 
-<% pi.each do |pipeline| %>
-  <% pipeline.components.each do |k, v| %>
-    <% if v[:job] and v[:job][:uuid] == @object.uuid %>
-      <div>
-      <b><%= k %></b> component of <%= link_to_if_arvados_object pipeline, friendly_name: true %> 
-      created at <%= render_localized_date(pipeline.created_at) %>.
-      </div>
+    <% pi.each do |pipeline| %>
+      <% pipeline.components.each do |k, v| %>
+        <% if v[:job] and v[:job][:uuid] == @object.uuid %>
+          <div>
+            <b><%= k %></b>
+            component of
+            <%= link_to_if_arvados_object pipeline, friendly_name: true %>
+            created at
+            <%= render_localized_date(pipeline.created_at) %>.
+          </div>
+        <% end %>
+      <% end %>
     <% end %>
-  <% end %>
-<% end %>
-</div>
+  </div>
 </div>
index 74c53ca157d892244824a918f3d5a446b854573b..566014e4f328e256e81a8ba33d31794b41a6a29f 100644 (file)
@@ -1,37 +1,10 @@
 <% content_for :tab_line_buttons do %>
-    <% if @object.state == "Running" %>
-    <%= form_tag "/jobs/#{@object.uuid}/cancel", style: "display:inline; padding-left: 1em" do |f| %>
-      <%= button_tag "Cancel running job", {class: 'btn btn-sm btn-danger', id: "cancel-job-button"} %>
-    <% end %>
-  <% else %>
-    <%= form_tag '/jobs', style: "display:inline; padding-left: 1em" do |f| %>
-      <% [:script, :script_version, :repository, :supplied_script_version, :nondeterministic].each do |d| %>
-        <%= hidden_field :job, d, :value => @object[d] %>
-      <% end %>
-      <% [:script_parameters, :runtime_constraints].each do |d| %>
-        <%= hidden_field :job, d, :value => JSON.dump(@object[d]) %>
-      <% end %>
-      <%= button_tag ({class: 'btn btn-sm btn-primary', id: "re-run-same-job-button",
-                       title: 'Re-run job using the same script version as this run'}) do %>
-        <i class="fa fa-fw fa-gear"></i> Re-run same version
-      <% end %>
-    <% end %>
-  <% if @object.respond_to? :supplied_script_version and !@object.supplied_script_version.nil? and !@object.supplied_script_version.empty? and @object.script_version != @object.supplied_script_version%>
-      <%= form_tag '/jobs', style: "display:inline" do |f| %>
-      <% [:script, :repository, :supplied_script_version, :nondeterministic].each do |d| %>
-        <%= hidden_field :job, d, :value => @object[d] %>
-      <% end %>
-      <%= hidden_field :job, :script_version, :value => @object[:supplied_script_version] %>
-      <% [:script_parameters, :runtime_constraints].each do |d| %>
-        <%= hidden_field :job, d, :value => JSON.dump(@object[d]) %>
-      <% end %>
-      <%= button_tag ({class: 'btn btn-sm btn-primary', id: "re-run-latest-job-button",
-                       title: 'Re-run job using the latest script version'}) do%>
-        <i class="fa fa-fw fa-gear"></i> Re-run latest version
-      <% end %>
-    <% end %>
-  <% end %>
-<% end %>
+  <div class="pane-loaded arv-log-event-listener arv-refresh-on-state-change"
+       data-pane-content-url="<%= url_for(params.merge(tab_pane: "job_buttons")) %>"
+       data-object-uuid="<%= @object.uuid %>"
+       style="display: inline">
+    <%= render partial: 'show_job_buttons', locals: {object: @object}%>
+  </div>
 <% end %>
 
 <%= render partial: 'title_and_buttons' %>
index 85a1530204f18a6633feb07415438faec80fb66b..9176652ca34a9a43ad59fda8dd3f8b9941966c0f 100644 (file)
@@ -78,7 +78,7 @@
           <% if current_job[:state].in? ["Queued", "Running"] %>
             <%# column offset 11 %>
             <div class="col-md-1 pipeline-instance-spacing">
-              <%= form_tag "/jobs/#{current_job[:uuid]}/cancel", style: "display:inline; padding-left: 1em" do |f| %>
+              <%= form_tag "/jobs/#{current_job[:uuid]}/cancel", remote: true, style: "display:inline; padding-left: 1em" do |f| %>
                 <%= hidden_field_tag :return_to, url_for(@object) %>
                 <%= button_tag "Cancel", {class: 'btn btn-xs btn-danger', id: "cancel-job-button"} %>
               <% end %>
index 94cbf984897c66753ff6728a7d2f2196d9f540dd..7735997748389e1d3fa68713dc53c39626b961bf 100644 (file)
@@ -1,14 +1,13 @@
 <% if !@object.state.in? ['New', 'Ready'] %>
 
-  <div class="pull-right" style="padding-left: 1em">
-    Current state: <span class="badge badge-info" data-pipeline-state="<%= @object.state %>">
-      <% if @object.state == "RunningOnServer" %>
-        Active
-      <% else %>
-        <%= @object.state %>
-      <% end %>
-    </span>&nbsp;
-  </div>
+  <%
+     job_uuids = @object.components.map { |k,j| j.is_a? Hash and j[:job].andand[:uuid] }.compact
+     throttle = @object.state.start_with?('Running') ? 5000 : 15000
+     %>
+  <div class="arv-log-refresh-control"
+       data-load-throttle="<%= throttle %>"
+       data-object-uuids="<%= @object.uuid %> <%= job_uuids.join(' ') %>"
+       ></div>
 
   <%= render_pipeline_components("running", :json) %>
 
index 165a694e8b3e513ce470e698e9d91dc41af74d52..d99ac23ab8c969f08f50cb3132ac610502762e0a 100644 (file)
@@ -1,5 +1,18 @@
 <%# Summary %>
 
+<div class="pull-right" style="padding-left: 1em">
+  Current state: <span class="badge badge-info" data-pipeline-state="<%= @object.state %>">
+    <% if @object.state == "RunningOnServer" %>
+      Active
+    <% else %>
+      <%= @object.state %>
+    <% end %>
+  </span>&nbsp;
+</div>
+
+<% pipeline_jobs = render_pipeline_jobs %>
+<% job_uuids = pipeline_jobs.map { |j| j[:job].andand[:uuid] }.compact %>
+
 <% if @object.state == 'Paused' %>
   <p>
     This pipeline is paused.  Jobs that are
@@ -7,8 +20,8 @@
   </p>
 <% end %>
 
-<% tasks = JobTask.filter([['job_uuid', 'in', render_pipeline_jobs.map { |j| j[:job].andand[:uuid] }.compact]]).results %>
-<% runningtime = determine_wallclock_runtime(render_pipeline_jobs.map {|j| j[:job]}.compact) %>
+<% tasks = JobTask.filter([['job_uuid', 'in', job_uuids]]).results %>
+<% runningtime = determine_wallclock_runtime(pipeline_jobs.map {|j| j[:job]}.compact) %>
 
 <p>
   <% if @object.started_at %>
@@ -68,6 +81,6 @@
 
 <%# Components %>
 
-<% render_pipeline_jobs.each_with_index do |pj, i| %>
+<% pipeline_jobs.each_with_index do |pj, i| %>
   <%= render partial: 'running_component', locals: {tasks: tasks, pj: pj, i: i, expanded: false} %>
 <% end %>
index 2fdb45bdb681eacfe41e53d79de5e04857929435..bb756a08274044fd867ca3cd18c5e10946bb5a5e 100644 (file)
@@ -1,6 +1,12 @@
-<% log_history = stderr_log_history([@object.uuid] + pipeline_jobs(@object).collect{|x|x[:job].andand[:uuid]}.compact) %>
-<div class="arv-log-event-listener arv-log-event-handler-append-logs arv-job-log-window" id="pipeline_event_log_div" data-object-uuids="<%= @object.uuid %>" data-object-uuids-live="#Components tr[data-object-uuid]">
-  <% log_history.each do |entry| %>
-    <%=entry%><br/>
-  <% end %>
-</div>
+<% log_uuids = [@object.uuid] + pipeline_jobs(@object).collect{|x|x[:job].andand[:uuid]}.compact %>
+<% log_history = stderr_log_history(log_uuids) %>
+<div id="event_log_div"
+     class="arv-log-event-listener arv-log-event-handler-append-logs arv-log-event-subscribe-to-pipeline-job-uuids arv-job-log-window"
+     data-object-uuids="<%= log_uuids.join(' ') %>"
+     ><%= log_history.join("\n") %></div>
+
+<%# Applying a long throttle suppresses the auto-refresh of this
+    partial that would normally be triggered by arv-log-event. %>
+<div class="arv-log-refresh-control"
+     data-load-throttle="86486400000" <%# 1001 nights %>
+     ></div>
index 0b72f6997a3871e502e6c233d41207bfa28fa1fa..860e8091b26dd2780974748c145d265ef174cc16 100644 (file)
 <% content_for :tab_line_buttons do %>
 
   <div id="pipeline-instance-tab-buttons"
-       class="pane-anchor loaded"
-       href="#pipeline-instance-tab-buttons-pane"
+       class="pane-loaded arv-log-event-listener arv-refresh-on-state-change"
        data-pane-content-url="<%= url_for(params.merge(tab_pane: "tab_buttons")) %>"
+       data-object-uuid="<%= @object.uuid %>"
        >
-    <div id="pipeline-instance-tab-buttons-pane" class="active">
-      <%= render partial: 'show_tab_buttons', locals: {object: @object}%>
-    </div>
+    <%= render partial: 'show_tab_buttons', locals: {object: @object}%>
   </div>
+
 <% end %>
 
 <%= render partial: 'content', layout: 'content_layout', locals: {pane_list: controller.show_pane_list }%>
index 4c0450c6302dbd71d1ce581817650e133f3d0cd1..dc70da8babf79de337058299e52b89fa1c16bb26 100644 (file)
@@ -1,15 +1,7 @@
-<% content_for :js do %>
-    setInterval(function(){
-        $('#dashboard-content').trigger('arv:pane:reload');
-    }, 15000);
-<% end %>
-
-<div id="dashboard-content"
-     class="pane-anchor loaded"
-     href="#dashboard-content-pane"
+<div class="pane-loaded arv-log-event-listener arv-refresh-on-log-event"
      data-pane-content-url="<%= root_url tab_pane: "dashboard" %>"
+     data-object-uuid="all"
+     data-load-throttle="15000"
      >
-  <div id="dashboard-content-pane" class="active">
-    <%= render partial: 'show_dashboard' %>
-  </div>
+  <%= render partial: 'show_dashboard' %>
 </div>
diff --git a/apps/workbench/test/integration/websockets_test.rb b/apps/workbench/test/integration/websockets_test.rb
new file mode 100644 (file)
index 0000000..341975f
--- /dev/null
@@ -0,0 +1,160 @@
+require 'integration_helper'
+require 'selenium-webdriver'
+require 'headless'
+
+class WebsocketTest < ActionDispatch::IntegrationTest
+
+  setup do
+    headless = Headless.new
+    headless.start
+    Capybara.current_driver = :selenium
+  end
+
+  test "test page" do
+    visit(page_with_token("admin", "/websockets"))
+    fill_in("websocket-message-content", :with => "Stuff")
+    click_button("Send")
+    assert_text '"status":400'
+  end
+
+  test "test live logging" do
+    visit(page_with_token("admin", "/pipeline_instances/zzzzz-d1hrv-9fm8l10i9z2kqc6"))
+    click_link("Log")
+    assert_no_text '123 hello'
+
+    api = ArvadosApiClient.new
+
+    Thread.current[:arvados_api_token] = @@API_AUTHS["admin"]['api_token']
+    api.api("logs", "", {log: {
+                object_uuid: "zzzzz-d1hrv-9fm8l10i9z2kqc6",
+                event_type: "stderr",
+                properties: {"text" => "123 hello"}}})
+    assert_text '123 hello'
+    Thread.current[:arvados_api_token] = nil
+  end
+
+
+  [["pipeline_instances", api_fixture("pipeline_instances")['pipeline_with_newer_template']['uuid']],
+   ["jobs", api_fixture("jobs")['running']['uuid']]].each do |c|
+    test "test live logging scrolling #{c[0]}" do
+
+      controller = c[0]
+      uuid = c[1]
+
+      visit(page_with_token("admin", "/#{controller}/#{uuid}"))
+      click_link("Log")
+      assert_no_text '123 hello'
+
+      api = ArvadosApiClient.new
+
+      text = ""
+      (1..1000).each do |i|
+        text << "#{i} hello\n"
+      end
+
+      Thread.current[:arvados_api_token] = @@API_AUTHS["admin"]['api_token']
+      api.api("logs", "", {log: {
+                  object_uuid: uuid,
+                  event_type: "stderr",
+                  properties: {"text" => text}}})
+      assert_text '1000 hello'
+
+      # First test that when we're already at the bottom of the page, it scrolls down
+      # when a new line is added.
+      old_top = page.evaluate_script("$('#event_log_div').scrollTop()")
+
+      api.api("logs", "", {log: {
+                  object_uuid: uuid,
+                  event_type: "stderr",
+                  properties: {"text" => "1001 hello\n"}}})
+      assert_text '1001 hello'
+
+      # Check that new value of scrollTop is greater than the old one
+      assert page.evaluate_script("$('#event_log_div').scrollTop()") > old_top
+
+      # Now scroll to 30 pixels from the top
+      page.execute_script "$('#event_log_div').scrollTop(30)"
+      assert_equal 30, page.evaluate_script("$('#event_log_div').scrollTop()")
+
+      api.api("logs", "", {log: {
+                  object_uuid: uuid,
+                  event_type: "stderr",
+                  properties: {"text" => "1002 hello\n"}}})
+      assert_text '1002 hello'
+
+      # Check that we haven't changed scroll position
+      assert_equal 30, page.evaluate_script("$('#event_log_div').scrollTop()")
+
+      Thread.current[:arvados_api_token] = nil
+    end
+  end
+
+  test "pipeline instance arv-refresh-on-log-event" do
+    Thread.current[:arvados_api_token] = @@API_AUTHS["admin"]['api_token']
+    # Do something and check that the pane reloads.
+    p = PipelineInstance.create({state: "RunningOnServer",
+                                  components: {
+                                    c1: {
+                                      script: "test_hash.py",
+                                      script_version: "1de84a854e2b440dc53bf42f8548afa4c17da332"
+                                    }
+                                  }
+                                })
+
+    visit(page_with_token("admin", "/pipeline_instances/#{p.uuid}"))
+
+    assert_text 'Active'
+    assert page.has_link? 'Pause'
+    assert_no_text 'Complete'
+    assert page.has_no_link? 'Re-run with latest'
+
+    p.state = "Complete"
+    p.save!
+
+    assert_no_text 'Active'
+    assert page.has_no_link? 'Pause'
+    assert_text 'Complete'
+    assert page.has_link? 'Re-run with latest'
+
+    Thread.current[:arvados_api_token] = nil
+  end
+
+  test "job arv-refresh-on-log-event" do
+    Thread.current[:arvados_api_token] = @@API_AUTHS["admin"]['api_token']
+    # Do something and check that the pane reloads.
+    p = Job.where(uuid: api_fixture('jobs')['running_will_be_completed']['uuid']).results.first
+
+    visit(page_with_token("admin", "/jobs/#{p.uuid}"))
+
+    assert_no_text 'complete'
+    assert_no_text 'Re-run same version'
+
+    p.state = "Complete"
+    p.save!
+
+    assert_text 'complete'
+    assert_text 'Re-run same version'
+
+    Thread.current[:arvados_api_token] = nil
+  end
+
+  test "dashboard arv-refresh-on-log-event" do
+    Thread.current[:arvados_api_token] = @@API_AUTHS["admin"]['api_token']
+
+    visit(page_with_token("admin", "/"))
+
+    assert_no_text 'test dashboard arv-refresh-on-log-event'
+
+    # Do something and check that the pane reloads.
+    p = PipelineInstance.create({state: "RunningOnServer",
+                                  name: "test dashboard arv-refresh-on-log-event",
+                                  components: {
+                                  }
+                                })
+
+    assert_text 'test dashboard arv-refresh-on-log-event'
+
+    Thread.current[:arvados_api_token] = nil
+  end
+
+end
index 49b9c7f123ce8f3e88bc12e9724b9b493ca4f2b8..ab2ac395b4c487b85643d6df5989934acd7e630f 100644 (file)
@@ -97,18 +97,19 @@ end
 class ApiServerForTests
   ARV_API_SERVER_DIR = File.expand_path('../../../../services/api', __FILE__)
   SERVER_PID_PATH = File.expand_path('tmp/pids/wbtest-server.pid', ARV_API_SERVER_DIR)
+  WEBSOCKET_PID_PATH = File.expand_path('tmp/pids/wstest-server.pid', ARV_API_SERVER_DIR)
   @main_process_pid = $$
 
-  def self._system(*cmd)
+  def _system(*cmd)
     $stderr.puts "_system #{cmd.inspect}"
     Bundler.with_clean_env do
-      if not system({'RAILS_ENV' => 'test'}, *cmd)
+      if not system({'RAILS_ENV' => 'test', "ARVADOS_WEBSOCKETS" => (if @websocket then "ws-only" end)}, *cmd)
         raise RuntimeError, "#{cmd[0]} returned exit code #{$?.exitstatus}"
       end
     end
   end
 
-  def self.make_ssl_cert
+  def make_ssl_cert
     unless File.exists? './self-signed.key'
       _system('openssl', 'req', '-new', '-x509', '-nodes',
               '-out', './self-signed.pem',
@@ -118,42 +119,55 @@ class ApiServerForTests
     end
   end
 
-  def self.kill_server
+  def kill_server
     if (pid = find_server_pid)
       $stderr.puts "Sending TERM to API server, pid #{pid}"
       Process.kill 'TERM', pid
     end
   end
 
-  def self.find_server_pid
+  def find_server_pid
     pid = nil
     begin
-      pid = IO.read(SERVER_PID_PATH).to_i
+      pid = IO.read(@pidfile).to_i
       $stderr.puts "API server is running, pid #{pid.inspect}"
     rescue Errno::ENOENT
     end
     return pid
   end
 
-  def self.run(args=[])
+  def run(args=[])
     ::MiniTest.after_run do
       self.kill_server
     end
 
+    @websocket = args.include?("--websockets")
+
+    @pidfile = if @websocket
+                 WEBSOCKET_PID_PATH
+               else
+                 SERVER_PID_PATH
+               end
+
     # Kill server left over from previous test run
     self.kill_server
 
     Capybara.javascript_driver = :poltergeist
     Dir.chdir(ARV_API_SERVER_DIR) do |apidir|
       ENV["NO_COVERAGE_TEST"] = "1"
-      make_ssl_cert
-      _system('bundle', 'exec', 'rake', 'db:test:load')
-      _system('bundle', 'exec', 'rake', 'db:fixtures:load')
-      _system('bundle', 'exec', 'passenger', 'start', '-d', '-p3000',
-              '--pid-file', SERVER_PID_PATH,
-              '--ssl',
-              '--ssl-certificate', 'self-signed.pem',
-              '--ssl-certificate-key', 'self-signed.key')
+      if @websocket
+        _system('bundle', 'exec', 'passenger', 'start', '-d', '-p3333',
+                '--pid-file', @pidfile)
+      else
+        make_ssl_cert
+        _system('bundle', 'exec', 'rake', 'db:test:load')
+        _system('bundle', 'exec', 'rake', 'db:fixtures:load')
+        _system('bundle', 'exec', 'passenger', 'start', '-d', '-p3000',
+                '--pid-file', @pidfile,
+                '--ssl',
+                '--ssl-certificate', 'self-signed.pem',
+                '--ssl-certificate-key', 'self-signed.key')
+      end
       timeout = Time.now.tv_sec + 10
       good_pid = false
       while (not good_pid) and (Time.now.tv_sec < timeout)
@@ -206,5 +220,6 @@ class RequestDuck
 end
 
 if ENV["RAILS_ENV"].eql? 'test'
-  ApiServerForTests.run
+  ApiServerForTests.new.run
+  ApiServerForTests.new.run ["--websockets"]
 end
index 227dd75b276995f05351aa42fc639f477016b525..500f6f6bd0eb97b4740a0a680e1ac05b97bb579a 100644 (file)
@@ -18,6 +18,8 @@ This command will download the latest copy of the Arvados docker images. It also
 
 This installation method assumes your web browser and the Arvados docker containers run on the same host. 
 
+If you prefer, you can also download the installation script and inspect it before running it. The @http://get.arvados.org@ url redirects to <a href="https://raw.githubusercontent.com/curoverse/arvados-dev/master/install/easy-docker-install.sh">https://raw.githubusercontent.com/curoverse/arvados-dev/master/install/easy-docker-install.sh</a>, which is the installation script.
+
 h2. Installation from source
 
 It is also possible to build the Arvados docker images from source. The instructions are available "here":install-docker.html.
index 3a90d6d3b0fb64d2ff49f4262b6e6c5ff3924cc9..390ed3b3afeb4922c81ec48682b1c6a81206e641 100644 (file)
@@ -177,22 +177,22 @@ class CollectionReader(CollectionBase):
             return
         error_via_api = None
         error_via_keep = None
-        should_try_keep = (not self._manifest_text and
+        should_try_keep = ((self._manifest_text is None) and
                            util.keep_locator_pattern.match(
                 self._manifest_locator))
-        if (not self._manifest_text and
+        if ((self._manifest_text is None) and
             util.signed_locator_pattern.match(self._manifest_locator)):
             error_via_keep = self._populate_from_keep()
-        if not self._manifest_text:
+        if self._manifest_text is None:
             error_via_api = self._populate_from_api_server()
             if error_via_api is not None and not should_try_keep:
                 raise error_via_api
-        if (not self._manifest_text and
+        if ((self._manifest_text is None) and
             not error_via_keep and
             should_try_keep):
             # Looks like a keep locator, and we didn't already try keep above
             error_via_keep = self._populate_from_keep()
-        if not self._manifest_text:
+        if self._manifest_text is None:
             # Nothing worked!
             raise arvados.errors.NotFoundError(
                 ("Failed to retrieve collection '{}' " +
index 82c6424425460f59b5c361f99c1f0b1627a5b509..16244025994a001cba4d988c9ffe221948235098 100644 (file)
@@ -95,27 +95,19 @@ def run(websockets=False, reuse_server=False):
         subprocess.call(['bundle', 'exec', 'rake', 'db:test:load'])
         subprocess.call(['bundle', 'exec', 'rake', 'db:fixtures:load'])
 
+        subprocess.call(['bundle', 'exec', 'rails', 'server', '-d',
+                         '--pid',
+                         os.path.join(os.getcwd(), SERVER_PID_PATH),
+                         '-p3000'])
+        os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3000"
+
         if websockets:
-            os.environ["ARVADOS_WEBSOCKETS"] = "true"
-            subprocess.call(['openssl', 'req', '-new', '-x509', '-nodes',
-                             '-out', './self-signed.pem',
-                             '-keyout', './self-signed.key',
-                             '-days', '3650',
-                             '-subj', '/CN=localhost'])
+            os.environ["ARVADOS_WEBSOCKETS"] = "ws-only"
             subprocess.call(['bundle', 'exec',
                              'passenger', 'start', '-d', '-p3333',
                              '--pid-file',
-                             os.path.join(os.getcwd(), WEBSOCKETS_SERVER_PID_PATH),
-                             '--ssl',
-                             '--ssl-certificate', 'self-signed.pem',
-                             '--ssl-certificate-key', 'self-signed.key'])
-            os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3333"
-        else:
-            subprocess.call(['bundle', 'exec', 'rails', 'server', '-d',
-                             '--pid',
-                             os.path.join(os.getcwd(), SERVER_PID_PATH),
-                             '-p3000'])
-            os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3000"
+                             os.path.join(os.getcwd(), WEBSOCKETS_SERVER_PID_PATH)
+                         ])
 
         pid = find_server_pid(SERVER_PID_PATH)
 
index c4c7ca238af4004c602e1819c28ab3aeca11319c..e275f384659e877a9817d481f51c9b367410b448 100644 (file)
@@ -801,6 +801,13 @@ class CollectionReaderTestCase(unittest.TestCase, CollectionTestMixin):
             [[f.size(), f.stream_name(), f.name()]
              for f in reader.all_streams()[0].all_files()])
 
+    def test_read_empty_collection(self):
+        client = self.api_client_mock(200)
+        self.mock_get_collection(client, 200, 'empty')
+        reader = arvados.CollectionReader('d41d8cd98f00b204e9800998ecf8427e+0',
+                                          api_client=client)
+        self.assertEqual('', reader.manifest_text())
+
 
 @tutil.skip_sleep
 class CollectionWriterTestCase(unittest.TestCase, CollectionTestMixin):
index 92b3ca1fa688a75043f36a67540fb748ef3cc5ad..1b1566d29e6ee486a7224be6b361ac22dc830598 100644 (file)
@@ -45,6 +45,7 @@ test:
   blob_signing_key: zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc
   user_profile_notification_address: arvados@example.com
   workbench_address: https://localhost:3001/
+  websocket_address: ws://127.0.0.1:3333/websocket
 
 common:
   uuid_prefix: <%= Digest::MD5.hexdigest(`hostname`).to_i(16).to_s(36)[0..4] %>
index 5d9323a5f677b1a669a327fe9c883dc21495a39f..ebd5165669f3d396bfa35e266a2e217ef4285ee5 100755 (executable)
@@ -93,6 +93,9 @@ class Dispatcher
   def slurm_status
     slurm_nodes = {}
     each_slurm_line("sinfo", "%t") do |hostname, state|
+      # Treat nodes in idle* state as down, because the * means that slurm
+      # hasn't been able to communicate with it recently.
+      state.sub!(/^idle\*/, "down")
       state.sub!(/\W+$/, "")
       state = "down" unless %w(idle alloc down).include?(state)
       slurm_nodes[hostname] = {state: state, job: nil}
@@ -202,9 +205,15 @@ class Dispatcher
     rescue
       $stderr.puts "dispatch: log.create failed"
     end
-    job.state = "Failed"
-    if not job.save
-      $stderr.puts "dispatch: job.save failed"
+
+    begin
+      job.lock @authorizations[job.uuid].user.uuid
+      job.state = "Failed"
+      if not job.save
+        $stderr.puts "dispatch: save failed setting job #{job.uuid} to failed"
+      end
+    rescue ArvadosModel::AlreadyLockedError
+      $stderr.puts "dispatch: tried to mark job #{job.uuid} as failed but it was already locked by someone else"
     end
   end
 
index ebcecf401ea1d8929ba0c26a2c60ea2ee14ae9f3..6721c12eedc0b868978cbaa937b1060b8376e8a4 100644 (file)
@@ -297,6 +297,30 @@ job_in_subproject:
   script_version: 4fe459abe02d9b365932b8f5dc419439ab4e2577
   state: Complete
 
+running_will_be_completed:
+  uuid: zzzzz-8i9sb-rshmckwoma9pjh8
+  owner_uuid: zzzzz-j7d0g-v955i6s2oi1cbso
+  cancelled_at: ~
+  cancelled_by_user_uuid: ~
+  cancelled_by_client_uuid: ~
+  created_at: <%= 3.minute.ago.to_s(:db) %>
+  started_at: <%= 3.minute.ago.to_s(:db) %>
+  finished_at: ~
+  script_version: 1de84a854e2b440dc53bf42f8548afa4c17da332
+  running: true
+  success: ~
+  output: ~
+  priority: 0
+  log: ~
+  is_locked_by_uuid: zzzzz-tpzed-d9tiejq69daie8f
+  tasks_summary:
+    failed: 0
+    todo: 3
+    running: 1
+    done: 1
+  runtime_constraints: {}
+  state: Running
+
 graph_stage1:
   uuid: zzzzz-8i9sb-graphstage10000
   owner_uuid: zzzzz-j7d0g-v955i6s2oi1cbso