add jobs#log_tail_follow

diff --git a/services/api/Gemfile b/services/api/Gemfile
index 6ca9d4709921b4b6c1421e19753baaee68188041..f11141149f7e3ec5f6d1d039523d5ac3d31e0c4d 100644 (file)
--- a/services/api/Gemfile
+++ b/services/api/Gemfile
@@ -50,3 +50,4 @@ gem 'omniauth', '1.1.1'
 gem 'omniauth-oauth2', '1.1.1'
 
 gem 'andand'
+gem 'redis'

diff --git a/services/api/Gemfile.lock b/services/api/Gemfile.lock
index b41c98354cecf531bc87f86c1b1005b74fce2b96..83a324cf3b05ff925c2eb00277dc0c8f900b21ec 100644 (file)
--- a/services/api/Gemfile.lock
+++ b/services/api/Gemfile.lock
@@ -36,7 +36,7 @@ GEM
     andand (1.3.3)
     arel (2.2.3)
     builder (3.0.4)
-    capistrano (2.14.1)
+    capistrano (2.15.4)
       highline
       net-scp (>= 1.0.0)
       net-sftp (>= 2.0.0)
@@ -48,66 +48,64 @@ GEM
     coffee-script (2.2.0)
       coffee-script-source
       execjs
-    coffee-script-source (1.4.0)
-    daemon_controller (1.1.0)
+    coffee-script-source (1.6.2)
+    daemon_controller (1.1.4)
     erubis (2.7.0)
     execjs (1.4.0)
       multi_json (~> 1.0)
-    faraday (0.8.4)
+    faraday (0.8.7)
       multipart-post (~> 1.1)
-    fastthread (1.0.7)
     hashie (1.2.0)
-    highline (1.6.15)
-    hike (1.2.1)
+    highline (1.6.19)
+    hike (1.2.3)
     httpauth (0.2.0)
-    i18n (0.6.1)
-    jquery-rails (2.1.4)
+    i18n (0.6.4)
+    jquery-rails (3.0.1)
       railties (>= 3.0, < 5.0)
       thor (>= 0.14, < 2.0)
-    json (1.7.6)
-    jwt (0.1.5)
+    json (1.8.0)
+    jwt (0.1.6)
       multi_json (>= 1.0)
-    libv8 (3.11.8.13)
+    libv8 (3.11.8.17)
     mail (2.3.3)
       i18n (>= 0.4.0)
       mime-types (~> 1.16)
       treetop (~> 1.4.8)
-    mime-types (1.19)
+    mime-types (1.23)
     multi_json (1.2.0)
-    multipart-post (1.1.5)
-    net-scp (1.0.4)
-      net-ssh (>= 1.99.1)
-    net-sftp (2.0.5)
-      net-ssh (>= 2.0.9)
-    net-ssh (2.6.3)
-    net-ssh-gateway (1.1.0)
-      net-ssh (>= 1.99.1)
-    oauth2 (0.8.0)
+    multipart-post (1.2.0)
+    net-scp (1.1.1)
+      net-ssh (>= 2.6.5)
+    net-sftp (2.1.2)
+      net-ssh (>= 2.6.5)
+    net-ssh (2.6.7)
+    net-ssh-gateway (1.2.0)
+      net-ssh (>= 2.6.5)
+    oauth2 (0.8.1)
       faraday (~> 0.8)
       httpauth (~> 0.1)
       jwt (~> 0.1.4)
       multi_json (~> 1.0)
       rack (~> 1.2)
-    oj (2.0.3)
+    oj (2.1.2)
     omniauth (1.1.1)
       hashie (~> 1.2)
       rack
     omniauth-oauth2 (1.1.1)
       oauth2 (~> 0.8.0)
       omniauth (~> 1.0)
-    passenger (3.0.19)
-      daemon_controller (>= 1.0.0)
-      fastthread (>= 1.0.1)
+    passenger (4.0.5)
+      daemon_controller (>= 1.1.0)
       rack
       rake (>= 0.8.1)
-    pg (0.14.1)
+    pg (0.15.1)
     polyglot (0.3.3)
-    rack (1.3.9)
+    rack (1.3.10)
     rack-cache (1.2)
       rack (>= 0.4)
     rack-mount (0.8.3)
       rack (>= 1.0.0)
-    rack-ssl (1.3.2)
+    rack-ssl (1.3.3)
       rack
     rack-test (0.6.2)
       rack (>= 1.0)
@@ -126,11 +124,12 @@ GEM
       rake (>= 0.8.7)
       rdoc (~> 3.4)
       thor (~> 0.14.6)
-    rake (10.0.3)
-    rdoc (3.12)
+    rake (10.1.0)
+    rdoc (3.12.2)
       json (~> 1.4)
-    ref (1.0.2)
-    rvm-capistrano (1.2.7)
+    redis (3.0.4)
+    ref (1.0.5)
+    rvm-capistrano (1.3.2)
       capistrano (>= 2.0.0)
     sass (3.1.21)
     sass-rails (3.1.5)
@@ -142,16 +141,16 @@ GEM
       hike (~> 1.2)
       rack (~> 1.0)
       tilt (~> 1.1, != 1.3.0)
-    therubyracer (0.11.3)
+    therubyracer (0.11.4)
       libv8 (~> 3.11.8.12)
       ref
     thor (0.14.6)
-    tilt (1.3.3)
-    treetop (1.4.12)
+    tilt (1.3.7)
+    treetop (1.4.14)
       polyglot
       polyglot (>= 0.3.1)
-    tzinfo (0.3.35)
-    uglifier (1.3.0)
+    tzinfo (0.3.37)
+    uglifier (2.1.1)
       execjs (>= 0.3.0)
       multi_json (~> 1.0, >= 1.0.2)
 
@@ -170,6 +169,7 @@ DEPENDENCIES
   passenger
   pg
   rails (= 3.1.10)
+  redis
   rvm-capistrano
   sass-rails (= 3.1.5)
   therubyracer

diff --git a/services/api/app/controllers/arvados/v1/jobs_controller.rb b/services/api/app/controllers/arvados/v1/jobs_controller.rb
index efd08e8398bbdea0be6d396bbaecf11b38a8f6c8..759f437fd580e647b35c9dc386580b0f9061cc6b 100644 (file)
--- a/services/api/app/controllers/arvados/v1/jobs_controller.rb
+++ b/services/api/app/controllers/arvados/v1/jobs_controller.rb
@@ -40,6 +40,36 @@ class Arvados::V1::JobsController < ApplicationController
     super
   end
 
+  class LogStreamer
+    def initialize(job)
+      @job = job
+    end
+    def each
+      if @job.finished_at
+        yield "#{@job.uuid} finished at #{@job.finished_at}\n"
+        return
+      end
+      @redis = Redis.new(:timeout => 0)
+      @redis.subscribe(@job.uuid) do |event|
+        event.message do |channel, msg|
+          if msg == "end"
+            @redis.unsubscribe @job.uuid
+          else
+            yield "#{msg}\n"
+          end
+        end
+      end
+    end
+  end
+
+  def log_tail_follow
+    if !@object.andand.uuid
+      return render_not_found
+    end
+    self.response.headers['Last-Modified'] = Time.now.ctime.to_s
+    self.response_body = LogStreamer.new @object
+  end
+
   def queue
     load_where_param
     @where.merge!({

diff --git a/services/api/config/routes.rb b/services/api/config/routes.rb
index b6132c162d6b9b771ecf3c5ed4adc8176708eb1e..ca4f7b0ed06e0d7433a5efc1bdf75fbcbc7b6f53 100644 (file)
--- a/services/api/config/routes.rb
+++ b/services/api/config/routes.rb
@@ -84,6 +84,7 @@ Server::Application.routes.draw do
       match '/users/current' => 'users#current'
       match '/users/system' => 'users#system'
       match '/jobs/queue' => 'jobs#queue'
+      match '/jobs/:uuid/log_tail_follow' => 'jobs#log_tail_follow'
       match '/virtual_machines/get_all_logins' => 'virtual_machines#get_all_logins'
       match '/repositories/get_all_permissions' => 'repositories#get_all_permissions'
       resources :collections

diff --git a/services/api/script/crunch-dispatch.rb b/services/api/script/crunch-dispatch.rb
index ab49aabc298c8186d099cf8bbfb9d5fddeefc46a..762afe9c344df1061865cf3efb764cf2723e25ad 100755 (executable)
--- a/services/api/script/crunch-dispatch.rb
+++ b/services/api/script/crunch-dispatch.rb
@@ -17,6 +17,8 @@ require File.dirname(__FILE__) + '/../config/boot'
 require File.dirname(__FILE__) + '/../config/environment'
 require 'open3'
 
+$redis ||= Redis.new
+
 class Dispatcher
   include ApplicationHelper
 
@@ -169,6 +171,7 @@ class Dispatcher
           lines.each do |line|
             $stderr.print "#{job_uuid} ! " unless line.index(job_uuid)
             $stderr.puts line
+            $redis.publish job_uuid, "#{Time.now.ctime.to_s} #{line}"
           end
         end
       end
@@ -213,6 +216,7 @@ class Dispatcher
     job_done = j_done[:job]
     $stderr.puts "dispatch: child #{pid_done} exit"
     $stderr.puts "dispatch: job #{job_done.uuid} end"
+    $redis.publish job_done.uuid, "end"
 
     # Ensure every last drop of stdout and stderr is consumed
     read_pipes