services/api/app/controllers/arvados/v1/jobs_controller.rb

   1 class Arvados::V1::JobsController < ApplicationController
   2   accept_attribute_as_json :script_parameters, Hash
   3   accept_attribute_as_json :runtime_constraints, Hash
   4   accept_attribute_as_json :tasks_summary, Hash
   5   skip_before_filter :find_object_by_uuid, :only => :queue
   6   skip_before_filter :render_404_if_no_object, :only => :queue
   7
   8   def index
   9     want_ancestor = @where[:script_version_descends_from]
  10     if want_ancestor
  11       # Check for missing commit_ancestor rows, and create them if
  12       # possible.
  13       @objects.
  14         dup.
  15         includes(:commit_ancestors). # I wish Rails would let me
  16                                      # specify here which
  17                                      # commit_ancestors I am
  18                                      # interested in.
  19         each do |o|
  20         if o.commit_ancestors.
  21             select { |ca| ca.ancestor == want_ancestor }.
  22             empty? and !o.script_version.nil?
  23           begin
  24             o.commit_ancestors << CommitAncestor.find_or_create_by_descendant_and_ancestor(o.script_version, want_ancestor)
  25           rescue
  26           end
  27         end
  28         o.commit_ancestors.
  29           select { |ca| ca.ancestor == want_ancestor }.
  30           select(&:is).
  31           first
  32       end
  33       # Now it is safe to do an .includes().where() because we are no
  34       # longer interested in jobs that have other ancestors but not
  35       # want_ancestor.
  36       @objects = @objects.
  37         includes(:commit_ancestors).
  38         where('commit_ancestors.ancestor = ? and commit_ancestors.is = ?',
  39               want_ancestor, true)
  40     end
  41     super
  42   end
  43
  44   def cancel
  45     @object.update_attributes cancelled_at: Time.now
  46     show
  47   end
  48
  49   class LogStreamer
  50     Q_UPDATE_INTERVAL = 12
  51     def initialize(job, opts={})
  52       @job = job
  53       @opts = opts
  54     end
  55     def each
  56       if @job.finished_at
  57         yield "#{@job.uuid} finished at #{@job.finished_at}\n"
  58         return
  59       end
  60       while not @job.started_at
  61         # send a summary (job queue + available nodes) to the client
  62         # every few seconds while waiting for the job to start
  63         last_ack_at ||= Time.now - Q_UPDATE_INTERVAL - 1
  64         if Time.now - last_ack_at >= Q_UPDATE_INTERVAL
  65           nodes_in_state = {idle: 0, alloc: 0}
  66           ActiveRecord::Base.uncached do
  67             Node.where('hostname is not ?', nil).collect do |n|
  68               if n.info[:slurm_state]
  69                 nodes_in_state[n.info[:slurm_state]] ||= 0
  70                 nodes_in_state[n.info[:slurm_state]] += 1
  71               end
  72             end
  73           end
  74           job_queue = Job.queue
  75           n_queued_before_me = 0
  76           job_queue.each do |j|
  77             break if j.uuid == @job.uuid
  78             n_queued_before_me += 1
  79           end
  80           yield "#{Time.now}" \
  81             " job #{@job.uuid}" \
  82             " queue_position #{n_queued_before_me}" \
  83             " queue_size #{job_queue.size}" \
  84             " nodes_idle #{nodes_in_state[:idle]}" \
  85             " nodes_alloc #{nodes_in_state[:alloc]}\n"
  86           last_ack_at = Time.now
  87         end
  88         sleep 3
  89         ActiveRecord::Base.uncached do
  90           @job.reload
  91         end
  92       end
  93       @redis = Redis.new(:timeout => 0)
  94       if @redis.exists @job.uuid
  95         # A log buffer exists. Start by showing the last few KB.
  96         @redis.
  97           getrange(@job.uuid, 0 - [@opts[:buffer_size], 1].max, -1).
  98           sub(/^[^\n]*\n?/, '').
  99           split("\n").
 100           each do |line|
 101           yield "#{line}\n"
 102         end
 103       end
 104       # TODO: avoid missing log entries between getrange() above and
 105       # subscribe() below.
 106       @redis.subscribe(@job.uuid) do |event|
 107         event.message do |channel, msg|
 108           if msg == "end"
 109             @redis.unsubscribe @job.uuid
 110           else
 111             yield "#{msg}\n"
 112           end
 113         end
 114       end
 115     end
 116   end
 117
 118   def self._log_tail_follow_requires_parameters
 119     {
 120       buffer_size: {type: 'integer', required: false, default: 2**13}
 121     }
 122   end
 123   def log_tail_follow
 124     if !@object.andand.uuid
 125       return render_not_found
 126     end
 127     if client_accepts_plain_text_stream
 128       self.response.headers['Last-Modified'] = Time.now.ctime.to_s
 129       self.response_body = LogStreamer.new @object, {
 130         buffer_size: (params[:buffer_size].to_i rescue 2**13)
 131       }
 132     else
 133       render json: {
 134         href: url_for(uuid: @object.uuid),
 135         comment: ('To retrieve the log stream as plain text, ' +
 136                   'use a request header like "Accept: text/plain"')
 137       }
 138     end
 139   end
 140
 141   def queue
 142     load_where_param
 143     @where.merge!({
 144                     started_at: nil,
 145                     is_locked_by_uuid: nil,
 146                     cancelled_at: nil
 147                   })
 148     params[:order] ||= 'priority desc, created_at'
 149     find_objects_for_index
 150     index
 151   end
 152
 153   def self._queue_requires_parameters
 154     self._index_requires_parameters
 155   end
 156 end