services/api/app/controllers/arvados/v1/jobs_controller.rb

   1 class Arvados::V1::JobsController < ApplicationController
   2   accept_attribute_as_json :script_parameters, Hash
   3   accept_attribute_as_json :runtime_constraints, Hash
   4   accept_attribute_as_json :tasks_summary, Hash
   5   skip_before_filter :find_object_by_uuid, :only => :queue
   6   skip_before_filter :render_404_if_no_object, :only => :queue
   7
   8   def index
   9     return super unless @where.is_a? Hash
  10     want_ancestor = @where[:script_version_descends_from]
  11     if want_ancestor
  12       # Check for missing commit_ancestor rows, and create them if
  13       # possible.
  14       @objects.
  15         dup.
  16         includes(:commit_ancestors). # I wish Rails would let me
  17                                      # specify here which
  18                                      # commit_ancestors I am
  19                                      # interested in.
  20         each do |o|
  21         if o.commit_ancestors.
  22             select { |ca| ca.ancestor == want_ancestor }.
  23             empty? and !o.script_version.nil?
  24           begin
  25             o.commit_ancestors << CommitAncestor.find_or_create_by_descendant_and_ancestor(o.script_version, want_ancestor)
  26           rescue
  27           end
  28         end
  29         o.commit_ancestors.
  30           select { |ca| ca.ancestor == want_ancestor }.
  31           select(&:is).
  32           first
  33       end
  34       # Now it is safe to do an .includes().where() because we are no
  35       # longer interested in jobs that have other ancestors but not
  36       # want_ancestor.
  37       @objects = @objects.
  38         includes(:commit_ancestors).
  39         where('commit_ancestors.ancestor = ? and commit_ancestors.is = ?',
  40               want_ancestor, true)
  41     end
  42     super
  43   end
  44
  45   def cancel
  46     reload_object_before_update
  47     @object.update_attributes cancelled_at: Time.now
  48     show
  49   end
  50
  51   class LogStreamer
  52     Q_UPDATE_INTERVAL = 12
  53     def initialize(job, opts={})
  54       @job = job
  55       @opts = opts
  56     end
  57     def each
  58       if @job.finished_at
  59         yield "#{@job.uuid} finished at #{@job.finished_at}\n"
  60         return
  61       end
  62       while not @job.started_at
  63         # send a summary (job queue + available nodes) to the client
  64         # every few seconds while waiting for the job to start
  65         last_ack_at ||= Time.now - Q_UPDATE_INTERVAL - 1
  66         if Time.now - last_ack_at >= Q_UPDATE_INTERVAL
  67           nodes_in_state = {idle: 0, alloc: 0}
  68           ActiveRecord::Base.uncached do
  69             Node.where('hostname is not ?', nil).collect do |n|
  70               if n.info[:slurm_state]
  71                 nodes_in_state[n.info[:slurm_state]] ||= 0
  72                 nodes_in_state[n.info[:slurm_state]] += 1
  73               end
  74             end
  75           end
  76           job_queue = Job.queue
  77           n_queued_before_me = 0
  78           job_queue.each do |j|
  79             break if j.uuid == @job.uuid
  80             n_queued_before_me += 1
  81           end
  82           yield "#{Time.now}" \
  83             " job #{@job.uuid}" \
  84             " queue_position #{n_queued_before_me}" \
  85             " queue_size #{job_queue.size}" \
  86             " nodes_idle #{nodes_in_state[:idle]}" \
  87             " nodes_alloc #{nodes_in_state[:alloc]}\n"
  88           last_ack_at = Time.now
  89         end
  90         sleep 3
  91         ActiveRecord::Base.uncached do
  92           @job.reload
  93         end
  94       end
  95       @redis = Redis.new(:timeout => 0)
  96       if @redis.exists @job.uuid
  97         # A log buffer exists. Start by showing the last few KB.
  98         @redis.
  99           getrange(@job.uuid, 0 - [@opts[:buffer_size], 1].max, -1).
 100           sub(/^[^\n]*\n?/, '').
 101           split("\n").
 102           each do |line|
 103           yield "#{line}\n"
 104         end
 105       end
 106       # TODO: avoid missing log entries between getrange() above and
 107       # subscribe() below.
 108       @redis.subscribe(@job.uuid) do |event|
 109         event.message do |channel, msg|
 110           if msg == "end"
 111             @redis.unsubscribe @job.uuid
 112           else
 113             yield "#{msg}\n"
 114           end
 115         end
 116       end
 117     end
 118   end
 119
 120   def self._log_tail_follow_requires_parameters
 121     {
 122       buffer_size: {type: 'integer', required: false, default: 2**13}
 123     }
 124   end
 125   def log_tail_follow
 126     if !@object.andand.uuid
 127       return render_not_found
 128     end
 129     if client_accepts_plain_text_stream
 130       self.response.headers['Last-Modified'] = Time.now.ctime.to_s
 131       self.response_body = LogStreamer.new @object, {
 132         buffer_size: (params[:buffer_size].to_i rescue 2**13)
 133       }
 134     else
 135       render json: {
 136         href: url_for(uuid: @object.uuid),
 137         comment: ('To retrieve the log stream as plain text, ' +
 138                   'use a request header like "Accept: text/plain"')
 139       }
 140     end
 141   end
 142
 143   def queue
 144     load_where_param
 145     @where.merge!({
 146                     started_at: nil,
 147                     is_locked_by_uuid: nil,
 148                     cancelled_at: nil
 149                   })
 150     params[:order] ||= 'priority desc, created_at'
 151     find_objects_for_index
 152     index
 153   end
 154
 155   def self._queue_requires_parameters
 156     self._index_requires_parameters
 157   end
 158 end