services/api/app/controllers/arvados/v1/jobs_controller.rb

   1 class Arvados::V1::JobsController < ApplicationController
   2   accept_attribute_as_json :script_parameters, Hash
   3   accept_attribute_as_json :runtime_constraints, Hash
   4   accept_attribute_as_json :tasks_summary, Hash
   5   skip_before_filter :find_object_by_uuid, :only => :queue
   6
   7   def index
   8     want_ancestor = @where[:script_version_descends_from]
   9     if want_ancestor
  10       # Check for missing commit_ancestor rows, and create them if
  11       # possible.
  12       @objects.
  13         dup.
  14         includes(:commit_ancestors). # I wish Rails would let me
  15                                      # specify here which
  16                                      # commit_ancestors I am
  17                                      # interested in.
  18         each do |o|
  19         if o.commit_ancestors.
  20             select { |ca| ca.ancestor == want_ancestor }.
  21             empty? and !o.script_version.nil?
  22           begin
  23             o.commit_ancestors << CommitAncestor.find_or_create_by_descendant_and_ancestor(o.script_version, want_ancestor)
  24           rescue
  25           end
  26         end
  27         o.commit_ancestors.
  28           select { |ca| ca.ancestor == want_ancestor }.
  29           select(&:is).
  30           first
  31       end
  32       # Now it is safe to do an .includes().where() because we are no
  33       # longer interested in jobs that have other ancestors but not
  34       # want_ancestor.
  35       @objects = @objects.
  36         includes(:commit_ancestors).
  37         where('commit_ancestors.ancestor = ? and commit_ancestors.is = ?',
  38               want_ancestor, true)
  39     end
  40     super
  41   end
  42
  43   class LogStreamer
  44     Q_UPDATE_INTERVAL = 12
  45     def initialize(job, opts={})
  46       @job = job
  47       @opts = opts
  48     end
  49     def each
  50       if @job.finished_at
  51         yield "#{@job.uuid} finished at #{@job.finished_at}\n"
  52         return
  53       end
  54       while not @job.started_at
  55         # send a summary (job queue + available nodes) to the client
  56         # every few seconds while waiting for the job to start
  57         last_ack_at ||= Time.now - Q_UPDATE_INTERVAL - 1
  58         if Time.now - last_ack_at >= Q_UPDATE_INTERVAL
  59           nodes_in_state = {idle: 0, alloc: 0}
  60           Node.where('hostname is not ?', nil).collect do |n|
  61             if n.info[:slurm_state]
  62               nodes_in_state[n.info[:slurm_state]] ||= 0
  63               nodes_in_state[n.info[:slurm_state]] += 1
  64             end
  65           end
  66           job_queue = Job.queue
  67           n_queued_before_me = 0
  68           job_queue.each do |j|
  69             break if j.uuid == @job.uuid
  70             n_queued_before_me += 1
  71           end
  72           yield "#{Time.now}" \
  73             " job #{@job.uuid}" \
  74             " queue_position #{n_queued_before_me}" \
  75             " queue_size #{job_queue.size}" \
  76             " nodes_idle #{nodes_in_state[:idle]}" \
  77             " nodes_alloc #{nodes_in_state[:alloc]}\n"
  78           last_ack_at = Time.now
  79         end
  80         sleep 3
  81         @job.reload
  82       end
  83       @redis = Redis.new(:timeout => 0)
  84       @redis.subscribe(@job.uuid) do |event|
  85         if @redis.exists @job.uuid
  86           # A log buffer exists. Start by showing the last few KB.
  87           @redis.
  88             getrange(@job.uuid, 0 - [@opts[:buffer_size], 1].max, -1).
  89             sub(/^[^\n]*\n?/, '').
  90             split("\n").
  91             each do |line|
  92             yield "#{line}\n"
  93           end
  94         end
  95         # TODO: avoid duplicating the last few lines of the log
  96         # file. Use the fact that timestamps are lexicographically
  97         # ordered.
  98         event.message do |channel, msg|
  99           if msg == "end"
 100             @redis.unsubscribe @job.uuid
 101           else
 102             yield "#{msg}\n"
 103           end
 104         end
 105       end
 106     end
 107   end
 108
 109   def self._log_tail_follow_requires_parameters
 110     {
 111       buffer_size: {type: 'integer', required: false}
 112     }
 113   end
 114   def log_tail_follow
 115     if !@object.andand.uuid
 116       return render_not_found
 117     end
 118     self.response.headers['Last-Modified'] = Time.now.ctime.to_s
 119     self.response_body = LogStreamer.new @object, {
 120       buffer_size: (params[:buffer_size] || 2**13)
 121     }
 122   end
 123
 124   def queue
 125     load_where_param
 126     @where.merge!({
 127                     started_at: nil,
 128                     is_locked_by_uuid: nil,
 129                     cancelled_at: nil
 130                   })
 131     params[:order] ||= 'priority desc, created_at'
 132     find_objects_for_index
 133     index
 134   end
 135
 136   def self._queue_requires_parameters
 137     self._index_requires_parameters
 138   end
 139 end