services/api/app/controllers/arvados/v1/jobs_controller.rb

   1 class Arvados::V1::JobsController < ApplicationController
   2   accept_attribute_as_json :script_parameters, Hash
   3   accept_attribute_as_json :runtime_constraints, Hash
   4   accept_attribute_as_json :tasks_summary, Hash
   5   skip_before_filter :find_object_by_uuid, :only => :queue
   6
   7   def index
   8     want_ancestor = @where[:script_version_descends_from]
   9     if want_ancestor
  10       # Check for missing commit_ancestor rows, and create them if
  11       # possible.
  12       @objects.
  13         dup.
  14         includes(:commit_ancestors). # I wish Rails would let me
  15                                      # specify here which
  16                                      # commit_ancestors I am
  17                                      # interested in.
  18         each do |o|
  19         if o.commit_ancestors.
  20             select { |ca| ca.ancestor == want_ancestor }.
  21             empty? and !o.script_version.nil?
  22           begin
  23             o.commit_ancestors << CommitAncestor.find_or_create_by_descendant_and_ancestor(o.script_version, want_ancestor)
  24           rescue
  25           end
  26         end
  27         o.commit_ancestors.
  28           select { |ca| ca.ancestor == want_ancestor }.
  29           select(&:is).
  30           first
  31       end
  32       # Now it is safe to do an .includes().where() because we are no
  33       # longer interested in jobs that have other ancestors but not
  34       # want_ancestor.
  35       @objects = @objects.
  36         includes(:commit_ancestors).
  37         where('commit_ancestors.ancestor = ? and commit_ancestors.is = ?',
  38               want_ancestor, true)
  39     end
  40     super
  41   end
  42
  43   class LogStreamer
  44     Q_UPDATE_INTERVAL = 12
  45     def initialize(job, opts={})
  46       @job = job
  47       @opts = opts
  48     end
  49     def each
  50       if @job.finished_at
  51         yield "#{@job.uuid} finished at #{@job.finished_at}\n"
  52         return
  53       end
  54       while not @job.started_at
  55         # send a summary (job queue + available nodes) to the client
  56         # every few seconds while waiting for the job to start
  57         last_ack_at ||= Time.now - Q_UPDATE_INTERVAL - 1
  58         if Time.now - last_ack_at >= Q_UPDATE_INTERVAL
  59           nodes_in_state = {idle: 0, alloc: 0}
  60           ActiveRecord::Base.uncached do
  61             Node.where('hostname is not ?', nil).collect do |n|
  62               if n.info[:slurm_state]
  63                 nodes_in_state[n.info[:slurm_state]] ||= 0
  64                 nodes_in_state[n.info[:slurm_state]] += 1
  65               end
  66             end
  67           end
  68           job_queue = Job.queue
  69           n_queued_before_me = 0
  70           job_queue.each do |j|
  71             break if j.uuid == @job.uuid
  72             n_queued_before_me += 1
  73           end
  74           yield "#{Time.now}" \
  75             " job #{@job.uuid}" \
  76             " queue_position #{n_queued_before_me}" \
  77             " queue_size #{job_queue.size}" \
  78             " nodes_idle #{nodes_in_state[:idle]}" \
  79             " nodes_alloc #{nodes_in_state[:alloc]}\n"
  80           last_ack_at = Time.now
  81         end
  82         sleep 3
  83         ActiveRecord::Base.uncached do
  84           @job.reload
  85         end
  86       end
  87       @redis = Redis.new(:timeout => 0)
  88       if @redis.exists @job.uuid
  89         # A log buffer exists. Start by showing the last few KB.
  90         @redis.
  91           getrange(@job.uuid, 0 - [@opts[:buffer_size], 1].max, -1).
  92           sub(/^[^\n]*\n?/, '').
  93           split("\n").
  94           each do |line|
  95           yield "#{line}\n"
  96         end
  97       end
  98       # TODO: avoid missing log entries between getrange() above and
  99       # subscribe() below.
 100       @redis.subscribe(@job.uuid) do |event|
 101         event.message do |channel, msg|
 102           if msg == "end"
 103             @redis.unsubscribe @job.uuid
 104           else
 105             yield "#{msg}\n"
 106           end
 107         end
 108       end
 109     end
 110   end
 111
 112   def self._log_tail_follow_requires_parameters
 113     {
 114       buffer_size: {type: 'integer', required: false}
 115     }
 116   end
 117   def log_tail_follow
 118     if !@object.andand.uuid
 119       return render_not_found
 120     end
 121     self.response.headers['Last-Modified'] = Time.now.ctime.to_s
 122     self.response_body = LogStreamer.new @object, {
 123       buffer_size: (params[:buffer_size] || 2**13)
 124     }
 125   end
 126
 127   def queue
 128     load_where_param
 129     @where.merge!({
 130                     started_at: nil,
 131                     is_locked_by_uuid: nil,
 132                     cancelled_at: nil
 133                   })
 134     params[:order] ||= 'priority desc, created_at'
 135     find_objects_for_index
 136     index
 137   end
 138
 139   def self._queue_requires_parameters
 140     self._index_requires_parameters
 141   end
 142 end