services/api/app/controllers/arvados/v1/jobs_controller.rb

   1 class Arvados::V1::JobsController < ApplicationController
   2   accept_attribute_as_json :script_parameters, Hash
   3   accept_attribute_as_json :runtime_constraints, Hash
   4   accept_attribute_as_json :tasks_summary, Hash
   5   skip_before_filter :find_object_by_uuid, :only => :queue
   6
   7   def index
   8     want_ancestor = @where[:script_version_descends_from]
   9     if want_ancestor
  10       # Check for missing commit_ancestor rows, and create them if
  11       # possible.
  12       @objects.
  13         dup.
  14         includes(:commit_ancestors). # I wish Rails would let me
  15                                      # specify here which
  16                                      # commit_ancestors I am
  17                                      # interested in.
  18         each do |o|
  19         if o.commit_ancestors.
  20             select { |ca| ca.ancestor == want_ancestor }.
  21             empty? and !o.script_version.nil?
  22           begin
  23             o.commit_ancestors << CommitAncestor.find_or_create_by_descendant_and_ancestor(o.script_version, want_ancestor)
  24           rescue
  25           end
  26         end
  27         o.commit_ancestors.
  28           select { |ca| ca.ancestor == want_ancestor }.
  29           select(&:is).
  30           first
  31       end
  32       # Now it is safe to do an .includes().where() because we are no
  33       # longer interested in jobs that have other ancestors but not
  34       # want_ancestor.
  35       @objects = @objects.
  36         includes(:commit_ancestors).
  37         where('commit_ancestors.ancestor = ? and commit_ancestors.is = ?',
  38               want_ancestor, true)
  39     end
  40     super
  41   end
  42
  43   class LogStreamer
  44     Q_UPDATE_INTERVAL = 12
  45     def initialize(job)
  46       @job = job
  47     end
  48     def each
  49       if @job.finished_at
  50         yield "#{@job.uuid} finished at #{@job.finished_at}\n"
  51         return
  52       end
  53       while not @job.started_at
  54         # send a summary (job queue + available nodes) to the client
  55         # every few seconds while waiting for the job to start
  56         last_ack_at ||= Time.now - Q_UPDATE_INTERVAL - 1
  57         if Time.now - last_ack_at >= Q_UPDATE_INTERVAL
  58           nodes_in_state = {idle: 0, alloc: 0}
  59           Node.where('hostname is not ?', nil).collect do |n|
  60             if n.info[:slurm_state]
  61               nodes_in_state[n.info[:slurm_state]] ||= 0
  62               nodes_in_state[n.info[:slurm_state]] += 1
  63             end
  64           end
  65           job_queue = Job.queue
  66           n_queued_before_me = 0
  67           job_queue.each do |j|
  68             break if j.uuid == @job.uuid
  69             n_queued_before_me += 1
  70           end
  71           yield "#{Time.now}" \
  72             " job #{@job.uuid}" \
  73             " queue_position #{n_queued_before_me}" \
  74             " queue_size #{job_queue.size}" \
  75             " nodes_idle #{nodes_in_state[:idle]}" \
  76             " nodes_alloc #{nodes_in_state[:alloc]}\n"
  77           last_ack_at = Time.now
  78         end
  79         sleep 3
  80         @job.reload
  81       end
  82       @redis = Redis.new(:timeout => 0)
  83       @redis.subscribe(@job.uuid) do |event|
  84         event.message do |channel, msg|
  85           if msg == "end"
  86             @redis.unsubscribe @job.uuid
  87           else
  88             yield "#{msg}\n"
  89           end
  90         end
  91       end
  92     end
  93   end
  94
  95   def log_tail_follow
  96     if !@object.andand.uuid
  97       return render_not_found
  98     end
  99     self.response.headers['Last-Modified'] = Time.now.ctime.to_s
 100     self.response_body = LogStreamer.new @object
 101   end
 102
 103   def queue
 104     load_where_param
 105     @where.merge!({
 106                     started_at: nil,
 107                     is_locked_by_uuid: nil,
 108                     cancelled_at: nil
 109                   })
 110     params[:order] ||= 'priority desc, created_at'
 111     find_objects_for_index
 112     index
 113   end
 114
 115   def self._queue_requires_parameters
 116     self._index_requires_parameters
 117   end
 118 end