services/api/app/models/job.rb

   1 class Job < ArvadosModel
   2   include HasUuid
   3   include KindAndEtag
   4   include CommonApiTemplate
   5   attr_protected :docker_image_locator
   6   serialize :script_parameters, Hash
   7   serialize :runtime_constraints, Hash
   8   serialize :tasks_summary, Hash
   9   before_create :ensure_unique_submit_id
  10   after_commit :trigger_crunch_dispatch_if_cancelled, :on => :update
  11   before_validation :set_priority
  12   before_validation :update_state_from_old_state_attrs
  13   validate :ensure_script_version_is_commit
  14   validate :find_docker_image_locator
  15   validate :validate_status
  16   validate :validate_state_change
  17   before_save :update_timestamps_when_state_changes
  18
  19   has_many :commit_ancestors, :foreign_key => :descendant, :primary_key => :script_version
  20   has_many(:nodes, foreign_key: :job_uuid, primary_key: :uuid)
  21
  22   class SubmitIdReused < StandardError
  23   end
  24
  25   api_accessible :user, extend: :common do |t|
  26     t.add :submit_id
  27     t.add :priority
  28     t.add :script
  29     t.add :script_parameters
  30     t.add :script_version
  31     t.add :cancelled_at
  32     t.add :cancelled_by_client_uuid
  33     t.add :cancelled_by_user_uuid
  34     t.add :started_at
  35     t.add :finished_at
  36     t.add :output
  37     t.add :success
  38     t.add :running
  39     t.add :state
  40     t.add :is_locked_by_uuid
  41     t.add :log
  42     t.add :runtime_constraints
  43     t.add :tasks_summary
  44     t.add :dependencies
  45     t.add :nondeterministic
  46     t.add :repository
  47     t.add :supplied_script_version
  48     t.add :docker_image_locator
  49     t.add :queue_position
  50     t.add :node_uuids
  51     t.add :description
  52   end
  53
  54   # Supported states for a job
  55   States = [
  56             (Queued = 'Queued'),
  57             (Running = 'Running'),
  58             (Cancelled = 'Cancelled'),
  59             (Failed = 'Failed'),
  60             (Complete = 'Complete'),
  61            ]
  62
  63   def assert_finished
  64     update_attributes(finished_at: finished_at || Time.now,
  65                       success: success.nil? ? false : success,
  66                       running: false)
  67   end
  68
  69   def node_uuids
  70     nodes.map(&:uuid)
  71   end
  72
  73   def self.queue
  74     self.where('state = ?', Queued).order('priority desc, created_at')
  75   end
  76
  77   def queue_position
  78     Job::queue.each_with_index do |job, index|
  79       if job[:uuid] == self.uuid
  80         return index
  81       end
  82     end
  83     nil
  84   end
  85
  86   def self.running
  87     self.where('running = ?', true).
  88       order('priority desc, created_at')
  89   end
  90
  91   def lock locked_by_uuid
  92     transaction do
  93       self.reload
  94       unless self.state == Queued and self.is_locked_by_uuid.nil?
  95         raise AlreadyLockedError
  96       end
  97       self.state = Running
  98       self.is_locked_by_uuid = locked_by_uuid
  99       self.save!
 100     end
 101   end
 102
 103   protected
 104
 105   def foreign_key_attributes
 106     super + %w(output log)
 107   end
 108
 109   def skip_uuid_read_permission_check
 110     super + %w(cancelled_by_client_uuid)
 111   end
 112
 113   def skip_uuid_existence_check
 114     super + %w(output log)
 115   end
 116
 117   def set_priority
 118     if self.priority.nil?
 119       self.priority = 0
 120     end
 121     true
 122   end
 123
 124   def ensure_script_version_is_commit
 125     if self.state == Running
 126       # Apparently client has already decided to go for it. This is
 127       # needed to run a local job using a local working directory
 128       # instead of a commit-ish.
 129       return true
 130     end
 131     if new_record? or script_version_changed?
 132       sha1 = Commit.find_commit_range(current_user, self.repository, nil, self.script_version, nil)[0] rescue nil
 133       if sha1
 134         self.supplied_script_version = self.script_version if self.supplied_script_version.nil? or self.supplied_script_version.empty?
 135         self.script_version = sha1
 136       else
 137         self.errors.add :script_version, "#{self.script_version} does not resolve to a commit"
 138         return false
 139       end
 140     end
 141   end
 142
 143   def ensure_unique_submit_id
 144     if !submit_id.nil?
 145       if Job.where('submit_id=?',self.submit_id).first
 146         raise SubmitIdReused.new
 147       end
 148     end
 149     true
 150   end
 151
 152   def find_docker_image_locator
 153     # Find the Collection that holds the Docker image specified in the
 154     # runtime constraints, and store its locator in docker_image_locator.
 155     unless runtime_constraints.is_a? Hash
 156       # We're still in validation stage, so we can't assume
 157       # runtime_constraints isn't something horrible like an array or
 158       # a string. Treat those cases as "no docker image supplied";
 159       # other validations will fail anyway.
 160       self.docker_image_locator = nil
 161       return true
 162     end
 163     image_search = runtime_constraints['docker_image']
 164     image_tag = runtime_constraints['docker_image_tag']
 165     if image_search.nil?
 166       self.docker_image_locator = nil
 167       true
 168     elsif coll = Collection.for_latest_docker_image(image_search, image_tag)
 169       self.docker_image_locator = coll.portable_data_hash
 170       true
 171     else
 172       errors.add(:docker_image_locator, "not found for #{image_search}")
 173       false
 174     end
 175   end
 176
 177   def dependencies
 178     deps = {}
 179     queue = self.script_parameters.values
 180     while not queue.empty?
 181       queue = queue.flatten.compact.collect do |v|
 182         if v.is_a? Hash
 183           v.values
 184         elsif v.is_a? String
 185           v.match(/^(([0-9a-f]{32})\b(\+[^,]+)?,?)*$/) do |locator|
 186             deps[locator.to_s] = true
 187           end
 188           nil
 189         end
 190       end
 191     end
 192     deps.keys
 193   end
 194
 195   def permission_to_update
 196     if is_locked_by_uuid_was and !(current_user and
 197                                    (current_user.uuid == is_locked_by_uuid_was or
 198                                     current_user.uuid == system_user.uuid))
 199       if script_changed? or
 200           script_parameters_changed? or
 201           script_version_changed? or
 202           (!cancelled_at_was.nil? and
 203            (cancelled_by_client_uuid_changed? or
 204             cancelled_by_user_uuid_changed? or
 205             cancelled_at_changed?)) or
 206           started_at_changed? or
 207           finished_at_changed? or
 208           running_changed? or
 209           success_changed? or
 210           output_changed? or
 211           log_changed? or
 212           tasks_summary_changed? or
 213           state_changed?
 214         logger.warn "User #{current_user.uuid if current_user} tried to change protected job attributes on locked #{self.class.to_s} #{uuid_was}"
 215         return false
 216       end
 217     end
 218     if !is_locked_by_uuid_changed?
 219       super
 220     else
 221       if !current_user
 222         logger.warn "Anonymous user tried to change lock on #{self.class.to_s} #{uuid_was}"
 223         false
 224       elsif is_locked_by_uuid_was and is_locked_by_uuid_was != current_user.uuid
 225         logger.warn "User #{current_user.uuid} tried to steal lock on #{self.class.to_s} #{uuid_was} from #{is_locked_by_uuid_was}"
 226         false
 227       elsif !is_locked_by_uuid.nil? and is_locked_by_uuid != current_user.uuid
 228         logger.warn "User #{current_user.uuid} tried to lock #{self.class.to_s} #{uuid_was} with uuid #{is_locked_by_uuid}"
 229         false
 230       else
 231         super
 232       end
 233     end
 234   end
 235
 236   def update_modified_by_fields
 237     if self.cancelled_at_changed?
 238       # Ensure cancelled_at cannot be set to arbitrary non-now times,
 239       # or changed once it is set.
 240       if self.cancelled_at and not self.cancelled_at_was
 241         self.cancelled_at = Time.now
 242         self.cancelled_by_user_uuid = current_user.uuid
 243         self.cancelled_by_client_uuid = current_api_client.andand.uuid
 244         @need_crunch_dispatch_trigger = true
 245       else
 246         self.cancelled_at = self.cancelled_at_was
 247         self.cancelled_by_user_uuid = self.cancelled_by_user_uuid_was
 248         self.cancelled_by_client_uuid = self.cancelled_by_client_uuid_was
 249       end
 250     end
 251     super
 252   end
 253
 254   def trigger_crunch_dispatch_if_cancelled
 255     if @need_crunch_dispatch_trigger
 256       File.open(Rails.configuration.crunch_refresh_trigger, 'wb') do
 257         # That's all, just create/touch a file for crunch-job to see.
 258       end
 259     end
 260   end
 261
 262   def update_timestamps_when_state_changes
 263     return if not (state_changed? or new_record?)
 264
 265     case state
 266     when Running
 267       self.started_at ||= Time.now
 268     when Failed, Complete
 269       self.finished_at ||= Time.now
 270     when Cancelled
 271       self.cancelled_at ||= Time.now
 272     end
 273
 274     # TODO: Remove the following case block when old "success" and
 275     # "running" attrs go away. Until then, this ensures we still
 276     # expose correct success/running flags to older clients, even if
 277     # some new clients are writing only the new state attribute.
 278     case state
 279     when Queued
 280       self.running = false
 281       self.success = nil
 282     when Running
 283       self.running = true
 284       self.success = nil
 285     when Cancelled, Failed
 286       self.running = false
 287       self.success = false
 288     when Complete
 289       self.running = false
 290       self.success = true
 291     end
 292     self.running ||= false # Default to false instead of nil.
 293
 294     true
 295   end
 296
 297   def update_state_from_old_state_attrs
 298     # If a client has touched the legacy state attrs, update the
 299     # "state" attr to agree with the updated values of the legacy
 300     # attrs.
 301     #
 302     # TODO: Remove this method when old "success" and "running" attrs
 303     # go away.
 304     if cancelled_at_changed? or
 305         success_changed? or
 306         running_changed? or
 307         state.nil?
 308       if cancelled_at
 309         self.state = Cancelled
 310       elsif success == false
 311         self.state = Failed
 312       elsif success == true
 313         self.state = Complete
 314       elsif running == true
 315         self.state = Running
 316       else
 317         self.state = Queued
 318       end
 319     end
 320     true
 321   end
 322
 323   def validate_status
 324     if self.state.in?(States)
 325       true
 326     else
 327       errors.add :state, "#{state.inspect} must be one of: #{States.inspect}"
 328       false
 329     end
 330   end
 331
 332   def validate_state_change
 333     ok = true
 334     if self.state_changed?
 335       ok = case self.state_was
 336            when nil
 337              # state isn't set yet
 338              true
 339            when Queued
 340              # Permit going from queued to any state
 341              true
 342            when Running
 343              # From running, may only transition to a finished state
 344              [Complete, Failed, Cancelled].include? self.state
 345            when Complete, Failed, Cancelled
 346              # Once in a finished state, don't permit any more state changes
 347              false
 348            else
 349              # Any other state transition is also invalid
 350              false
 351            end
 352       if not ok
 353         errors.add :state, "invalid change from #{self.state_was} to #{self.state}"
 354       end
 355     end
 356     ok
 357   end
 358 end