1 require 'log_reuse_info'
4 class Job < ArvadosModel
7 include CommonApiTemplate
8 extend CurrentApiClient
10 serialize :components, Hash
11 attr_protected :arvados_sdk_version, :docker_image_locator
12 serialize :script_parameters, Hash
13 serialize :runtime_constraints, Hash
14 serialize :tasks_summary, Hash
15 before_create :ensure_unique_submit_id
16 after_commit :trigger_crunch_dispatch_if_cancelled, :on => :update
17 before_validation :set_priority
18 before_validation :update_state_from_old_state_attrs
19 before_validation :update_script_parameters_digest
20 validate :ensure_script_version_is_commit
21 validate :find_docker_image_locator
22 validate :find_arvados_sdk_version
23 validate :validate_status
24 validate :validate_state_change
25 validate :ensure_no_collection_uuids_in_script_params
26 before_save :tag_version_in_internal_repository
27 before_save :update_timestamps_when_state_changes
29 has_many :commit_ancestors, :foreign_key => :descendant, :primary_key => :script_version
30 has_many(:nodes, foreign_key: :job_uuid, primary_key: :uuid)
32 class SubmitIdReused < StandardError
35 api_accessible :user, extend: :common do |t|
39 t.add :script_parameters
42 t.add :cancelled_by_client_uuid
43 t.add :cancelled_by_user_uuid
50 t.add :is_locked_by_uuid
52 t.add :runtime_constraints
54 t.add :nondeterministic
56 t.add :supplied_script_version
57 t.add :arvados_sdk_version
58 t.add :docker_image_locator
65 # Supported states for a job
68 (Running = 'Running'),
69 (Cancelled = 'Cancelled'),
71 (Complete = 'Complete'),
75 @need_crunch_dispatch_trigger = false
78 def self.limit_index_columns_read
83 update_attributes(finished_at: finished_at || db_current_time,
84 success: success.nil? ? false : success,
93 self.where('state = ?', Queued).order('priority desc, created_at')
97 # We used to report this accurately, but the implementation made queue
98 # API requests O(n**2) for the size of the queue. See #8800.
99 # We've soft-disabled it because it's not clear we even want this
100 # functionality: now that we have Node Manager with support for multiple
101 # node sizes, "queue position" tells you very little about when a job will
103 state == Queued ? 0 : nil
107 self.where('running = ?', true).
108 order('priority desc, created_at')
111 def lock locked_by_uuid
113 unless self.state == Queued and self.is_locked_by_uuid.nil?
114 raise AlreadyLockedError
117 self.is_locked_by_uuid = locked_by_uuid
122 def update_script_parameters_digest
123 self.script_parameters_digest = self.class.sorted_hash_digest(script_parameters)
126 def self.searchable_columns operator
127 super - ["script_parameters_digest"]
130 def self.full_text_searchable_columns
131 super - ["script_parameters_digest"]
134 def self.load_job_specific_filters attrs, orig_filters, read_users
135 # Convert Job-specific @filters entries into general SQL filters.
136 script_info = {"repository" => nil, "script" => nil}
137 git_filters = Hash.new do |hash, key|
138 hash[key] = {"max_version" => "HEAD", "exclude_versions" => []}
141 orig_filters.each do |attr, operator, operand|
142 if (script_info.has_key? attr) and (operator == "=")
143 if script_info[attr].nil?
144 script_info[attr] = operand
145 elsif script_info[attr] != operand
146 raise ArgumentError.new("incompatible #{attr} filters")
151 git_filters[attr]["min_version"] = operand
153 git_filters[attr]["exclude_versions"] += Array.wrap(operand)
154 when "in docker", "not in docker"
155 image_hashes = Array.wrap(operand).flat_map do |search_term|
156 image_search, image_tag = search_term.split(':', 2)
158 find_all_for_docker_image(image_search, image_tag, read_users, filter_compatible_format: false).
159 map(&:portable_data_hash)
161 filters << [attr, operator.sub(/ docker$/, ""), image_hashes]
163 filters << [attr, operator, operand]
167 # Build a real script_version filter from any "not? in git" filters.
168 git_filters.each_pair do |attr, filter|
170 when "script_version"
171 script_info.each_pair do |key, value|
173 raise ArgumentError.new("script_version filter needs #{key} filter")
176 filter["repository"] = script_info["repository"]
177 if attrs[:script_version]
178 filter["max_version"] = attrs[:script_version]
180 # Using HEAD, set earlier by the hash default, is fine.
182 when "arvados_sdk_version"
183 filter["repository"] = "arvados"
185 raise ArgumentError.new("unknown attribute for git filter: #{attr}")
187 revisions = Commit.find_commit_range(filter["repository"],
188 filter["min_version"],
189 filter["max_version"],
190 filter["exclude_versions"])
193 new("error searching #{filter['repository']} from " +
194 "'#{filter['min_version']}' to '#{filter['max_version']}', " +
195 "excluding #{filter['exclude_versions']}")
197 filters.append([attr, "in", revisions])
203 def self.find_reusable attrs, params, filters, read_users
204 if filters.empty? # Translate older creation parameters into filters.
206 [["repository", "=", attrs[:repository]],
207 ["script", "=", attrs[:script]],
208 ["script_version", "not in git", params[:exclude_script_versions]],
209 ].reject { |filter| filter.last.nil? or filter.last.empty? }
210 if !params[:minimum_script_version].blank?
211 filters << ["script_version", "in git",
212 params[:minimum_script_version]]
214 filters += default_git_filters("script_version", attrs[:repository],
215 attrs[:script_version])
217 if image_search = attrs[:runtime_constraints].andand["docker_image"]
218 if image_tag = attrs[:runtime_constraints]["docker_image_tag"]
219 image_search += ":#{image_tag}"
221 image_locator = Collection.
222 for_latest_docker_image(image_search).andand.portable_data_hash
226 filters << ["docker_image_locator", "=", image_locator]
227 if sdk_version = attrs[:runtime_constraints].andand["arvados_sdk_version"]
228 filters += default_git_filters("arvados_sdk_version", "arvados", sdk_version)
230 filters = load_job_specific_filters(attrs, filters, read_users)
233 # Check specified filters for some reasonableness.
234 filter_names = filters.map { |f| f.first }.uniq
235 ["repository", "script"].each do |req_filter|
236 if not filter_names.include?(req_filter)
237 return send_error("#{req_filter} filter required")
241 # Search for a reusable Job, and return it if found.
242 candidates = Job.readable_by(current_user)
243 log_reuse_info { "starting with #{candidates.count} jobs readable by current user #{current_user.uuid}" }
245 candidates = candidates.where(
246 'state = ? or (owner_uuid = ? and state in (?))',
247 Job::Complete, current_user.uuid, [Job::Queued, Job::Running])
248 log_reuse_info(candidates) { "after filtering on job state ((state=Complete) or (state=Queued/Running and (submitted by current user)))" }
250 digest = Job.sorted_hash_digest(attrs[:script_parameters])
251 candidates = candidates.where('script_parameters_digest = ?', digest)
252 log_reuse_info(candidates) { "after filtering on script_parameters_digest #{digest}" }
254 candidates = candidates.where('nondeterministic is distinct from ?', true)
255 log_reuse_info(candidates) { "after filtering on !nondeterministic" }
257 # prefer Running jobs over Queued
258 candidates = candidates.order('state desc, created_at')
260 candidates = apply_filters candidates, filters
261 log_reuse_info(candidates) { "after filtering on repo, script, and custom filters #{filters.inspect}" }
265 candidates.each do |j|
266 if j.state != Job::Complete
268 # We'll use this if we don't find a job that has completed
269 log_reuse_info { "job #{j.uuid} is reusable, but unfinished; continuing search for completed jobs" }
272 log_reuse_info { "job #{j.uuid} is unfinished and we already have #{incomplete_job.uuid}; ignoring" }
274 elsif chosen == false
275 # Ignore: we have already decided not to reuse any completed
277 log_reuse_info { "job #{j.uuid} with output #{j.output} ignored, see above" }
278 elsif Rails.configuration.reuse_job_if_outputs_differ
279 if Collection.readable_by(current_user).find_by_portable_data_hash(j.output)
280 log_reuse_info { "job #{j.uuid} with output #{j.output} is reusable; decision is final." }
283 # Ignore: keep locking for an incomplete job or one whose
284 # output is readable.
285 log_reuse_info { "job #{j.uuid} output #{j.output} unavailable to user; continuing search" }
288 if chosen.output != j.output
289 # If two matching jobs produced different outputs, run a new
290 # job (or use one that's already running/queued) instead of
291 # choosing one arbitrarily.
292 log_reuse_info { "job #{j.uuid} output #{j.output} disagrees; forgetting about #{chosen.uuid} and ignoring any other finished jobs (see reuse_job_if_outputs_differ in application.default.yml)" }
295 log_reuse_info { "job #{j.uuid} output #{j.output} agrees with chosen #{chosen.uuid}; continuing search in case other candidates have different outputs" }
297 # ...and that's the only thing we need to do once we've chosen
299 elsif !Collection.readable_by(current_user).find_by_portable_data_hash(j.output)
300 # This user cannot read the output of this job. Any other
301 # completed job will have either the same output (making it
302 # unusable) or a different output (making it unusable because
303 # reuse_job_if_outputs_different is turned off). Therefore,
304 # any further investigation of reusable jobs is futile.
305 log_reuse_info { "job #{j.uuid} output #{j.output} is unavailable to user; this means no finished job can be reused (see reuse_job_if_outputs_differ in application.default.yml)" }
308 log_reuse_info { "job #{j.uuid} with output #{j.output} can be reused; continuing search in case other candidates have different outputs" }
312 j = chosen || incomplete_job
314 log_reuse_info { "done, #{j.uuid} was selected" }
316 log_reuse_info { "done, nothing suitable" }
321 def self.default_git_filters(attr_name, repo_name, refspec)
322 # Add a filter to @filters for `attr_name` = the latest commit available
323 # in `repo_name` at `refspec`. No filter is added if refspec can't be
325 commits = Commit.find_commit_range(repo_name, nil, refspec, nil)
326 if commit_hash = commits.first
327 [[attr_name, "=", commit_hash]]
333 def cancel(cascade: false, need_transaction: true)
335 ActiveRecord::Base.transaction do
336 cancel(cascade: cascade, need_transaction: false)
341 if self.state.in?([Queued, Running])
342 self.state = Cancelled
344 elsif self.state != Cancelled
345 raise InvalidStateTransitionError
350 # cancel all children; they could be jobs or pipeline instances
351 children = self.components.andand.collect{|_, u| u}.compact
353 return if children.empty?
355 # cancel any child jobs
356 Job.where(uuid: children, state: [Queued, Running]).each do |job|
357 job.cancel(cascade: cascade, need_transaction: false)
360 # cancel any child pipelines
361 PipelineInstance.where(uuid: children, state: [PipelineInstance::RunningOnServer, PipelineInstance::RunningOnClient]).each do |pi|
362 pi.cancel(cascade: cascade, need_transaction: false)
368 def self.sorted_hash_digest h
369 Digest::MD5.hexdigest(Oj.dump(deep_sort_hash(h)))
372 def foreign_key_attributes
373 super + %w(output log)
376 def skip_uuid_read_permission_check
377 super + %w(cancelled_by_client_uuid)
380 def skip_uuid_existence_check
381 super + %w(output log)
385 if self.priority.nil?
391 def ensure_script_version_is_commit
393 # Apparently client has already decided to go for it. This is
394 # needed to run a local job using a local working directory
395 # instead of a commit-ish.
398 if new_record? or repository_changed? or script_version_changed?
399 sha1 = Commit.find_commit_range(repository,
400 nil, script_version, nil).first
402 errors.add :script_version, "#{script_version} does not resolve to a commit"
405 if supplied_script_version.nil? or supplied_script_version.empty?
406 self.supplied_script_version = script_version
408 self.script_version = sha1
413 def tag_version_in_internal_repository
415 # No point now. See ensure_script_version_is_commit.
418 # Won't be saved, and script_version might not even be valid.
420 elsif new_record? or repository_changed? or script_version_changed?
424 Commit.tag_in_internal_repository repository, script_version, uuid
432 def ensure_unique_submit_id
434 if Job.where('submit_id=?',self.submit_id).first
435 raise SubmitIdReused.new
441 def resolve_runtime_constraint(key, attr_sym)
442 if ((runtime_constraints.is_a? Hash) and
443 (search = runtime_constraints[key]))
444 ok, result = yield search
446 ok, result = true, nil
449 send("#{attr_sym}=".to_sym, result)
451 errors.add(attr_sym, result)
456 def find_arvados_sdk_version
457 resolve_runtime_constraint("arvados_sdk_version",
458 :arvados_sdk_version) do |git_search|
459 commits = Commit.find_commit_range("arvados",
460 nil, git_search, nil)
462 [false, "#{git_search} does not resolve to a commit"]
463 elsif not runtime_constraints["docker_image"]
464 [false, "cannot be specified without a Docker image constraint"]
466 [true, commits.first]
471 def find_docker_image_locator
472 if runtime_constraints.is_a? Hash
473 runtime_constraints['docker_image'] ||=
474 Rails.configuration.default_docker_image_for_jobs
477 resolve_runtime_constraint("docker_image",
478 :docker_image_locator) do |image_search|
479 image_tag = runtime_constraints['docker_image_tag']
480 if coll = Collection.for_latest_docker_image(image_search, image_tag)
481 [true, coll.portable_data_hash]
483 [false, "not found for #{image_search}"]
488 def permission_to_update
489 if is_locked_by_uuid_was and !(current_user and
490 (current_user.uuid == is_locked_by_uuid_was or
491 current_user.uuid == system_user.uuid))
492 if script_changed? or
493 script_parameters_changed? or
494 script_version_changed? or
495 (!cancelled_at_was.nil? and
496 (cancelled_by_client_uuid_changed? or
497 cancelled_by_user_uuid_changed? or
498 cancelled_at_changed?)) or
499 started_at_changed? or
500 finished_at_changed? or
505 tasks_summary_changed? or
506 (state_changed? && state != Cancelled) or
508 logger.warn "User #{current_user.uuid if current_user} tried to change protected job attributes on locked #{self.class.to_s} #{uuid_was}"
512 if !is_locked_by_uuid_changed?
516 logger.warn "Anonymous user tried to change lock on #{self.class.to_s} #{uuid_was}"
518 elsif is_locked_by_uuid_was and is_locked_by_uuid_was != current_user.uuid
519 logger.warn "User #{current_user.uuid} tried to steal lock on #{self.class.to_s} #{uuid_was} from #{is_locked_by_uuid_was}"
521 elsif !is_locked_by_uuid.nil? and is_locked_by_uuid != current_user.uuid
522 logger.warn "User #{current_user.uuid} tried to lock #{self.class.to_s} #{uuid_was} with uuid #{is_locked_by_uuid}"
530 def update_modified_by_fields
531 if self.cancelled_at_changed?
532 # Ensure cancelled_at cannot be set to arbitrary non-now times,
533 # or changed once it is set.
534 if self.cancelled_at and not self.cancelled_at_was
535 self.cancelled_at = db_current_time
536 self.cancelled_by_user_uuid = current_user.uuid
537 self.cancelled_by_client_uuid = current_api_client.andand.uuid
538 @need_crunch_dispatch_trigger = true
540 self.cancelled_at = self.cancelled_at_was
541 self.cancelled_by_user_uuid = self.cancelled_by_user_uuid_was
542 self.cancelled_by_client_uuid = self.cancelled_by_client_uuid_was
548 def trigger_crunch_dispatch_if_cancelled
549 if @need_crunch_dispatch_trigger
550 File.open(Rails.configuration.crunch_refresh_trigger, 'wb') do
551 # That's all, just create/touch a file for crunch-job to see.
556 def update_timestamps_when_state_changes
557 return if not (state_changed? or new_record?)
561 self.started_at ||= db_current_time
562 when Failed, Complete
563 self.finished_at ||= db_current_time
565 self.cancelled_at ||= db_current_time
568 # TODO: Remove the following case block when old "success" and
569 # "running" attrs go away. Until then, this ensures we still
570 # expose correct success/running flags to older clients, even if
571 # some new clients are writing only the new state attribute.
579 when Cancelled, Failed
586 self.running ||= false # Default to false instead of nil.
588 @need_crunch_dispatch_trigger = true
593 def update_state_from_old_state_attrs
594 # If a client has touched the legacy state attrs, update the
595 # "state" attr to agree with the updated values of the legacy
598 # TODO: Remove this method when old "success" and "running" attrs
600 if cancelled_at_changed? or
605 self.state = Cancelled
606 elsif success == false
608 elsif success == true
609 self.state = Complete
610 elsif running == true
620 if self.state.in?(States)
623 errors.add :state, "#{state.inspect} must be one of: #{States.inspect}"
628 def validate_state_change
630 if self.state_changed?
631 ok = case self.state_was
633 # state isn't set yet
636 # Permit going from queued to any state
639 # From running, may only transition to a finished state
640 [Complete, Failed, Cancelled].include? self.state
641 when Complete, Failed, Cancelled
642 # Once in a finished state, don't permit any more state changes
645 # Any other state transition is also invalid
649 errors.add :state, "invalid change from #{self.state_was} to #{self.state}"
655 def ensure_no_collection_uuids_in_script_params
656 # Fail validation if any script_parameters field includes a string containing a
657 # collection uuid pattern.
658 if self.script_parameters_changed?
659 if recursive_hash_search(self.script_parameters, Collection.uuid_regex)
660 self.errors.add :script_parameters, "must use portable_data_hash instead of collection uuid"
667 # recursive_hash_search searches recursively through hashes and
668 # arrays in 'thing' for string fields matching regular expression
669 # 'pattern'. Returns true if pattern is found, false otherwise.
670 def recursive_hash_search thing, pattern
673 return true if recursive_hash_search v, pattern
675 elsif thing.is_a? Array
677 return true if recursive_hash_search k, pattern
679 elsif thing.is_a? String
680 return true if thing.match pattern