3 class Job < ArvadosModel
6 include CommonApiTemplate
7 extend CurrentApiClient
8 serialize :components, Hash
9 attr_protected :arvados_sdk_version, :docker_image_locator
10 serialize :script_parameters, Hash
11 serialize :runtime_constraints, Hash
12 serialize :tasks_summary, Hash
13 before_create :ensure_unique_submit_id
14 after_commit :trigger_crunch_dispatch_if_cancelled, :on => :update
15 before_validation :set_priority
16 before_validation :update_state_from_old_state_attrs
17 before_validation :update_script_parameters_digest
18 validate :ensure_script_version_is_commit
19 validate :find_docker_image_locator
20 validate :find_arvados_sdk_version
21 validate :validate_status
22 validate :validate_state_change
23 validate :ensure_no_collection_uuids_in_script_params
24 before_save :tag_version_in_internal_repository
25 before_save :update_timestamps_when_state_changes
27 has_many :commit_ancestors, :foreign_key => :descendant, :primary_key => :script_version
28 has_many(:nodes, foreign_key: :job_uuid, primary_key: :uuid)
30 class SubmitIdReused < StandardError
33 api_accessible :user, extend: :common do |t|
37 t.add :script_parameters
40 t.add :cancelled_by_client_uuid
41 t.add :cancelled_by_user_uuid
48 t.add :is_locked_by_uuid
50 t.add :runtime_constraints
52 t.add :nondeterministic
54 t.add :supplied_script_version
55 t.add :arvados_sdk_version
56 t.add :docker_image_locator
63 # Supported states for a job
66 (Running = 'Running'),
67 (Cancelled = 'Cancelled'),
69 (Complete = 'Complete'),
73 @need_crunch_dispatch_trigger = false
76 def self.limit_index_columns_read
81 update_attributes(finished_at: finished_at || db_current_time,
82 success: success.nil? ? false : success,
91 self.where('state = ?', Queued).order('priority desc, created_at')
95 # We used to report this accurately, but the implementation made queue
96 # API requests O(n**2) for the size of the queue. See #8800.
97 # We've soft-disabled it because it's not clear we even want this
98 # functionality: now that we have Node Manager with support for multiple
99 # node sizes, "queue position" tells you very little about when a job will
101 state == Queued ? 0 : nil
105 self.where('running = ?', true).
106 order('priority desc, created_at')
109 def lock locked_by_uuid
111 unless self.state == Queued and self.is_locked_by_uuid.nil?
112 raise AlreadyLockedError
115 self.is_locked_by_uuid = locked_by_uuid
120 def update_script_parameters_digest
121 self.script_parameters_digest = self.class.sorted_hash_digest(script_parameters)
124 def self.searchable_columns operator
125 super - ["script_parameters_digest"]
128 def self.full_text_searchable_columns
129 super - ["script_parameters_digest"]
132 def self.load_job_specific_filters attrs, orig_filters, read_users
133 # Convert Job-specific @filters entries into general SQL filters.
134 script_info = {"repository" => nil, "script" => nil}
135 git_filters = Hash.new do |hash, key|
136 hash[key] = {"max_version" => "HEAD", "exclude_versions" => []}
139 orig_filters.each do |attr, operator, operand|
140 if (script_info.has_key? attr) and (operator == "=")
141 if script_info[attr].nil?
142 script_info[attr] = operand
143 elsif script_info[attr] != operand
144 raise ArgumentError.new("incompatible #{attr} filters")
149 git_filters[attr]["min_version"] = operand
151 git_filters[attr]["exclude_versions"] += Array.wrap(operand)
152 when "in docker", "not in docker"
153 image_hashes = Array.wrap(operand).flat_map do |search_term|
154 image_search, image_tag = search_term.split(':', 2)
156 find_all_for_docker_image(image_search, image_tag, read_users, filter_compatible_format: false).
157 map(&:portable_data_hash)
159 filters << [attr, operator.sub(/ docker$/, ""), image_hashes]
161 filters << [attr, operator, operand]
165 # Build a real script_version filter from any "not? in git" filters.
166 git_filters.each_pair do |attr, filter|
168 when "script_version"
169 script_info.each_pair do |key, value|
171 raise ArgumentError.new("script_version filter needs #{key} filter")
174 filter["repository"] = script_info["repository"]
175 if attrs[:script_version]
176 filter["max_version"] = attrs[:script_version]
178 # Using HEAD, set earlier by the hash default, is fine.
180 when "arvados_sdk_version"
181 filter["repository"] = "arvados"
183 raise ArgumentError.new("unknown attribute for git filter: #{attr}")
185 revisions = Commit.find_commit_range(filter["repository"],
186 filter["min_version"],
187 filter["max_version"],
188 filter["exclude_versions"])
191 new("error searching #{filter['repository']} from " +
192 "'#{filter['min_version']}' to '#{filter['max_version']}', " +
193 "excluding #{filter['exclude_versions']}")
195 filters.append([attr, "in", revisions])
201 # log_reuse_info logs whatever the given block returns, if
202 # log_reuse_decisions is enabled. It accepts a block instead of a
203 # string because in some cases constructing the strings involves
204 # doing database queries, and we want to skip those queries when
205 # logging is disabled.
206 def self.log_reuse_info
207 if Rails.configuration.log_reuse_decisions
208 Rails.logger.info("find_reusable: " + yield)
212 def self.find_reusable attrs, params, filters, read_users
213 if filters.empty? # Translate older creation parameters into filters.
215 [["repository", "=", attrs[:repository]],
216 ["script", "=", attrs[:script]],
217 ["script_version", "not in git", params[:exclude_script_versions]],
218 ].reject { |filter| filter.last.nil? or filter.last.empty? }
219 if !params[:minimum_script_version].blank?
220 filters << ["script_version", "in git",
221 params[:minimum_script_version]]
223 filters += default_git_filters("script_version", attrs[:repository],
224 attrs[:script_version])
226 if image_search = attrs[:runtime_constraints].andand["docker_image"]
227 if image_tag = attrs[:runtime_constraints]["docker_image_tag"]
228 image_search += ":#{image_tag}"
230 image_locator = Collection.
231 for_latest_docker_image(image_search).andand.portable_data_hash
235 filters << ["docker_image_locator", "=", image_locator]
236 if sdk_version = attrs[:runtime_constraints].andand["arvados_sdk_version"]
237 filters += default_git_filters("arvados_sdk_version", "arvados", sdk_version)
239 filters = load_job_specific_filters(attrs, filters, read_users)
242 # Check specified filters for some reasonableness.
243 filter_names = filters.map { |f| f.first }.uniq
244 ["repository", "script"].each do |req_filter|
245 if not filter_names.include?(req_filter)
246 return send_error("#{req_filter} filter required")
250 # Search for a reusable Job, and return it if found.
251 candidates = Job.readable_by(current_user)
252 log_reuse_info { "starting with #{candidates.count} jobs readable by current user #{current_user.uuid}" }
254 candidates = candidates.where(
255 'state = ? or (owner_uuid = ? and state in (?))',
256 Job::Complete, current_user.uuid, [Job::Queued, Job::Running])
257 log_reuse_info { "have #{candidates.count} candidates after filtering on job state ((state=Complete) or (state=Queued/Running and (submitted by current user)))" }
259 digest = Job.sorted_hash_digest(attrs[:script_parameters])
260 candidates = candidates.where('script_parameters_digest = ?', digest)
261 log_reuse_info { "have #{candidates.count} candidates after filtering on script_parameters_digest #{digest}" }
263 candidates = candidates.where('nondeterministic is distinct from ?', true)
264 log_reuse_info { "have #{candidates.count} candidates after filtering on !nondeterministic" }
266 # prefer Running jobs over Queued
267 candidates = candidates.order('state desc, created_at')
269 candidates = apply_filters candidates, filters
270 log_reuse_info { "have #{candidates.count} candidates after filtering on repo, script, and custom filters #{filters.inspect}" }
274 candidates.each do |j|
275 if j.state != Job::Complete
277 # We'll use this if we don't find a job that has completed
278 log_reuse_info { "job #{j.uuid} is reusable, but unfinished; continuing search for completed jobs" }
281 log_reuse_info { "job #{j.uuid} is unfinished and we already have #{incomplete_job.uuid}; ignoring" }
283 elsif chosen == false
284 # Ignore: we have already decided not to reuse any completed
286 log_reuse_info { "job #{j.uuid} with output #{j.output} ignored, see above" }
287 elsif Rails.configuration.reuse_job_if_outputs_differ
288 if Collection.readable_by(current_user).find_by_portable_data_hash(j.output)
289 log_reuse_info { "job #{j.uuid} with output #{j.output} is reusable; decision is final." }
292 # Ignore: keep locking for an incomplete job or one whose
293 # output is readable.
294 log_reuse_info { "job #{j.uuid} output #{j.output} unavailable to user; continuing search" }
297 if chosen.output != j.output
298 # If two matching jobs produced different outputs, run a new
299 # job (or use one that's already running/queued) instead of
300 # choosing one arbitrarily.
301 log_reuse_info { "job #{j.uuid} output #{j.output} disagrees; forgetting about #{chosen.uuid} and ignoring any other finished jobs (see reuse_job_if_outputs_differ in application.default.yml)" }
304 log_reuse_info { "job #{j.uuid} output #{j.output} agrees with chosen #{chosen.uuid}; continuing search in case other candidates have different outputs" }
306 # ...and that's the only thing we need to do once we've chosen
308 elsif !Collection.readable_by(current_user).find_by_portable_data_hash(j.output)
309 # This user cannot read the output of this job. Any other
310 # completed job will have either the same output (making it
311 # unusable) or a different output (making it unusable because
312 # reuse_job_if_outputs_different is turned off). Therefore,
313 # any further investigation of reusable jobs is futile.
314 log_reuse_info { "job #{j.uuid} output #{j.output} is unavailable to user; this means no finished job can be reused (see reuse_job_if_outputs_differ in application.default.yml)" }
317 log_reuse_info { "job #{j.uuid} with output #{j.output} can be reused; continuing search in case other candidates have different outputs" }
321 j = chosen || incomplete_job
323 log_reuse_info { "done, #{j.uuid} was selected" }
325 log_reuse_info { "done, nothing suitable" }
330 def self.default_git_filters(attr_name, repo_name, refspec)
331 # Add a filter to @filters for `attr_name` = the latest commit available
332 # in `repo_name` at `refspec`. No filter is added if refspec can't be
334 commits = Commit.find_commit_range(repo_name, nil, refspec, nil)
335 if commit_hash = commits.first
336 [[attr_name, "=", commit_hash]]
342 def cancel(cascade: false, need_transaction: true)
344 ActiveRecord::Base.transaction do
345 cancel(cascade: cascade, need_transaction: false)
350 if self.state.in?([Queued, Running])
351 self.state = Cancelled
353 elsif self.state != Cancelled
354 raise InvalidStateTransitionError
359 # cancel all children; they could be jobs or pipeline instances
360 children = self.components.andand.collect{|_, u| u}.compact
362 return if children.empty?
364 # cancel any child jobs
365 Job.where(uuid: children, state: [Queued, Running]).each do |job|
366 job.cancel(cascade: cascade, need_transaction: false)
369 # cancel any child pipelines
370 PipelineInstance.where(uuid: children, state: [PipelineInstance::RunningOnServer, PipelineInstance::RunningOnClient]).each do |pi|
371 pi.cancel(cascade: cascade, need_transaction: false)
377 def self.sorted_hash_digest h
378 Digest::MD5.hexdigest(Oj.dump(deep_sort_hash(h)))
381 def foreign_key_attributes
382 super + %w(output log)
385 def skip_uuid_read_permission_check
386 super + %w(cancelled_by_client_uuid)
389 def skip_uuid_existence_check
390 super + %w(output log)
394 if self.priority.nil?
400 def ensure_script_version_is_commit
402 # Apparently client has already decided to go for it. This is
403 # needed to run a local job using a local working directory
404 # instead of a commit-ish.
407 if new_record? or repository_changed? or script_version_changed?
408 sha1 = Commit.find_commit_range(repository,
409 nil, script_version, nil).first
411 errors.add :script_version, "#{script_version} does not resolve to a commit"
414 if supplied_script_version.nil? or supplied_script_version.empty?
415 self.supplied_script_version = script_version
417 self.script_version = sha1
422 def tag_version_in_internal_repository
424 # No point now. See ensure_script_version_is_commit.
427 # Won't be saved, and script_version might not even be valid.
429 elsif new_record? or repository_changed? or script_version_changed?
433 Commit.tag_in_internal_repository repository, script_version, uuid
441 def ensure_unique_submit_id
443 if Job.where('submit_id=?',self.submit_id).first
444 raise SubmitIdReused.new
450 def resolve_runtime_constraint(key, attr_sym)
451 if ((runtime_constraints.is_a? Hash) and
452 (search = runtime_constraints[key]))
453 ok, result = yield search
455 ok, result = true, nil
458 send("#{attr_sym}=".to_sym, result)
460 errors.add(attr_sym, result)
465 def find_arvados_sdk_version
466 resolve_runtime_constraint("arvados_sdk_version",
467 :arvados_sdk_version) do |git_search|
468 commits = Commit.find_commit_range("arvados",
469 nil, git_search, nil)
471 [false, "#{git_search} does not resolve to a commit"]
472 elsif not runtime_constraints["docker_image"]
473 [false, "cannot be specified without a Docker image constraint"]
475 [true, commits.first]
480 def find_docker_image_locator
481 if runtime_constraints.is_a? Hash
482 runtime_constraints['docker_image'] ||=
483 Rails.configuration.default_docker_image_for_jobs
486 resolve_runtime_constraint("docker_image",
487 :docker_image_locator) do |image_search|
488 image_tag = runtime_constraints['docker_image_tag']
489 if coll = Collection.for_latest_docker_image(image_search, image_tag)
490 [true, coll.portable_data_hash]
492 [false, "not found for #{image_search}"]
497 def permission_to_update
498 if is_locked_by_uuid_was and !(current_user and
499 (current_user.uuid == is_locked_by_uuid_was or
500 current_user.uuid == system_user.uuid))
501 if script_changed? or
502 script_parameters_changed? or
503 script_version_changed? or
504 (!cancelled_at_was.nil? and
505 (cancelled_by_client_uuid_changed? or
506 cancelled_by_user_uuid_changed? or
507 cancelled_at_changed?)) or
508 started_at_changed? or
509 finished_at_changed? or
514 tasks_summary_changed? or
515 (state_changed? && state != Cancelled) or
517 logger.warn "User #{current_user.uuid if current_user} tried to change protected job attributes on locked #{self.class.to_s} #{uuid_was}"
521 if !is_locked_by_uuid_changed?
525 logger.warn "Anonymous user tried to change lock on #{self.class.to_s} #{uuid_was}"
527 elsif is_locked_by_uuid_was and is_locked_by_uuid_was != current_user.uuid
528 logger.warn "User #{current_user.uuid} tried to steal lock on #{self.class.to_s} #{uuid_was} from #{is_locked_by_uuid_was}"
530 elsif !is_locked_by_uuid.nil? and is_locked_by_uuid != current_user.uuid
531 logger.warn "User #{current_user.uuid} tried to lock #{self.class.to_s} #{uuid_was} with uuid #{is_locked_by_uuid}"
539 def update_modified_by_fields
540 if self.cancelled_at_changed?
541 # Ensure cancelled_at cannot be set to arbitrary non-now times,
542 # or changed once it is set.
543 if self.cancelled_at and not self.cancelled_at_was
544 self.cancelled_at = db_current_time
545 self.cancelled_by_user_uuid = current_user.uuid
546 self.cancelled_by_client_uuid = current_api_client.andand.uuid
547 @need_crunch_dispatch_trigger = true
549 self.cancelled_at = self.cancelled_at_was
550 self.cancelled_by_user_uuid = self.cancelled_by_user_uuid_was
551 self.cancelled_by_client_uuid = self.cancelled_by_client_uuid_was
557 def trigger_crunch_dispatch_if_cancelled
558 if @need_crunch_dispatch_trigger
559 File.open(Rails.configuration.crunch_refresh_trigger, 'wb') do
560 # That's all, just create/touch a file for crunch-job to see.
565 def update_timestamps_when_state_changes
566 return if not (state_changed? or new_record?)
570 self.started_at ||= db_current_time
571 when Failed, Complete
572 self.finished_at ||= db_current_time
574 self.cancelled_at ||= db_current_time
577 # TODO: Remove the following case block when old "success" and
578 # "running" attrs go away. Until then, this ensures we still
579 # expose correct success/running flags to older clients, even if
580 # some new clients are writing only the new state attribute.
588 when Cancelled, Failed
595 self.running ||= false # Default to false instead of nil.
597 @need_crunch_dispatch_trigger = true
602 def update_state_from_old_state_attrs
603 # If a client has touched the legacy state attrs, update the
604 # "state" attr to agree with the updated values of the legacy
607 # TODO: Remove this method when old "success" and "running" attrs
609 if cancelled_at_changed? or
614 self.state = Cancelled
615 elsif success == false
617 elsif success == true
618 self.state = Complete
619 elsif running == true
629 if self.state.in?(States)
632 errors.add :state, "#{state.inspect} must be one of: #{States.inspect}"
637 def validate_state_change
639 if self.state_changed?
640 ok = case self.state_was
642 # state isn't set yet
645 # Permit going from queued to any state
648 # From running, may only transition to a finished state
649 [Complete, Failed, Cancelled].include? self.state
650 when Complete, Failed, Cancelled
651 # Once in a finished state, don't permit any more state changes
654 # Any other state transition is also invalid
658 errors.add :state, "invalid change from #{self.state_was} to #{self.state}"
664 def ensure_no_collection_uuids_in_script_params
665 # Fail validation if any script_parameters field includes a string containing a
666 # collection uuid pattern.
667 if self.script_parameters_changed?
668 if recursive_hash_search(self.script_parameters, Collection.uuid_regex)
669 self.errors.add :script_parameters, "must use portable_data_hash instead of collection uuid"
676 # recursive_hash_search searches recursively through hashes and
677 # arrays in 'thing' for string fields matching regular expression
678 # 'pattern'. Returns true if pattern is found, false otherwise.
679 def recursive_hash_search thing, pattern
682 return true if recursive_hash_search v, pattern
684 elsif thing.is_a? Array
686 return true if recursive_hash_search k, pattern
688 elsif thing.is_a? String
689 return true if thing.match pattern