X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/2a469c4874895b05ee137e2382fd882680b3feb2..0eb72b526bf8bbb011551ecf019f604e17a534f1:/services/api/app/models/job.rb diff --git a/services/api/app/models/job.rb b/services/api/app/models/job.rb index d73d629905..8de3897eab 100644 --- a/services/api/app/models/job.rb +++ b/services/api/app/models/job.rb @@ -1,8 +1,16 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +require 'log_reuse_info' +require 'safe_json' + class Job < ArvadosModel include HasUuid include KindAndEtag include CommonApiTemplate extend CurrentApiClient + extend LogReuseInfo serialize :components, Hash attr_protected :arvados_sdk_version, :docker_image_locator serialize :script_parameters, Hash @@ -71,6 +79,10 @@ class Job < ArvadosModel @need_crunch_dispatch_trigger = false end + def self.limit_index_columns_read + ["components"] + end + def assert_finished update_attributes(finished_at: finished_at || db_current_time, success: success.nil? ? false : success, @@ -147,7 +159,7 @@ class Job < ArvadosModel image_hashes = Array.wrap(operand).flat_map do |search_term| image_search, image_tag = search_term.split(':', 2) Collection. - find_all_for_docker_image(image_search, image_tag, read_users). + find_all_for_docker_image(image_search, image_tag, read_users, filter_compatible_format: false). map(&:portable_data_hash) end filters << [attr, operator.sub(/ docker$/, ""), image_hashes] @@ -231,45 +243,98 @@ class Job < ArvadosModel end # Search for a reusable Job, and return it if found. - candidates = Job. - readable_by(current_user). - where('state = ? or (owner_uuid = ? and state in (?))', - Job::Complete, current_user.uuid, [Job::Queued, Job::Running]). - where('script_parameters_digest = ?', Job.sorted_hash_digest(attrs[:script_parameters])). - where('nondeterministic is distinct from ?', true). - order('state desc, created_at') # prefer Running jobs over Queued + candidates = Job.readable_by(current_user) + log_reuse_info { "starting with #{candidates.count} jobs readable by current user #{current_user.uuid}" } + + candidates = candidates.where( + 'state = ? or (owner_uuid = ? and state in (?))', + Job::Complete, current_user.uuid, [Job::Queued, Job::Running]) + log_reuse_info(candidates) { "after filtering on job state ((state=Complete) or (state=Queued/Running and (submitted by current user)))" } + + digest = Job.sorted_hash_digest(attrs[:script_parameters]) + candidates = candidates.where('script_parameters_digest = ?', digest) + log_reuse_info(candidates) { "after filtering on script_parameters_digest #{digest}" } + + candidates = candidates.where('nondeterministic is distinct from ?', true) + log_reuse_info(candidates) { "after filtering on !nondeterministic" } + + # prefer Running jobs over Queued + candidates = candidates.order('state desc, created_at') + candidates = apply_filters candidates, filters + log_reuse_info(candidates) { "after filtering on repo, script, and custom filters #{filters.inspect}" } + chosen = nil + chosen_output = nil incomplete_job = nil candidates.each do |j| if j.state != Job::Complete - # We'll use this if we don't find a job that has completed - incomplete_job ||= j - next - end - - if chosen == false - # We have already decided not to reuse any completed job - next - elsif chosen - if chosen.output != j.output + if !incomplete_job + # We'll use this if we don't find a job that has completed + log_reuse_info { "job #{j.uuid} is reusable, but unfinished; continuing search for completed jobs" } + incomplete_job = j + else + log_reuse_info { "job #{j.uuid} is unfinished and we already have #{incomplete_job.uuid}; ignoring" } + end + elsif chosen == false + # Ignore: we have already decided not to reuse any completed + # job. + log_reuse_info { "job #{j.uuid} with output #{j.output} ignored, see above" } + elsif j.output.nil? + log_reuse_info { "job #{j.uuid} has nil output" } + elsif j.log.nil? + log_reuse_info { "job #{j.uuid} has nil log" } + elsif Rails.configuration.reuse_job_if_outputs_differ + if !Collection.readable_by(current_user).find_by_portable_data_hash(j.output) + # Ignore: keep looking for an incomplete job or one whose + # output is readable. + log_reuse_info { "job #{j.uuid} output #{j.output} unavailable to user; continuing search" } + elsif !Collection.readable_by(current_user).find_by_portable_data_hash(j.log) + # Ignore: keep looking for an incomplete job or one whose + # log is readable. + log_reuse_info { "job #{j.uuid} log #{j.log} unavailable to user; continuing search" } + else + log_reuse_info { "job #{j.uuid} with output #{j.output} is reusable; decision is final." } + return j + end + elsif chosen_output + if chosen_output != j.output # If two matching jobs produced different outputs, run a new # job (or use one that's already running/queued) instead of # choosing one arbitrarily. + log_reuse_info { "job #{j.uuid} output #{j.output} disagrees; forgetting about #{chosen.uuid} and ignoring any other finished jobs (see reuse_job_if_outputs_differ in application.default.yml)" } chosen = false + else + log_reuse_info { "job #{j.uuid} output #{j.output} agrees with chosen #{chosen.uuid}; continuing search in case other candidates have different outputs" } end # ...and that's the only thing we need to do once we've chosen # a job to reuse. elsif !Collection.readable_by(current_user).find_by_portable_data_hash(j.output) - # As soon as the output we will end up returning (if any) is - # decided, check whether it will be visible to the user; if - # not, any further investigation of reusable jobs is futile. + # This user cannot read the output of this job. Any other + # completed job will have either the same output (making it + # unusable) or a different output (making it unusable because + # reuse_job_if_outputs_different is turned off). Therefore, + # any further investigation of reusable jobs is futile. + log_reuse_info { "job #{j.uuid} output #{j.output} is unavailable to user; this means no finished job can be reused (see reuse_job_if_outputs_differ in application.default.yml)" } chosen = false + elsif !Collection.readable_by(current_user).find_by_portable_data_hash(j.log) + # This user cannot read the log of this job, don't try to reuse the + # job but consider if the output is consistent. + log_reuse_info { "job #{j.uuid} log #{j.log} is unavailable to user; continuing search" } + chosen_output = j.output else + log_reuse_info { "job #{j.uuid} with output #{j.output} can be reused; continuing search in case other candidates have different outputs" } chosen = j + chosen_output = j.output end end - chosen || incomplete_job + j = chosen || incomplete_job + if j + log_reuse_info { "done, #{j.uuid} was selected" } + else + log_reuse_info { "done, nothing suitable" } + end + return j end def self.default_git_filters(attr_name, repo_name, refspec) @@ -423,10 +488,11 @@ class Job < ArvadosModel end def find_docker_image_locator - runtime_constraints['docker_image'] = - Rails.configuration.default_docker_image_for_jobs if ((runtime_constraints.is_a? Hash) and - (runtime_constraints['docker_image']).nil? and - Rails.configuration.default_docker_image_for_jobs) + if runtime_constraints.is_a? Hash + runtime_constraints['docker_image'] ||= + Rails.configuration.default_docker_image_for_jobs + end + resolve_runtime_constraint("docker_image", :docker_image_locator) do |image_search| image_tag = runtime_constraints['docker_image_tag']