X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/3d65beeef57b2dd7a60171c8dce05785ef1ae383..1c4bf0f3bbc09808f4937dafa6c3023604b73dce:/services/api/app/models/container.rb?ds=sidebyside diff --git a/services/api/app/models/container.rb b/services/api/app/models/container.rb index e6643d4c70..d2e76f74e3 100644 --- a/services/api/app/models/container.rb +++ b/services/api/app/models/container.rb @@ -5,7 +5,7 @@ require 'log_reuse_info' require 'whitelist_update' require 'safe_json' -require 'update_priority' +require 'update_priorities' class Container < ArvadosModel include ArvadosModelUpdates @@ -50,8 +50,6 @@ class Container < ArvadosModel before_save :clear_runtime_status_when_queued after_save :update_cr_logs after_save :handle_completed - after_save :propagate_priority - after_commit { UpdatePriority.run_update_thread } has_many :container_requests, :foreign_key => :container_uuid, :class_name => 'ContainerRequest', :primary_key => :uuid belongs_to :auth, :class_name => 'ApiClientAuthorization', :foreign_key => :auth_uuid, :primary_key => :uuid @@ -131,35 +129,8 @@ class Container < ArvadosModel # priority of a user-submitted request is a function of # user-assigned priority and request creation time. def update_priority! - return if ![Queued, Locked, Running].include?(state) - p = ContainerRequest. - where('container_uuid=? and priority>0', uuid). - includes(:requesting_container). - lock(true). - map do |cr| - if cr.requesting_container - cr.requesting_container.priority - else - (cr.priority << 50) - (cr.created_at.to_time.to_f * 1000).to_i - end - end.max || 0 - update_attributes!(priority: p) - end - - def propagate_priority - return true unless saved_change_to_priority? - act_as_system_user do - # Update the priority of child container requests to match new - # priority of the parent container (ignoring requests with no - # container assigned, because their priority doesn't matter). - ContainerRequest. - where(requesting_container_uuid: self.uuid, - state: ContainerRequest::Committed). - where('container_uuid is not null'). - includes(:container). - map(&:container). - map(&:update_priority!) - end + update_priorities uuid + reload end # Create a new container (or find an existing one) to satisfy the @@ -372,7 +343,7 @@ class Container < ArvadosModel # Check for non-failing Running candidates and return the most likely to finish sooner. log_reuse_info { "checking for state=Running..." } running = candidates.where(state: Running). - where("(runtime_status->'error') is null"). + where("(runtime_status->'error') is null and priority > 0"). order('progress desc, started_at asc'). limit(1).first if running @@ -386,10 +357,15 @@ class Container < ArvadosModel locked_or_queued = candidates. where("state IN (?)", [Locked, Queued]). order('state asc, priority desc, created_at asc'). - limit(1).first - if locked_or_queued - log_reuse_info { "done, reusing container #{locked_or_queued.uuid} with state=#{locked_or_queued.state}" } - return locked_or_queued + limit(1) + if !attrs[:scheduling_parameters]['preemptible'] + locked_or_queued = locked_or_queued. + where("not ((scheduling_parameters::jsonb)->>'preemptible')::boolean") + end + chosen = locked_or_queued.first + if chosen + log_reuse_info { "done, reusing container #{chosen.uuid} with state=#{chosen.state}" } + return chosen else log_reuse_info { "have no containers in Locked or Queued state" } end @@ -648,7 +624,7 @@ class Container < ArvadosModel # each requesting CR. return if self.final? || !saved_change_to_log? leave_modified_by_user_alone do - ContainerRequest.where(container_uuid: self.uuid).each do |cr| + ContainerRequest.where(container_uuid: self.uuid, state: ContainerRequest::Committed).each do |cr| cr.update_collections(container: self, collections: ['log']) cr.save! end @@ -759,7 +735,22 @@ class Container < ArvadosModel self.with_lock do act_as_system_user do if self.state == Cancelled - retryable_requests = ContainerRequest.where("container_uuid = ? and priority > 0 and state = 'Committed' and container_count < container_count_max", uuid) + # Cancelled means the container didn't run to completion. + # This happens either because it was cancelled by the user + # or because there was an infrastructure failure. We want + # to retry infrastructure failures automatically. + # + # Seach for live container requests to determine if we + # should retry the container. + retryable_requests = ContainerRequest. + joins('left outer join containers as requesting_container on container_requests.requesting_container_uuid = requesting_container.uuid'). + where("container_requests.container_uuid = ? and "+ + "container_requests.priority > 0 and "+ + "container_requests.owner_uuid not in (select group_uuid from trashed_groups) and "+ + "(requesting_container.priority is null or (requesting_container.state = 'Running' and requesting_container.priority > 0)) and "+ + "container_requests.state = 'Committed' and "+ + "container_requests.container_count < container_requests.container_count_max", uuid). + order('container_requests.uuid asc') else retryable_requests = [] end @@ -832,13 +823,13 @@ class Container < ArvadosModel # Cancel outstanding container requests made by this container. ContainerRequest. - includes(:container). where(requesting_container_uuid: uuid, - state: ContainerRequest::Committed).each do |cr| + state: ContainerRequest::Committed). + in_batches(of: 15).each_record do |cr| leave_modified_by_user_alone do - cr.update_attributes!(priority: 0) - cr.container.reload - if cr.container.state == Container::Queued || cr.container.state == Container::Locked + cr.set_priority_zero + container_state = Container.where(uuid: cr.container_uuid).pluck(:state).first + if container_state == Container::Queued || container_state == Container::Locked # If the child container hasn't started yet, finalize the # child CR now instead of leaving it "on hold", i.e., # Queued with priority 0. (OTOH, if the child is already