X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/765604b096c464b242e60860ace75bd6645fd4e7..d6446b03e2f5d5079a870bdd7b963456dc12b485:/services/api/app/models/container.rb diff --git a/services/api/app/models/container.rb b/services/api/app/models/container.rb index 7837e0812b..ee338b81ff 100644 --- a/services/api/app/models/container.rb +++ b/services/api/app/models/container.rb @@ -5,6 +5,7 @@ require 'log_reuse_info' require 'whitelist_update' require 'safe_json' +require 'update_priorities' class Container < ArvadosModel include ArvadosModelUpdates @@ -49,10 +50,16 @@ class Container < ArvadosModel before_save :clear_runtime_status_when_queued after_save :update_cr_logs after_save :handle_completed - after_save :propagate_priority - has_many :container_requests, :foreign_key => :container_uuid, :class_name => 'ContainerRequest', :primary_key => :uuid - belongs_to :auth, :class_name => 'ApiClientAuthorization', :foreign_key => :auth_uuid, :primary_key => :uuid + has_many :container_requests, + class_name: 'ContainerRequest', + foreign_key: 'container_uuid', + primary_key: 'uuid' + belongs_to :auth, + class_name: 'ApiClientAuthorization', + foreign_key: 'auth_uuid', + primary_key: 'uuid', + optional: true api_accessible :user, extend: :common do |t| t.add :command @@ -129,35 +136,8 @@ class Container < ArvadosModel # priority of a user-submitted request is a function of # user-assigned priority and request creation time. def update_priority! - return if ![Queued, Locked, Running].include?(state) - p = ContainerRequest. - where('container_uuid=? and priority>0', uuid). - includes(:requesting_container). - lock(true). - map do |cr| - if cr.requesting_container - cr.requesting_container.priority - else - (cr.priority << 50) - (cr.created_at.to_time.to_f * 1000).to_i - end - end.max || 0 - update_attributes!(priority: p) - end - - def propagate_priority - return true unless saved_change_to_priority? - act_as_system_user do - # Update the priority of child container requests to match new - # priority of the parent container (ignoring requests with no - # container assigned, because their priority doesn't matter). - ContainerRequest. - where(requesting_container_uuid: self.uuid, - state: ContainerRequest::Committed). - where('container_uuid is not null'). - includes(:container). - map(&:container). - map(&:update_priority!) - end + update_priorities uuid + reload end # Create a new container (or find an existing one) to satisfy the @@ -339,7 +319,7 @@ class Container < ArvadosModel resolved_runtime_constraints.delete('cuda') ].uniq end - reusable_runtime_constraints = hash_product(runtime_constraint_variations) + reusable_runtime_constraints = hash_product(**runtime_constraint_variations) .map { |v| resolved_runtime_constraints.merge(v) } candidates = candidates.where_serialized(:runtime_constraints, reusable_runtime_constraints, md5: true, multivalue: true) @@ -370,7 +350,7 @@ class Container < ArvadosModel # Check for non-failing Running candidates and return the most likely to finish sooner. log_reuse_info { "checking for state=Running..." } running = candidates.where(state: Running). - where("(runtime_status->'error') is null"). + where("(runtime_status->'error') is null and priority > 0"). order('progress desc, started_at asc'). limit(1).first if running @@ -384,10 +364,15 @@ class Container < ArvadosModel locked_or_queued = candidates. where("state IN (?)", [Locked, Queued]). order('state asc, priority desc, created_at asc'). - limit(1).first - if locked_or_queued - log_reuse_info { "done, reusing container #{locked_or_queued.uuid} with state=#{locked_or_queued.state}" } - return locked_or_queued + limit(1) + if !attrs[:scheduling_parameters]['preemptible'] + locked_or_queued = locked_or_queued. + where("not ((scheduling_parameters::jsonb)->>'preemptible')::boolean") + end + chosen = locked_or_queued.first + if chosen + log_reuse_info { "done, reusing container #{chosen.uuid} with state=#{chosen.state}" } + return chosen else log_reuse_info { "have no containers in Locked or Queued state" } end @@ -401,7 +386,7 @@ class Container < ArvadosModel if self.state != Queued raise LockFailedError.new("cannot lock when #{self.state}") end - self.update_attributes!(state: Locked) + self.update!(state: Locked) end end @@ -419,7 +404,7 @@ class Container < ArvadosModel if self.state != Locked raise InvalidStateTransitionError.new("cannot unlock when #{self.state}") end - self.update_attributes!(state: Queued) + self.update!(state: Queued) end end @@ -646,7 +631,7 @@ class Container < ArvadosModel # each requesting CR. return if self.final? || !saved_change_to_log? leave_modified_by_user_alone do - ContainerRequest.where(container_uuid: self.uuid).each do |cr| + ContainerRequest.where(container_uuid: self.uuid, state: ContainerRequest::Committed).each do |cr| cr.update_collections(container: self, collections: ['log']) cr.save! end @@ -664,7 +649,7 @@ class Container < ArvadosModel # ensure the token doesn't validate later in the same # transaction (e.g., in a test case) by satisfying expires_at > # transaction timestamp. - self.auth.andand.update_attributes(expires_at: db_transaction_time) + self.auth.andand.update(expires_at: db_transaction_time) self.auth = nil return elsif self.auth @@ -757,7 +742,22 @@ class Container < ArvadosModel self.with_lock do act_as_system_user do if self.state == Cancelled - retryable_requests = ContainerRequest.where("container_uuid = ? and priority > 0 and state = 'Committed' and container_count < container_count_max", uuid) + # Cancelled means the container didn't run to completion. + # This happens either because it was cancelled by the user + # or because there was an infrastructure failure. We want + # to retry infrastructure failures automatically. + # + # Seach for live container requests to determine if we + # should retry the container. + retryable_requests = ContainerRequest. + joins('left outer join containers as requesting_container on container_requests.requesting_container_uuid = requesting_container.uuid'). + where("container_requests.container_uuid = ? and "+ + "container_requests.priority > 0 and "+ + "container_requests.owner_uuid not in (select group_uuid from trashed_groups) and "+ + "(requesting_container.priority is null or (requesting_container.state = 'Running' and requesting_container.priority > 0)) and "+ + "container_requests.state = 'Committed' and "+ + "container_requests.container_count < container_requests.container_count_max", uuid). + order('container_requests.uuid asc') else retryable_requests = [] end @@ -830,19 +830,19 @@ class Container < ArvadosModel # Cancel outstanding container requests made by this container. ContainerRequest. - includes(:container). where(requesting_container_uuid: uuid, - state: ContainerRequest::Committed).each do |cr| + state: ContainerRequest::Committed). + in_batches(of: 15).each_record do |cr| leave_modified_by_user_alone do - cr.update_attributes!(priority: 0) - cr.container.reload - if cr.container.state == Container::Queued || cr.container.state == Container::Locked + cr.set_priority_zero + container_state = Container.where(uuid: cr.container_uuid).pluck(:state).first + if container_state == Container::Queued || container_state == Container::Locked # If the child container hasn't started yet, finalize the # child CR now instead of leaving it "on hold", i.e., # Queued with priority 0. (OTOH, if the child is already # running, leave it alone so it can get cancelled the # usual way, get a copy of the log collection, etc.) - cr.update_attributes!(state: ContainerRequest::Final) + cr.update!(state: ContainerRequest::Final) end end end