X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/01ea0e9faa0b29ef747699f7f4b728d4e888ef83..e0fade6bbda39812854fdfc316e8904886d23fe2:/services/api/app/models/container_request.rb diff --git a/services/api/app/models/container_request.rb b/services/api/app/models/container_request.rb index e84026a369..a264bbfe81 100644 --- a/services/api/app/models/container_request.rb +++ b/services/api/app/models/container_request.rb @@ -11,17 +11,22 @@ class ContainerRequest < ArvadosModel serialize :mounts, Hash serialize :runtime_constraints, Hash serialize :command, Array + serialize :scheduling_parameters, Hash before_validation :fill_field_defaults, :if => :new_record? + before_validation :validate_runtime_constraints + before_validation :validate_scheduling_parameters before_validation :set_container validates :command, :container_image, :output_path, :cwd, :presence => true validate :validate_state_change validate :validate_change after_save :update_priority + after_save :finalize_if_needed before_create :set_requesting_container_uuid api_accessible :user, extend: :common do |t| t.add :command + t.add :container_count t.add :container_count_max t.add :container_image t.add :container_uuid @@ -30,14 +35,19 @@ class ContainerRequest < ArvadosModel t.add :environment t.add :expires_at t.add :filters + t.add :log_uuid t.add :mounts t.add :name + t.add :output_name t.add :output_path + t.add :output_uuid t.add :priority t.add :properties t.add :requesting_container_uuid t.add :runtime_constraints + t.add :scheduling_parameters t.add :state + t.add :use_existing end # Supported states for a container request @@ -63,10 +73,64 @@ class ContainerRequest < ArvadosModel %w(modified_by_client_uuid container_uuid requesting_container_uuid) end - def container_completed! - # may implement retry logic here in the future. - self.state = ContainerRequest::Final - self.save! + def finalize_if_needed + if state == Committed && Container.find_by_uuid(container_uuid).final? + reload + act_as_system_user do + finalize! + end + end + end + + # Finalize the container request after the container has + # finished/cancelled. + def finalize! + out_coll = nil + log_coll = nil + c = Container.find_by_uuid(container_uuid) + ['output', 'log'].each do |out_type| + pdh = c.send(out_type) + next if pdh.nil? + if self.output_name and out_type == 'output' + coll_name = self.output_name + else + coll_name = "Container #{out_type} for request #{uuid}" + end + manifest = Collection.where(portable_data_hash: pdh).first.manifest_text + begin + coll = Collection.create!(owner_uuid: owner_uuid, + manifest_text: manifest, + portable_data_hash: pdh, + name: coll_name, + properties: { + 'type' => out_type, + 'container_request' => uuid, + }) + rescue ActiveRecord::RecordNotUnique => rn + # In case this is executed as part of a transaction: When a Postgres exception happens, + # the following statements on the same transaction become invalid, so a rollback is + # needed. One example are Unit Tests, every test is enclosed inside a transaction so + # that the database can be reverted before every new test starts. + # See: http://api.rubyonrails.org/classes/ActiveRecord/Transactions/ClassMethods.html#module-ActiveRecord::Transactions::ClassMethods-label-Exception+handling+and+rolling+back + ActiveRecord::Base.connection.execute 'ROLLBACK' + raise unless out_type == 'output' and self.output_name + # Postgres specific unique name check. See ApplicationController#create for + # a detailed explanation. + raise unless rn.original_exception.is_a? PG::UniqueViolation + err = rn.original_exception + detail = err.result.error_field(PG::Result::PG_DIAG_MESSAGE_DETAIL) + raise unless /^Key \(owner_uuid, name\)=\([a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15}, .*?\) already exists\./.match detail + # Output collection name collision detected: append a timestamp. + coll_name = "#{self.output_name} #{Time.now.getgm.strftime('%FT%TZ')}" + retry + end + if out_type == 'output' + out_coll = coll.uuid + else + log_coll = coll.uuid + end + end + update_attributes!(state: Final, output_uuid: out_coll, log_uuid: log_coll) end protected @@ -77,6 +141,8 @@ class ContainerRequest < ArvadosModel self.runtime_constraints ||= {} self.mounts ||= {} self.cwd ||= "." + self.container_count_max ||= Rails.configuration.container_count_max + self.scheduling_parameters ||= {} end # Create a new container (or find an existing one) to satisfy this @@ -86,13 +152,21 @@ class ContainerRequest < ArvadosModel c_runtime_constraints = runtime_constraints_for_container c_container_image = container_image_for_container c = act_as_system_user do - Container.create!(command: self.command, - cwd: self.cwd, - environment: self.environment, - output_path: self.output_path, - container_image: c_container_image, - mounts: c_mounts, - runtime_constraints: c_runtime_constraints) + c_attrs = {command: self.command, + cwd: self.cwd, + environment: self.environment, + output_path: self.output_path, + container_image: c_container_image, + mounts: c_mounts, + runtime_constraints: c_runtime_constraints} + + reusable = self.use_existing ? Container.find_reusable(c_attrs) : nil + if not reusable.nil? + reusable + else + c_attrs[:scheduling_parameters] = self.scheduling_parameters + Container.create!(c_attrs) + end end self.container_uuid = c.uuid end @@ -135,7 +209,7 @@ class ContainerRequest < ArvadosModel select(:portable_data_hash). first if !c - raise ActiveRecord::RecordNotFound.new "cannot mount collection #{uuid.inspect}: not found" + raise ArvadosModel::UnresolvableContainerError.new "cannot mount collection #{uuid.inspect}: not found" end if mount['portable_data_hash'].nil? # PDH not supplied by client @@ -153,7 +227,7 @@ class ContainerRequest < ArvadosModel def container_image_for_container coll = Collection.for_latest_docker_image(container_image) if !coll - raise ActiveRecord::RecordNotFound.new "docker image #{container_image.inspect} not found" + raise ArvadosModel::UnresolvableContainerError.new "docker image #{container_image.inspect} not found" end return coll.portable_data_hash end @@ -168,6 +242,46 @@ class ContainerRequest < ArvadosModel if state_changed? and state == Committed and container_uuid.nil? resolve end + if self.container_uuid != self.container_uuid_was + if self.container_count_changed? + errors.add :container_count, "cannot be updated directly." + return false + else + self.container_count += 1 + end + end + end + + def validate_runtime_constraints + case self.state + when Committed + ['vcpus', 'ram'].each do |k| + if not (runtime_constraints.include? k and + runtime_constraints[k].is_a? Integer and + runtime_constraints[k] > 0) + errors.add :runtime_constraints, "#{k} must be a positive integer" + end + end + + if runtime_constraints.include? 'keep_cache_ram' and + (!runtime_constraints['keep_cache_ram'].is_a?(Integer) or + runtime_constraints['keep_cache_ram'] <= 0) + errors.add :runtime_constraints, "keep_cache_ram must be a positive integer" + elsif !runtime_constraints.include? 'keep_cache_ram' + runtime_constraints['keep_cache_ram'] = Rails.configuration.container_default_keep_cache_ram + end + end + end + + def validate_scheduling_parameters + if self.state == Committed + if scheduling_parameters.include? 'partitions' and + (!scheduling_parameters['partitions'].is_a?(Array) || + scheduling_parameters['partitions'].reject{|x| !x.is_a?(String)}.size != + scheduling_parameters['partitions'].size) + errors.add :scheduling_parameters, "partitions must be an array of strings" + end + end end def validate_change @@ -180,7 +294,8 @@ class ContainerRequest < ArvadosModel :container_image, :cwd, :description, :environment, :filters, :mounts, :name, :output_path, :priority, :properties, :requesting_container_uuid, :runtime_constraints, - :state, :container_uuid + :state, :container_uuid, :use_existing, :scheduling_parameters, + :output_name when Committed if container_uuid.nil? @@ -192,14 +307,16 @@ class ContainerRequest < ArvadosModel end # Can update priority, container count, name and description - permitted.push :priority, :container_count_max, :container_uuid, :name, :description + permitted.push :priority, :container_count, :container_count_max, :container_uuid, + :name, :description if self.state_changed? # Allow create-and-commit in a single operation. permitted.push :command, :container_image, :cwd, :description, :environment, :filters, :mounts, :name, :output_path, :properties, :requesting_container_uuid, :runtime_constraints, - :state, :container_uuid + :state, :container_uuid, :use_existing, :scheduling_parameters, + :output_name end when Final @@ -207,8 +324,8 @@ class ContainerRequest < ArvadosModel errors.add :state, "of container request can only be set to Final by system." end - if self.state_changed? || self.name_changed? || self.description_changed? - permitted.push :state, :name, :description + if self.state_changed? || self.name_changed? || self.description_changed? || self.output_uuid_changed? || self.log_uuid_changed? + permitted.push :state, :name, :description, :output_uuid, :log_uuid else errors.add :state, "does not allow updates" end @@ -234,7 +351,7 @@ class ContainerRequest < ArvadosModel end def set_requesting_container_uuid - return true if self.requesting_container_uuid # already set + return !new_record? if self.requesting_container_uuid # already set token_uuid = current_api_client_authorization.andand.uuid container = Container.where('auth_uuid=?', token_uuid).order('created_at desc').first