X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/f30e59af90558e47a4156566d0209bc8efacd85a..b5c15775caf865d8fed5d4839d4a082f6518bba4:/services/api/app/models/container_request.rb diff --git a/services/api/app/models/container_request.rb b/services/api/app/models/container_request.rb index bec3deb295..f5789f31f6 100644 --- a/services/api/app/models/container_request.rb +++ b/services/api/app/models/container_request.rb @@ -12,18 +12,22 @@ class ContainerRequest < ArvadosModel include CommonApiTemplate include WhitelistUpdate - belongs_to :container, foreign_key: :container_uuid, primary_key: :uuid - belongs_to :requesting_container, { - class_name: 'Container', - foreign_key: :requesting_container_uuid, - primary_key: :uuid, - } + belongs_to :container, + foreign_key: 'container_uuid', + primary_key: 'uuid', + optional: true + belongs_to :requesting_container, + class_name: 'Container', + foreign_key: 'requesting_container_uuid', + primary_key: 'uuid', + optional: true # Posgresql JSONB columns should NOT be declared as serialized, Rails 5 # already know how to properly treat them. attribute :properties, :jsonbHash, default: {} attribute :secret_mounts, :jsonbHash, default: {} attribute :output_storage_classes, :jsonbArray, default: lambda { Rails.configuration.DefaultStorageClasses } + attribute :output_properties, :jsonbHash, default: {} serialize :environment, Hash serialize :mounts, Hash @@ -32,6 +36,7 @@ class ContainerRequest < ArvadosModel serialize :scheduling_parameters, Hash after_find :fill_container_defaults_after_find + after_initialize { @state_was_when_initialized = self.state_was } # see finalize_if_needed before_validation :fill_field_defaults, :if => :new_record? before_validation :fill_container_defaults validates :command, :container_image, :output_path, :cwd, :presence => true @@ -78,6 +83,8 @@ class ContainerRequest < ArvadosModel t.add :state t.add :use_existing t.add :output_storage_classes + t.add :output_properties + t.add :cumulative_cost end # Supported states for a container request @@ -100,7 +107,7 @@ class ContainerRequest < ArvadosModel :output_path, :priority, :runtime_token, :runtime_constraints, :state, :container_uuid, :use_existing, :scheduling_parameters, :secret_mounts, :output_name, :output_ttl, - :output_storage_classes] + :output_storage_classes, :output_properties] def self.any_preemptible_instances? Rails.configuration.InstanceTypes.any? do |k, v| @@ -160,7 +167,7 @@ class ContainerRequest < ArvadosModel end elsif state == Committed # Behave as if the container is cancelled - update_attributes!(state: Final) + update!(state: Final) end return true end @@ -171,8 +178,39 @@ class ContainerRequest < ArvadosModel def finalize! container = Container.find_by_uuid(container_uuid) if !container.nil? - update_collections(container: container) + # We don't want to add the container cost if the container was + # already finished when this CR was committed. But we are + # running in an after_save hook after a lock/reload, so + # state_was has already been updated to Committed regardless. + # Hence the need for @state_was_when_initialized. + if @state_was_when_initialized == Committed + # Add the final container cost to our cumulative cost (which + # may already be non-zero from previous attempts if + # container_count_max > 1). + self.cumulative_cost += container.cost + container.subrequests_cost + end + + # Add our cumulative cost to the subrequests_cost of the + # requesting container, if any. + if self.requesting_container_uuid + Container.where( + uuid: self.requesting_container_uuid, + state: Container::Running, + ).each do |c| + c.subrequests_cost += self.cumulative_cost + c.save! + end + end + update_collections(container: container) + # update_collections makes a log collection that includes all of the logs + # for all of the containers associated with this request. For requests + # that are retried, this is the primary way users can get logs for + # failed containers. + # The code below makes a log collection that is a verbatim copy of the + # container's logs. This is required for container reuse: a container + # will not be reused if the owner cannot read a collection with its logs. + # See the "readable log" section of Container.find_reusable(). if container.state == Container::Complete log_col = Collection.where(portable_data_hash: container.log).first if log_col @@ -193,10 +231,17 @@ class ContainerRequest < ArvadosModel end end end - update_attributes!(state: Final) + update!(state: Final) end def update_collections(container:, collections: ['log', 'output']) + + # Check if parent is frozen or trashed, in which case it isn't + # valid to create new collections in the project, so return + # without creating anything. + owner = Group.find_by_uuid(self.owner_uuid) + return if owner && !owner.admin_change_permitted + collections.each do |out_type| pdh = container.send(out_type) next if pdh.nil? @@ -222,11 +267,7 @@ class ContainerRequest < ArvadosModel owner_uuid: self.owner_uuid, name: coll_name, manifest_text: "", - storage_classes_desired: self.output_storage_classes, - properties: { - 'type' => out_type, - 'container_request' => uuid, - }) + storage_classes_desired: self.output_storage_classes) end if out_type == "log" @@ -238,11 +279,28 @@ class ContainerRequest < ArvadosModel manifest = dst.manifest_text end + merged_properties = {} + merged_properties['container_request'] = uuid + + if out_type == 'output' and !requesting_container_uuid.nil? + # output of a child process, give it "intermediate" type by + # default. + merged_properties['type'] = 'intermediate' + else + merged_properties['type'] = out_type + end + + if out_type == "output" + merged_properties.update(container.output_properties) + merged_properties.update(self.output_properties) + end + coll.assign_attributes( portable_data_hash: Digest::MD5.hexdigest(manifest) + '+' + manifest.bytesize.to_s, manifest_text: manifest, trash_at: trash_at, - delete_at: trash_at) + delete_at: trash_at, + properties: merged_properties) coll.save_with_unique_name! self.send(out_type + '_uuid=', coll.uuid) end @@ -252,6 +310,10 @@ class ContainerRequest < ArvadosModel super - ["mounts", "secret_mounts", "secret_mounts_md5", "runtime_token", "output_storage_classes"] end + def set_priority_zero + self.update!(priority: 0) if self.priority > 0 && self.state != Final + end + protected def fill_field_defaults @@ -279,6 +341,10 @@ class ContainerRequest < ArvadosModel return false end if state_changed? and state == Committed and container_uuid.nil? + if self.command.length > 0 and self.command[0] == "arvados-cwl-runner" + # Special case, arvados-cwl-runner processes are always considered "supervisors" + self.scheduling_parameters['supervisor'] = true + end while true c = Container.resolve(self) c.lock! @@ -296,10 +362,11 @@ class ContainerRequest < ArvadosModel self.container_count += 1 return if self.container_uuid_was.nil? - old_container = Container.find_by_uuid(self.container_uuid_was) - return if old_container.nil? + old_container_uuid = self.container_uuid_was + old_container_log = Container.where(uuid: old_container_uuid).pluck(:log).first + return if old_container_log.nil? - old_logs = Collection.where(portable_data_hash: old_container.log).first + old_logs = Collection.where(portable_data_hash: old_container_log).first return if old_logs.nil? log_coll = self.log_uuid.nil? ? nil : Collection.where(uuid: self.log_uuid).first @@ -314,7 +381,7 @@ class ContainerRequest < ArvadosModel # copy logs from old container into CR's log collection src = Arv::Collection.new(old_logs.manifest_text) dst = Arv::Collection.new(log_coll.manifest_text) - dst.cp_r("./", "log for container #{old_container.uuid}", src) + dst.cp_r("./", "log for container #{old_container_uuid}", src) manifest = dst.manifest_text log_coll.assign_attributes( @@ -403,8 +470,9 @@ class ContainerRequest < ArvadosModel def validate_scheduling_parameters if self.state == Committed - if scheduling_parameters.include? 'partitions' and - (!scheduling_parameters['partitions'].is_a?(Array) || + if scheduling_parameters.include?('partitions') and + !scheduling_parameters['partitions'].nil? and + (!scheduling_parameters['partitions'].is_a?(Array) || scheduling_parameters['partitions'].reject{|x| !x.is_a?(String)}.size != scheduling_parameters['partitions'].size) errors.add :scheduling_parameters, "partitions must be an array of strings" @@ -446,7 +514,7 @@ class ContainerRequest < ArvadosModel case self.state when Committed - permitted.push :priority, :container_count_max, :container_uuid + permitted.push :priority, :container_count_max, :container_uuid, :cumulative_cost if self.priority.nil? self.errors.add :priority, "cannot be nil" @@ -463,7 +531,7 @@ class ContainerRequest < ArvadosModel when Final if self.state_was == Committed # "Cancel" means setting priority=0, state=Committed - permitted.push :priority + permitted.push :priority, :cumulative_cost if current_user.andand.is_admin permitted.push :output_uuid, :log_uuid @@ -505,15 +573,8 @@ class ContainerRequest < ArvadosModel def update_priority return unless saved_change_to_state? || saved_change_to_priority? || saved_change_to_container_uuid? - act_as_system_user do - Container. - where('uuid in (?)', [container_uuid_before_last_save, self.container_uuid].compact). - map(&:update_priority!) - end - end - - def set_priority_zero - self.update_attributes!(priority: 0) if self.state != Final + update_priorities container_uuid_before_last_save if !container_uuid_before_last_save.nil? and container_uuid_before_last_save != self.container_uuid + update_priorities self.container_uuid if self.container_uuid end def set_requesting_container_uuid