X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/d6e4e65cf3aae701c3e02a814329757e869a1d6d..2a17d214467d5302e97008618ef5f560ff1fd45b:/services/api/app/models/collection.rb diff --git a/services/api/app/models/collection.rb b/services/api/app/models/collection.rb index 8579509de7..2cebd5438e 100644 --- a/services/api/app/models/collection.rb +++ b/services/api/app/models/collection.rb @@ -1,24 +1,37 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + require 'arvados/keep' +require 'sweep_trashed_objects' +require 'trashable' class Collection < ArvadosModel + extend CurrentApiClient extend DbCurrentTime include HasUuid include KindAndEtag include CommonApiTemplate + include Trashable serialize :properties, Hash + serialize :storage_classes_desired, Array + serialize :storage_classes_confirmed, Array before_validation :default_empty_manifest + before_validation :default_storage_classes, on: :create before_validation :check_encoding before_validation :check_manifest_validity before_validation :check_signatures before_validation :strip_signatures_and_update_replication_confirmed validate :ensure_pdh_matches_manifest_text + validate :ensure_storage_classes_desired_is_not_empty + validate :ensure_storage_classes_contain_non_empty_strings + validate :versioning_metadata_updates, on: :update + validate :past_versions_cannot_be_updated, on: :update before_save :set_file_names - before_save :expires_at_not_in_past - - # Query only undeleted collections by default. - default_scope where("expires_at IS NULL or expires_at > statement_timestamp()") + before_save :set_file_count_and_total_size + around_update :manage_versioning api_accessible :user, extend: :common do |t| t.add :name @@ -26,10 +39,26 @@ class Collection < ArvadosModel t.add :properties t.add :portable_data_hash t.add :signed_manifest_text, as: :manifest_text + t.add :manifest_text, as: :unsigned_manifest_text t.add :replication_desired t.add :replication_confirmed t.add :replication_confirmed_at - t.add :expires_at + t.add :storage_classes_desired + t.add :storage_classes_confirmed + t.add :storage_classes_confirmed_at + t.add :delete_at + t.add :trash_at + t.add :is_trashed + t.add :version + t.add :current_version_uuid + t.add :preserve_version + t.add :file_count + t.add :file_size_total + end + + after_initialize do + @signatures_checked = false + @computed_pdh_for_manifest_text = false end def self.attributes_required_columns @@ -40,9 +69,10 @@ class Collection < ArvadosModel # API response, and never let clients select the # manifest_text column. # - # We need expires_at to determine the correct - # timestamp in signed_manifest_text. - 'manifest_text' => ['manifest_text', 'expires_at'], + # We need trash_at and is_trashed to determine the + # correct timestamp in signed_manifest_text. + 'manifest_text' => ['manifest_text', 'trash_at', 'is_trashed'], + 'unsigned_manifest_text' => ['manifest_text'], ) end @@ -50,6 +80,10 @@ class Collection < ArvadosModel super + ["updated_at", "file_names"] end + def self.limit_index_columns_read + ["manifest_text"] + end + FILE_TOKEN = /^[[:digit:]]+:[[:digit:]]+:/ def check_signatures return false if self.manifest_text.nil? @@ -61,16 +95,18 @@ class Collection < ArvadosModel # subsequent passes without checking any signatures. This is # important because the signatures have probably been stripped off # by the time we get to a second validation pass! - return true if @signatures_checked and @signatures_checked == computed_pdh + if @signatures_checked && @signatures_checked == computed_pdh + return true + end if self.manifest_text_changed? # Check permissions on the collection manifest. # If any signature cannot be verified, raise PermissionDeniedError # which will return 403 Permission denied to the client. - api_token = current_api_client_authorization.andand.api_token + api_token = Thread.current[:token] signing_opts = { api_token: api_token, - now: db_current_time.to_i, + now: @validation_timestamp.to_i, } self.manifest_text.each_line do |entry| entry.split.each do |tok| @@ -146,7 +182,7 @@ class Collection < ArvadosModel false elsif portable_data_hash[0..31] != computed_pdh[0..31] errors.add(:portable_data_hash, - "does not match computed hash #{computed_pdh}") + "'#{portable_data_hash}' does not match computed hash '#{computed_pdh}'") false else # Ignore the client-provided size part: always store @@ -162,27 +198,133 @@ class Collection < ArvadosModel true end + def set_file_count_and_total_size + if self.manifest_text_changed? + m = Keep::Manifest.new(self.manifest_text) + self.file_size_total = m.files_size + self.file_count = m.files_count + end + true + end + def manifest_files + return '' if !self.manifest_text + + done = {} names = '' - if self.manifest_text - self.manifest_text.scan(/ \d+:\d+:(\S+)/) do |name| - names << name.first.gsub('\040',' ') + "\n" - break if names.length > 2**12 + self.manifest_text.scan(/ \d+:\d+:(\S+)/) do |name| + next if done[name] + done[name] = true + names << name.first.gsub('\040',' ') + "\n" + end + self.manifest_text.scan(/^\.\/(\S+)/m) do |stream_name| + next if done[stream_name] + done[stream_name] = true + names << stream_name.first.gsub('\040',' ') + "\n" + end + names + end + + def default_empty_manifest + self.manifest_text ||= '' + end + + def skip_uuid_existence_check + # Avoid checking the existence of current_version_uuid, as it's + # assigned on creation of a new 'current version' collection, so + # the collection's UUID only lives on memory when the validation check + # is performed. + ['current_version_uuid'] + end + + def manage_versioning + should_preserve_version = should_preserve_version? # Time sensitive, cache value + return(yield) unless (should_preserve_version || syncable_updates.any?) + + # Put aside the changes because with_lock forces a record reload + changes = self.changes + snapshot = nil + with_lock do + # Copy the original state to save it as old version + if should_preserve_version + snapshot = self.dup + snapshot.uuid = nil # Reset UUID so it's created as a new record + snapshot.created_at = self.created_at + end + + # Restore requested changes on the current version + changes.keys.each do |attr| + if attr == 'preserve_version' && changes[attr].last == false + next # Ignore false assignment, once true it'll be true until next version + end + self.attributes = {attr => changes[attr].last} + if attr == 'uuid' + # Also update the current version reference + self.attributes = {'current_version_uuid' => changes[attr].last} + end + end + + if should_preserve_version + self.version += 1 + self.preserve_version = false + end + + yield + + sync_past_versions if syncable_updates.any? + if snapshot + snapshot.attributes = self.syncable_updates + snapshot.manifest_text = snapshot.signed_manifest_text + snapshot.save end end + end - if self.manifest_text and names.length < 2**12 - self.manifest_text.scan(/^\.\/(\S+)/m) do |stream_name| - names << stream_name.first.gsub('\040',' ') + "\n" - break if names.length > 2**12 + def syncable_updates + updates = {} + (syncable_attrs & self.changes.keys).each do |attr| + if attr == 'uuid' + # Point old versions to current version's new UUID + updates['current_version_uuid'] = self.changes[attr].last + else + updates[attr] = self.changes[attr].last + end + end + return updates + end + + def sync_past_versions + updates = self.syncable_updates + Collection.where('current_version_uuid = ? AND uuid != ?', self.uuid_was, self.uuid_was).each do |c| + c.attributes = updates + # Use a different validation context to skip the 'old_versions_cannot_be_updated' + # validator, as on this case it is legal to update some fields. + leave_modified_by_user_alone do + leave_modified_at_alone do + c.save(context: :update_old_versions) + end end end + end - names[0,2**12] + def versionable_updates?(attrs) + (['manifest_text', 'description', 'properties', 'name'] & attrs).any? end - def default_empty_manifest - self.manifest_text ||= '' + def syncable_attrs + ['uuid', 'owner_uuid', 'delete_at', 'trash_at', 'is_trashed', 'replication_desired', 'storage_classes_desired'] + end + + def should_preserve_version? + return false unless (Rails.configuration.collection_versioning && versionable_updates?(self.changes.keys)) + + idle_threshold = Rails.configuration.preserve_version_if_idle + if !self.preserve_version_was && + (idle_threshold < 0 || + (idle_threshold > 0 && self.modified_at_was > db_current_time-idle_threshold.seconds)) + return false + end + return true end def check_encoding @@ -197,7 +339,7 @@ class Collection < ArvadosModel utf8 = manifest_text utf8.force_encoding Encoding::UTF_8 if utf8.valid_encoding? and utf8 == manifest_text.encode(Encoding::UTF_8) - manifest_text = utf8 + self.manifest_text = utf8 return true end rescue @@ -218,11 +360,15 @@ class Collection < ArvadosModel end def signed_manifest_text - if has_attribute? :manifest_text - token = current_api_client_authorization.andand.api_token + if !has_attribute? :manifest_text + return nil + elsif is_trashed + return manifest_text + else + token = Thread.current[:token] exp = [db_current_time.to_i + Rails.configuration.blob_signature_ttl, - expires_at].compact.map(&:to_i).min - @signed_manifest_text = self.class.sign_manifest manifest_text, token, exp + trash_at].compact.map(&:to_i).min + self.class.sign_manifest manifest_text, token, exp end end @@ -283,10 +429,10 @@ class Collection < ArvadosModel hash_part = nil size_part = nil uuid.split('+').each do |token| - if token.match /^[0-9a-f]{32,}$/ + if token.match(/^[0-9a-f]{32,}$/) raise "uuid #{uuid} has multiple hash parts" if hash_part hash_part = token - elsif token.match /^\d+$/ + elsif token.match(/^\d+$/) raise "uuid #{uuid} has multiple size parts" if size_part size_part = token end @@ -295,8 +441,53 @@ class Collection < ArvadosModel [hash_part, size_part].compact.join '+' end - # Return array of Collection objects - def self.find_all_for_docker_image(search_term, search_tag=nil, readers=nil) + def self.get_compatible_images(readers, pattern, collections) + if collections.empty? + return [] + end + + migrations = Hash[ + Link.where('tail_uuid in (?) AND link_class=? AND links.owner_uuid=?', + collections.map(&:portable_data_hash), + 'docker_image_migration', + system_user_uuid). + order('links.created_at asc'). + map { |l| + [l.tail_uuid, l.head_uuid] + }] + + migrated_collections = Hash[ + Collection.readable_by(*readers). + where('portable_data_hash in (?)', migrations.values). + map { |c| + [c.portable_data_hash, c] + }] + + collections.map { |c| + # Check if the listed image is compatible first, if not, then try the + # migration link. + manifest = Keep::Manifest.new(c.manifest_text) + if manifest.exact_file_count?(1) and manifest.files[0][1] =~ pattern + c + elsif m = migrated_collections[migrations[c.portable_data_hash]] + manifest = Keep::Manifest.new(m.manifest_text) + if manifest.exact_file_count?(1) and manifest.files[0][1] =~ pattern + m + end + end + }.compact + end + + # Resolve a Docker repo+tag, hash, or collection PDH to an array of + # Collection objects, sorted by timestamp starting with the most recent + # match. + # + # If filter_compatible_format is true (the default), only return image + # collections which are support by the installation as indicated by + # Rails.configuration.docker_image_formats. Will follow + # 'docker_image_migration' links if search_term resolves to an incompatible + # image, but an equivalent compatible image is available. + def self.find_all_for_docker_image(search_term, search_tag=nil, readers=nil, filter_compatible_format: true) readers ||= [Thread.current[:user]] base_search = Link. readable_by(*readers). @@ -304,19 +495,29 @@ class Collection < ArvadosModel joins("JOIN collections ON links.head_uuid = collections.uuid"). order("links.created_at DESC") + if (Rails.configuration.docker_image_formats.include? 'v1' and + Rails.configuration.docker_image_formats.include? 'v2') or filter_compatible_format == false + pattern = /^(sha256:)?[0-9A-Fa-f]{64}\.tar$/ + elsif Rails.configuration.docker_image_formats.include? 'v2' + pattern = /^(sha256:)[0-9A-Fa-f]{64}\.tar$/ + elsif Rails.configuration.docker_image_formats.include? 'v1' + pattern = /^[0-9A-Fa-f]{64}\.tar$/ + else + raise "Unrecognized configuration for docker_image_formats #{Rails.configuration.docker_image_formats}" + end + # If the search term is a Collection locator that contains one file # that looks like a Docker image, return it. if loc = Keep::Locator.parse(search_term) loc.strip_hints! - coll_match = readable_by(*readers).where(portable_data_hash: loc.to_s).limit(1).first - if coll_match - # Check if the Collection contains exactly one file whose name - # looks like a saved Docker image. - manifest = Keep::Manifest.new(coll_match.manifest_text) - if manifest.exact_file_count?(1) and - (manifest.files[0][1] =~ /^(sha256:)?[0-9A-Fa-f]{64}\.tar$/) - return [coll_match] - end + coll_match = readable_by(*readers).where(portable_data_hash: loc.to_s).limit(1) + if coll_match.any? or Rails.configuration.remote_hosts.length == 0 + return get_compatible_images(readers, pattern, coll_match) + else + # Allow bare pdh that doesn't exist in the local database so + # that federated container requests which refer to remotely + # stored containers will validate. + return [Collection.new(portable_data_hash: loc.to_s)] end end @@ -341,11 +542,19 @@ class Collection < ArvadosModel # so that anything with an image timestamp is considered more recent than # anything without; then we use the link's created_at as a tiebreaker. uuid_timestamps = {} - matches.all.map do |link| + matches.each do |link| uuid_timestamps[link.head_uuid] = [(-link.properties["image_timestamp"].to_datetime.to_i rescue 0), -link.created_at.to_i] + end + + sorted = Collection.where('uuid in (?)', uuid_timestamps.keys).sort_by { |c| + uuid_timestamps[c.uuid] + } + compatible = get_compatible_images(readers, pattern, sorted) + if sorted.length > 0 and compatible.empty? + raise ArvadosModel::UnresolvableContainerError.new "Matching Docker image is incompatible with 'docker_image_formats' configuration." end - Collection.where('uuid in (?)', uuid_timestamps.keys).sort_by { |c| uuid_timestamps[c.uuid] } + compatible end def self.for_latest_docker_image(search_term, search_tag=nil, readers=nil) @@ -357,10 +566,26 @@ class Collection < ArvadosModel end def self.full_text_searchable_columns - super - ["manifest_text"] + super - ["manifest_text", "storage_classes_desired", "storage_classes_confirmed", "current_version_uuid"] + end + + def self.where *args + SweepTrashedObjects.sweep_if_stale + super end protected + + # Although the defaults for these columns is already set up on the schema, + # collection creation from an API client seems to ignore them, making the + # validation on empty desired storage classes return an error. + def default_storage_classes + if self.storage_classes_desired.nil? || self.storage_classes_desired.empty? + self.storage_classes_desired = ["default"] + end + self.storage_classes_confirmed ||= [] + end + def portable_manifest_text self.class.munge_manifest_locators(manifest_text) do |match| if match[2] # size @@ -388,21 +613,58 @@ class Collection < ArvadosModel end def ensure_permission_to_save - if (not current_user.andand.is_admin and - (replication_confirmed_at_changed? or replication_confirmed_changed?) and - not (replication_confirmed_at.nil? and replication_confirmed.nil?)) - raise ArvadosModel::PermissionDeniedError.new("replication_confirmed and replication_confirmed_at attributes cannot be changed, except by setting both to nil") + if (not current_user.andand.is_admin) + if (replication_confirmed_at_changed? or replication_confirmed_changed?) and + not (replication_confirmed_at.nil? and replication_confirmed.nil?) + raise ArvadosModel::PermissionDeniedError.new("replication_confirmed and replication_confirmed_at attributes cannot be changed, except by setting both to nil") + end + if (storage_classes_confirmed_changed? or storage_classes_confirmed_at_changed?) and + not (storage_classes_confirmed == [] and storage_classes_confirmed_at.nil?) + raise ArvadosModel::PermissionDeniedError.new("storage_classes_confirmed and storage_classes_confirmed_at attributes cannot be changed, except by setting them to [] and nil respectively") + end end super end - # If expires_at is being changed to a time in the past, change it to - # now. This allows clients to say "expires {client-current-time}" - # without failing due to clock skew, while avoiding odd log entries - # like "expiry date changed to {1 year ago}". - def expires_at_not_in_past - if expires_at_changed? and expires_at - self.expires_at = [db_current_time, expires_at].max + def ensure_storage_classes_desired_is_not_empty + if self.storage_classes_desired.empty? + raise ArvadosModel::InvalidStateTransitionError.new("storage_classes_desired shouldn't be empty") end end + + def ensure_storage_classes_contain_non_empty_strings + (self.storage_classes_desired + self.storage_classes_confirmed).each do |c| + if !c.is_a?(String) || c == '' + raise ArvadosModel::InvalidStateTransitionError.new("storage classes should only be non-empty strings") + end + end + end + + def past_versions_cannot_be_updated + # We check for the '_was' values just in case the update operation + # includes a change on current_version_uuid or uuid. + if current_version_uuid_was != uuid_was + errors.add(:base, "past versions cannot be updated") + false + end + end + + def versioning_metadata_updates + valid = true + if (current_version_uuid_was == uuid_was) && current_version_uuid_changed? + errors.add(:current_version_uuid, "cannot be updated") + valid = false + end + if version_changed? + errors.add(:version, "cannot be updated") + valid = false + end + valid + end + + def assign_uuid + super + self.current_version_uuid ||= self.uuid + true + end end