X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/5aacaabf6190d644bc957903f0f712959234fa07..67f0d86c20139eee996816d44ef75fa52288c515:/services/api/app/models/collection.rb diff --git a/services/api/app/models/collection.rb b/services/api/app/models/collection.rb index d112e09e97..ccfb35e496 100644 --- a/services/api/app/models/collection.rb +++ b/services/api/app/models/collection.rb @@ -5,10 +5,13 @@ class Collection < ArvadosModel include KindAndEtag include CommonApiTemplate + serialize :properties, Hash + before_validation :check_encoding before_validation :check_signatures before_validation :strip_manifest_text before_validation :set_portable_data_hash + before_validation :maybe_clear_replication_confirmed validate :ensure_hash_matches_manifest_text before_save :set_file_names @@ -21,14 +24,20 @@ class Collection < ArvadosModel t.add :properties t.add :portable_data_hash t.add :signed_manifest_text, as: :manifest_text + t.add :replication_desired + t.add :replication_confirmed + t.add :replication_confirmed_at end def self.attributes_required_columns - # If we don't list this explicitly, the params[:select] code gets - # confused by the way we expose signed_manifest_text as - # manifest_text in the API response, and never let clients select - # the manifest_text column. - super.merge('manifest_text' => ['manifest_text']) + super.merge( + # If we don't list manifest_text explicitly, the + # params[:select] code gets confused by the way we + # expose signed_manifest_text as manifest_text in the + # API response, and never let clients select the + # manifest_text column. + 'manifest_text' => ['manifest_text'], + ) end def check_signatures @@ -51,7 +60,7 @@ class Collection < ArvadosModel signing_opts = { key: Rails.configuration.blob_signing_key, api_token: api_token, - ttl: Rails.configuration.blob_signing_ttl, + ttl: Rails.configuration.blob_signature_ttl, } self.manifest_text.lines.each do |entry| entry.split[1..-1].each do |tok| @@ -129,28 +138,28 @@ class Collection < ArvadosModel def set_file_names if self.manifest_text_changed? - self.file_names = Collection.manifest_files self.manifest_text + self.file_names = manifest_files end true end - def self.manifest_files manifest_text + def manifest_files names = '' - if manifest_text - manifest_text.scan(/ \d+:\d+:(\S+)/) do |name| - names << name.first.gsub('\040',' ') + "\n" - break if names.length > 2**13 + if self.manifest_text + self.manifest_text.scan(/ \d+:\d+:(\S+)/) do |name| + names << name.first.gsub('\040',' ') + "\n" + break if names.length > 2**12 end end - if manifest_text and names.length < 2**13 - manifest_text.scan(/^\.\/(\S+)/m) do |stream_name| - names << stream_name.first.gsub('\040',' ') + "\n" - break if names.length > 2**13 + if self.manifest_text and names.length < 2**12 + self.manifest_text.scan(/^\.\/(\S+)/m) do |stream_name| + names << stream_name.first.gsub('\040',' ') + "\n" + break if names.length > 2**12 end end - names[0,2**13] + names[0,2**12] end def check_encoding @@ -175,22 +184,6 @@ class Collection < ArvadosModel end end - def redundancy_status - if redundancy_confirmed_as.nil? - 'unconfirmed' - elsif redundancy_confirmed_as < redundancy - 'degraded' - else - if redundancy_confirmed_at.nil? - 'unconfirmed' - elsif Time.now - redundancy_confirmed_at < 7.days - 'OK' - else - 'stale' - end - end - end - def signed_manifest_text if has_attribute? :manifest_text token = current_api_client_authorization.andand.api_token @@ -202,7 +195,7 @@ class Collection < ArvadosModel signing_opts = { key: Rails.configuration.blob_signing_key, api_token: token, - ttl: Rails.configuration.blob_signing_ttl, + ttl: Rails.configuration.blob_signature_ttl, } m = manifest.dup munge_manifest_locators!(m) do |loc| @@ -214,7 +207,7 @@ class Collection < ArvadosModel def self.munge_manifest_locators! manifest # Given a manifest text and a block, yield each locator, # and replace it with whatever the block returns. - manifest.andand.gsub!(/ [[:xdigit:]]{32}(\+[[:digit:]]+)?(\+\S+)/) do |word| + manifest.andand.gsub!(/ [[:xdigit:]]{32}(\+\S+)?/) do |word| if loc = Keep::Locator.parse(word.strip) " " + yield(loc) else @@ -223,6 +216,15 @@ class Collection < ArvadosModel end end + def self.each_manifest_locator manifest + # Given a manifest text and a block, yield each locator. + manifest.andand.scan(/ ([[:xdigit:]]{32}(\+\S+)?)/) do |word, _| + if loc = Keep::Locator.parse(word) + yield loc + end + end + end + def self.normalize_uuid uuid hash_part = nil size_part = nil @@ -300,11 +302,19 @@ class Collection < ArvadosModel super - ["manifest_text"] end + def self.full_text_searchable_columns + super - ["manifest_text"] + end + protected def portable_manifest_text portable_manifest = self[:manifest_text].dup self.class.munge_manifest_locators!(portable_manifest) do |loc| - loc.hash + '+' + loc.size.to_s + if loc.size + loc.hash + '+' + loc.size.to_s + else + loc.hash + end end portable_manifest end @@ -315,4 +325,32 @@ class Collection < ArvadosModel '+' + portable_manifest.bytesize.to_s) end + + def maybe_clear_replication_confirmed + if manifest_text_changed? + # If the new manifest_text contains locators whose hashes + # weren't in the old manifest_text, storage replication is no + # longer confirmed. + in_old_manifest = {} + self.class.each_manifest_locator(manifest_text_was) do |loc| + in_old_manifest[loc.hash] = true + end + self.class.each_manifest_locator(manifest_text) do |loc| + if not in_old_manifest[loc.hash] + self.replication_confirmed_at = nil + self.replication_confirmed = nil + break + end + end + end + end + + def ensure_permission_to_save + if (not current_user.andand.is_admin and + (replication_confirmed_at_changed? or replication_confirmed_changed?) and + not (replication_confirmed_at.nil? and replication_confirmed.nil?)) + raise ArvadosModel::PermissionDeniedError.new("replication_confirmed and replication_confirmed_at attributes cannot be changed, except by setting both to nil") + end + super + end end