require 'arvados/keep'
class Collection < ArvadosModel
+ extend DbCurrentTime
include HasUuid
include KindAndEtag
include CommonApiTemplate
+ serialize :properties, Hash
+
before_validation :check_encoding
before_validation :check_signatures
- before_validation :strip_manifest_text
+ before_validation :strip_manifest_text_and_clear_replication_confirmed
before_validation :set_portable_data_hash
- before_validation :maybe_clear_redundancy_confirmed
validate :ensure_hash_matches_manifest_text
before_save :set_file_names
t.add :replication_confirmed_at
end
+ LOCATOR_REGEXP = /^([[:xdigit:]]{32})(\+([[:digit:]]+))?(\+([[:upper:]][[:alnum:]+@_-]*))?$/
+
def self.attributes_required_columns
super.merge(
# If we don't list manifest_text explicitly, the
# subsequent passes without checking any signatures. This is
# important because the signatures have probably been stripped off
# by the time we get to a second validation pass!
- return true if @signatures_checked and @signatures_checked == compute_pdh
+ computed_pdh = compute_pdh
+ return true if @signatures_checked and @signatures_checked == computed_pdh
if self.manifest_text_changed?
# Check permissions on the collection manifest.
# which will return 403 Permission denied to the client.
api_token = current_api_client_authorization.andand.api_token
signing_opts = {
- key: Rails.configuration.blob_signing_key,
api_token: api_token,
- ttl: Rails.configuration.blob_signing_ttl,
+ now: db_current_time.to_i,
}
self.manifest_text.lines.each do |entry|
entry.split[1..-1].each do |tok|
end
end
end
- @signatures_checked = compute_pdh
+ @signatures_checked = computed_pdh
end
- def strip_manifest_text
+ def strip_manifest_text_and_clear_replication_confirmed
if self.manifest_text_changed?
+ in_old_manifest = {}
+ self.class.munge_manifest_locators!(manifest_text_was) do |match|
+ in_old_manifest[match[1]] = true
+ end
+
+ cleared_replication_confirmed = false
+
# Remove any permission signatures from the manifest.
- self.class.munge_manifest_locators!(self[:manifest_text]) do |loc|
- loc.without_signature.to_s
+ self[:manifest_text] = self.class.munge_manifest_locators!(self[:manifest_text]) do |match|
+ if not cleared_replication_confirmed and not in_old_manifest[match[1]]
+ self.replication_confirmed_at = nil
+ self.replication_confirmed = nil
+ cleared_replication_confirmed = true
+ end
+ self.class.locator_without_signature(match)
end
end
true
end
+ def self.locator_without_signature match
+ without_signature = match[1]
+ without_signature += match[2] if match[2]
+ if match[4]
+ hints = match[4].split('+').reject { |hint| hint.start_with?("A") }
+ without_signature += hints.join('+')
+ end
+ without_signature
+ end
+
def set_portable_data_hash
if (portable_data_hash.nil? or
portable_data_hash == "" or
def self.sign_manifest manifest, token
signing_opts = {
- key: Rails.configuration.blob_signing_key,
api_token: token,
- ttl: Rails.configuration.blob_signing_ttl,
+ expire: db_current_time.to_i + Rails.configuration.blob_signature_ttl,
}
m = manifest.dup
- munge_manifest_locators!(m) do |loc|
- Blob.sign_locator(loc.to_s, signing_opts)
+ m = munge_manifest_locators!(m) do |match|
+ Blob.sign_locator(locator_without_signature(match), signing_opts)
end
return m
end
def self.munge_manifest_locators! manifest
# Given a manifest text and a block, yield each locator,
# and replace it with whatever the block returns.
- manifest.andand.gsub!(/ [[:xdigit:]]{32}(\+[[:digit:]]+)?(\+\S+)/) do |word|
- if loc = Keep::Locator.parse(word.strip)
- " " + yield(loc)
- else
- " " + word
+ new_lines = []
+ lines = manifest.andand.split("\n")
+ lines.andand.each do |line|
+ words = line.split(' ')
+ new_words = []
+ words.each do |word|
+ if match = LOCATOR_REGEXP.match(word.strip)
+ new_words << yield(match)
+ else
+ new_words << word.strip
+ end
end
+ new_lines << new_words.join(' ')
end
- end
- def self.each_manifest_locator manifest
- # Given a manifest text and a block, yield each locator.
- manifest.andand.scan(/ ([[:xdigit:]]{32}(\+[[:digit:]]+)?(\+\S+))/) do |word|
- if loc = Keep::Locator.parse(word[1])
- yield loc
- end
+ if !new_lines.empty?
+ ends_with_newline = manifest.end_with?("\n")
+ manifest = new_lines.join("\n")
+ manifest += "\n" if ends_with_newline
end
+
+ manifest
end
def self.normalize_uuid uuid
protected
def portable_manifest_text
portable_manifest = self[:manifest_text].dup
- self.class.munge_manifest_locators!(portable_manifest) do |loc|
- loc.hash + '+' + loc.size.to_s
+ portable_manifest = self.class.munge_manifest_locators!(portable_manifest) do |match|
+ if match[2] # size
+ match[1] + match[2]
+ else
+ match[1]
+ end
end
portable_manifest
end
def compute_pdh
+ return @computed_pdh if @computed_pdh
portable_manifest = portable_manifest_text
- (Digest::MD5.hexdigest(portable_manifest) +
- '+' +
- portable_manifest.bytesize.to_s)
- end
-
- def maybe_clear_redundancy_confirmed
- if manifest_text_changed?
- # If the new manifest_text contains locators whose hashes
- # weren't in the old manifest_text, storage replication is no
- # longer confirmed.
- in_old_manifest = {}
- self.class.each_manifest_locator(manifest_text_was) do |loc|
- in_old_manifest[loc.hash] = true
- end
- self.class.each_manifest_locator(manifest_text) do |loc|
- if not in_old_manifest[loc.hash]
- replication_confirmed_at = nil
- replication_confirmed = nil
- break
- end
- end
- end
+ @computed_pdh = (Digest::MD5.hexdigest(portable_manifest) +
+ '+' +
+ portable_manifest.bytesize.to_s)
+ @computed_pdh
end
def ensure_permission_to_save
projects / arvados.git / blobdiff