require 'arvados/keep'
class Collection < ArvadosModel
+ extend DbCurrentTime
include HasUuid
include KindAndEtag
include CommonApiTemplate
+ serialize :properties, Hash
+
before_validation :check_encoding
before_validation :check_signatures
- before_validation :strip_manifest_text
+ before_validation :strip_manifest_text_and_clear_replication_confirmed
before_validation :set_portable_data_hash
validate :ensure_hash_matches_manifest_text
before_save :set_file_names
t.add :properties
t.add :portable_data_hash
t.add :signed_manifest_text, as: :manifest_text
+ t.add :replication_desired
+ t.add :replication_confirmed
+ t.add :replication_confirmed_at
end
+ LOCATOR_REGEXP = /^([[:xdigit:]]{32})(\+([[:digit:]]+))?(\+([[:upper:]][[:alnum:]+@_-]*))?$/
+
def self.attributes_required_columns
- # If we don't list this explicitly, the params[:select] code gets
- # confused by the way we expose signed_manifest_text as
- # manifest_text in the API response, and never let clients select
- # the manifest_text column.
- super.merge('manifest_text' => ['manifest_text'])
+ super.merge(
+ # If we don't list manifest_text explicitly, the
+ # params[:select] code gets confused by the way we
+ # expose signed_manifest_text as manifest_text in the
+ # API response, and never let clients select the
+ # manifest_text column.
+ 'manifest_text' => ['manifest_text'],
+ )
end
def check_signatures
# subsequent passes without checking any signatures. This is
# important because the signatures have probably been stripped off
# by the time we get to a second validation pass!
- return true if @signatures_checked and @signatures_checked == compute_pdh
+ computed_pdh = compute_pdh
+ return true if @signatures_checked and @signatures_checked == computed_pdh
if self.manifest_text_changed?
# Check permissions on the collection manifest.
# which will return 403 Permission denied to the client.
api_token = current_api_client_authorization.andand.api_token
signing_opts = {
- key: Rails.configuration.blob_signing_key,
api_token: api_token,
- ttl: Rails.configuration.blob_signing_ttl,
+ now: db_current_time.to_i,
}
self.manifest_text.lines.each do |entry|
entry.split[1..-1].each do |tok|
end
end
end
- @signatures_checked = compute_pdh
+ @signatures_checked = computed_pdh
end
- def strip_manifest_text
+ def strip_manifest_text_and_clear_replication_confirmed
if self.manifest_text_changed?
+ in_old_manifest = {}
+ self.class.munge_manifest_locators!(manifest_text_was) do |match|
+ in_old_manifest[match[1]] = true
+ end
+
+ cleared_replication_confirmed = false
+
# Remove any permission signatures from the manifest.
- self.class.munge_manifest_locators!(self[:manifest_text]) do |loc|
- loc.without_signature.to_s
+ self[:manifest_text] = self.class.munge_manifest_locators!(self[:manifest_text]) do |match|
+ if not cleared_replication_confirmed and not in_old_manifest[match[1]]
+ self.replication_confirmed_at = nil
+ self.replication_confirmed = nil
+ cleared_replication_confirmed = true
+ end
+ self.class.locator_without_signature(match)
end
end
true
end
+ def self.locator_without_signature match
+ without_signature = match[1]
+ without_signature += match[2] if match[2]
+ if match[4]
+ hints = match[4].split('+').reject { |hint| hint.start_with?("A") }
+ without_signature += hints.join('+')
+ end
+ without_signature
+ end
+
def set_portable_data_hash
if (portable_data_hash.nil? or
portable_data_hash == "" or
def set_file_names
if self.manifest_text_changed?
- self.file_names = Collection.manifest_files self.manifest_text
+ self.file_names = manifest_files
end
true
end
- def self.manifest_files manifest_text
+ def manifest_files
names = ''
- if manifest_text
- manifest_text.scan(/ \d+:\d+:(\S+)/) do |name|
- names << name.first.gsub('\040',' ') + "\n"
- break if names.length > 2**13
+ if self.manifest_text
+ self.manifest_text.scan(/ \d+:\d+:(\S+)/) do |name|
+ names << name.first.gsub('\040',' ') + "\n"
+ break if names.length > 2**12
end
end
- names[0,2**13]
+
+ if self.manifest_text and names.length < 2**12
+ self.manifest_text.scan(/^\.\/(\S+)/m) do |stream_name|
+ names << stream_name.first.gsub('\040',' ') + "\n"
+ break if names.length > 2**12
+ end
+ end
+
+ names[0,2**12]
end
def check_encoding
end
end
- def redundancy_status
- if redundancy_confirmed_as.nil?
- 'unconfirmed'
- elsif redundancy_confirmed_as < redundancy
- 'degraded'
- else
- if redundancy_confirmed_at.nil?
- 'unconfirmed'
- elsif Time.now - redundancy_confirmed_at < 7.days
- 'OK'
- else
- 'stale'
- end
- end
- end
-
def signed_manifest_text
if has_attribute? :manifest_text
token = current_api_client_authorization.andand.api_token
def self.sign_manifest manifest, token
signing_opts = {
- key: Rails.configuration.blob_signing_key,
api_token: token,
- ttl: Rails.configuration.blob_signing_ttl,
+ expire: db_current_time.to_i + Rails.configuration.blob_signature_ttl,
}
m = manifest.dup
- munge_manifest_locators!(m) do |loc|
- Blob.sign_locator(loc.to_s, signing_opts)
+ m = munge_manifest_locators!(m) do |match|
+ Blob.sign_locator(locator_without_signature(match), signing_opts)
end
return m
end
def self.munge_manifest_locators! manifest
# Given a manifest text and a block, yield each locator,
# and replace it with whatever the block returns.
- manifest.andand.gsub!(/ [[:xdigit:]]{32}(\+[[:digit:]]+)?(\+\S+)/) do |word|
- if loc = Keep::Locator.parse(word.strip)
- " " + yield(loc)
- else
- " " + word
+ new_lines = []
+ lines = manifest.andand.split("\n")
+ lines.andand.each do |line|
+ words = line.split(' ')
+ new_words = []
+ words.each do |word|
+ if match = LOCATOR_REGEXP.match(word.strip)
+ new_words << yield(match)
+ else
+ new_words << word.strip
+ end
end
+ new_lines << new_words.join(' ')
+ end
+
+ if !new_lines.empty?
+ ends_with_newline = manifest.end_with?("\n")
+ manifest = new_lines.join("\n")
+ manifest += "\n" if ends_with_newline
end
+
+ manifest
end
def self.normalize_uuid uuid
super - ["manifest_text"]
end
+ def self.full_text_searchable_columns
+ super - ["manifest_text"]
+ end
+
protected
def portable_manifest_text
portable_manifest = self[:manifest_text].dup
- self.class.munge_manifest_locators!(portable_manifest) do |loc|
- loc.hash + '+' + loc.size.to_s
+ portable_manifest = self.class.munge_manifest_locators!(portable_manifest) do |match|
+ if match[2] # size
+ match[1] + match[2]
+ else
+ match[1]
+ end
end
portable_manifest
end
def compute_pdh
+ return @computed_pdh if @computed_pdh
portable_manifest = portable_manifest_text
- (Digest::MD5.hexdigest(portable_manifest) +
- '+' +
- portable_manifest.bytesize.to_s)
+ @computed_pdh = (Digest::MD5.hexdigest(portable_manifest) +
+ '+' +
+ portable_manifest.bytesize.to_s)
+ @computed_pdh
+ end
+
+ def ensure_permission_to_save
+ if (not current_user.andand.is_admin and
+ (replication_confirmed_at_changed? or replication_confirmed_changed?) and
+ not (replication_confirmed_at.nil? and replication_confirmed.nil?))
+ raise ArvadosModel::PermissionDeniedError.new("replication_confirmed and replication_confirmed_at attributes cannot be changed, except by setting both to nil")
+ end
+ super
end
end
projects / arvados.git / blobdiff