1 # Copyright (C) The Arvados Authors. All rights reserved.
3 # SPDX-License-Identifier: AGPL-3.0
5 require 'arvados_model_updates'
7 require 'record_filters'
9 require 'request_error'
11 class ArvadosModel < ApplicationRecord
12 self.abstract_class = true
14 include ArvadosModelUpdates
15 include CurrentApiClient # current_user, current_api_client, etc.
19 after_find :schedule_restoring_changes
20 after_initialize :log_start_state
21 before_save :ensure_permission_to_save
22 before_save :ensure_owner_uuid_is_permitted
23 before_save :ensure_ownership_path_leads_to_user
24 before_destroy :ensure_owner_uuid_is_permitted
25 before_destroy :ensure_permission_to_destroy
26 before_create :update_modified_by_fields
27 before_update :maybe_update_modified_by_fields
28 after_create :log_create
29 after_update :log_update
30 after_destroy :log_destroy
31 before_validation :normalize_collection_uuids
32 before_validation :set_default_owner
33 validate :ensure_valid_uuids
35 # Note: This only returns permission links. It does not account for
36 # permissions obtained via user.is_admin or
37 # user.uuid==object.owner_uuid.
38 has_many(:permissions,
39 ->{where(link_class: 'permission')},
40 foreign_key: :head_uuid,
44 # If async is true at create or update, permission graph
45 # update is deferred allowing making multiple calls without the performance
47 attr_accessor :async_permissions_update
49 # Ignore listed attributes on mass assignments
50 def self.protected_attributes
54 class PermissionDeniedError < RequestError
60 class AlreadyLockedError < RequestError
66 class LockFailedError < RequestError
72 class InvalidStateTransitionError < RequestError
78 class UnauthorizedError < RequestError
84 class UnresolvableContainerError < RequestError
90 def self.kind_class(kind)
91 kind.match(/^arvados\#(.+)$/)[1].classify.safe_constantize rescue nil
95 "#{current_api_base}/#{self.class.to_s.pluralize.underscore}/#{self.uuid}"
98 def self.permit_attribute_params raw_params
99 # strong_parameters does not provide security: permissions are
100 # implemented with before_save hooks.
102 # The following permit! is necessary even with
103 # "ActionController::Parameters.permit_all_parameters = true",
104 # because permit_all does not permit nested attributes.
108 raw_params = raw_params.to_hash
109 raw_params.delete_if { |k, _| self.protected_attributes.include? k }
110 serialized_attributes.each do |colname, coder|
111 param = raw_params[colname.to_sym]
114 elsif !param.is_a?(coder.object_class)
115 raise ArgumentError.new("#{colname} parameter must be #{coder.object_class}, not #{param.class}")
116 elsif has_nonstring_keys?(param)
117 raise ArgumentError.new("#{colname} parameter cannot have non-string hash keys")
120 # Check JSONB columns that aren't listed on serialized_attributes
121 columns.select{|c| c.type == :jsonb}.collect{|j| j.name}.each do |colname|
122 if serialized_attributes.include? colname || raw_params[colname.to_sym].nil?
125 if has_nonstring_keys?(raw_params[colname.to_sym])
126 raise ArgumentError.new("#{colname} parameter cannot have non-string hash keys")
130 ActionController::Parameters.new(raw_params).permit!
133 def initialize raw_params={}, *args
134 super(self.class.permit_attribute_params(raw_params), *args)
137 # Reload "old attributes" for logging, too.
144 def self.create raw_params={}, *args
145 super(permit_attribute_params(raw_params), *args)
148 def update_attributes raw_params={}, *args
149 super(self.class.permit_attribute_params(raw_params), *args)
152 def self.selectable_attributes(template=:user)
153 # Return an array of attribute name strings that can be selected
154 # in the given template.
155 api_accessible_attributes(template).map { |attr_spec| attr_spec.first.to_s }
158 def self.searchable_columns operator
159 textonly_operator = !operator.match(/[<=>]/)
160 self.columns.select do |col|
164 when :datetime, :integer, :boolean
172 def self.attribute_column attr
173 self.columns.select { |col| col.name == attr.to_s }.first
176 def self.attributes_required_columns
177 # This method returns a hash. Each key is the name of an API attribute,
178 # and it's mapped to a list of database columns that must be fetched
179 # to generate that attribute.
180 # This implementation generates a simple map of attributes to
181 # matching column names. Subclasses can override this method
182 # to specify that method-backed API attributes need to fetch
183 # specific columns from the database.
184 all_columns = columns.map(&:name)
185 api_column_map = Hash.new { |hash, key| hash[key] = [] }
186 methods.grep(/^api_accessible_\w+$/).each do |method_name|
187 next if method_name == :api_accessible_attributes
188 send(method_name).each_pair do |api_attr_name, col_name|
189 col_name = col_name.to_s
190 if all_columns.include?(col_name)
191 api_column_map[api_attr_name.to_s] |= [col_name]
198 def self.ignored_select_attributes
199 ["href", "kind", "etag"]
202 def self.columns_for_attributes(select_attributes)
203 if select_attributes.empty?
204 raise ArgumentError.new("Attribute selection list cannot be empty")
206 api_column_map = attributes_required_columns
208 select_attributes.each do |s|
209 next if ignored_select_attributes.include? s
210 if not s.is_a? String or not api_column_map.include? s
214 if not invalid_attrs.empty?
215 raise ArgumentError.new("Invalid attribute(s): #{invalid_attrs.inspect}")
217 # Given an array of attribute names to select, return an array of column
218 # names that must be fetched from the database to satisfy the request.
219 select_attributes.flat_map { |attr| api_column_map[attr] }.uniq
222 def self.default_orders
223 ["#{table_name}.modified_at desc", "#{table_name}.uuid"]
226 def self.unique_columns
230 def self.limit_index_columns_read
231 # This method returns a list of column names.
232 # If an index request reads that column from the database,
233 # APIs that return lists will only fetch objects until reaching
234 # max_index_database_read bytes of data from those columns.
238 # If current user can manage the object, return an array of uuids of
239 # users and groups that have permission to write the object. The
240 # first two elements are always [self.owner_uuid, current user's
243 # If current user can write but not manage the object, return
244 # [self.owner_uuid, current user's uuid].
246 # If current user cannot write this object, just return
249 return [owner_uuid] if not current_user
250 unless (owner_uuid == current_user.uuid or
251 current_user.is_admin or
252 (current_user.groups_i_can(:manage) & [uuid, owner_uuid]).any?)
253 if ((current_user.groups_i_can(:write) + [current_user.uuid]) &
254 [uuid, owner_uuid]).any?
255 return [owner_uuid, current_user.uuid]
260 [owner_uuid, current_user.uuid] + permissions.collect do |p|
261 if ['can_write', 'can_manage'].index p.name
267 # Return a query with read permissions restricted to the union of the
268 # permissions of the members of users_list, i.e. if something is readable by
269 # any user in users_list, it will be readable in the query returned by this
271 def self.readable_by(*users_list)
272 # Get rid of troublesome nils
275 # Load optional keyword arguments, if they exist.
276 if users_list.last.is_a? Hash
277 kwargs = users_list.pop
282 # Collect the UUIDs of the authorized users.
283 sql_table = kwargs.fetch(:table_name, table_name)
284 include_trash = kwargs.fetch(:include_trash, false)
285 include_old_versions = kwargs.fetch(:include_old_versions, false)
288 user_uuids = users_list.map { |u| u.uuid }
290 # For details on how the trashed_groups table is constructed, see
291 # see db/migrate/20200501150153_permission_table.rb
293 exclude_trashed_records = ""
294 if !include_trash and (sql_table == "groups" or sql_table == "collections") then
295 # Only include records that are not trashed
296 exclude_trashed_records = "AND (#{sql_table}.trash_at is NULL or #{sql_table}.trash_at > statement_timestamp())"
299 if users_list.select { |u| u.is_admin }.any?
300 # Admin skips most permission checks, but still want to filter on trashed items.
302 if sql_table != "api_client_authorizations"
303 # Only include records where the owner is not trashed
304 sql_conds = "#{sql_table}.owner_uuid NOT IN (SELECT group_uuid FROM #{TRASHED_GROUPS} "+
305 "where trash_at <= statement_timestamp()) #{exclude_trashed_records}"
310 if !include_trash then
311 trashed_check = "AND target_uuid NOT IN (SELECT group_uuid FROM #{TRASHED_GROUPS} where trash_at <= statement_timestamp())"
314 # The core of the permission check is a join against the
315 # materialized_permissions table to determine if the user has at
316 # least read permission to either the object itself or its
317 # direct owner (if traverse_owned is true). See
318 # db/migrate/20200501150153_permission_table.rb for details on
319 # how the permissions are computed.
321 # A user can have can_manage access to another user, this grants
322 # full access to all that user's stuff. To implement that we
323 # need to include those other users in the permission query.
324 user_uuids_subquery = USER_UUIDS_SUBQUERY_TEMPLATE % {user: ":user_uuids", perm_level: 1}
326 # Note: it is possible to combine the direct_check and
327 # owner_check into a single EXISTS() clause, however it turns
328 # out query optimizer doesn't like it and forces a sequential
329 # table scan. Constructing the query with separate EXISTS()
330 # clauses enables it to use the index.
332 # see issue 13208 for details.
334 # Match a direct read permission link from the user to the record uuid
335 direct_check = "#{sql_table}.uuid IN (SELECT target_uuid FROM #{PERMISSION_VIEW} "+
336 "WHERE user_uuid IN (#{user_uuids_subquery}) AND perm_level >= 1 #{trashed_check})"
338 # Match a read permission for the user to the record's
339 # owner_uuid. This is so we can have a permissions table that
340 # mostly consists of users and groups (projects are a type of
341 # group) and not have to compute and list user permission to
342 # every single object in the system.
344 # Don't do this for API keys (special behavior) or groups
345 # (already covered by direct_check).
347 # The traverse_owned flag indicates whether the permission to
348 # read an object also implies transitive permission to read
349 # things the object owns. The situation where this is important
350 # are determining if we can read an object owned by another
351 # user. This makes it possible to have permission to read the
352 # user record without granting permission to read things the
355 if sql_table != "api_client_authorizations" and sql_table != "groups" then
356 owner_check = "#{sql_table}.owner_uuid IN (SELECT target_uuid FROM #{PERMISSION_VIEW} "+
357 "WHERE user_uuid IN (#{user_uuids_subquery}) AND perm_level >= 1 #{trashed_check} AND traverse_owned) "
358 direct_check = " OR " + direct_check
362 if sql_table == "links"
363 # Match any permission link that gives one of the authorized
364 # users some permission _or_ gives anyone else permission to
365 # view one of the authorized users.
366 links_cond = "OR (#{sql_table}.link_class IN (:permission_link_classes) AND "+
367 "(#{sql_table}.head_uuid IN (#{user_uuids_subquery}) OR #{sql_table}.tail_uuid IN (#{user_uuids_subquery})))"
370 sql_conds = "(#{owner_check} #{direct_check} #{links_cond}) #{exclude_trashed_records}"
374 if !include_old_versions && sql_table == "collections"
375 exclude_old_versions = "#{sql_table}.uuid = #{sql_table}.current_version_uuid"
377 sql_conds = exclude_old_versions
379 sql_conds += " AND #{exclude_old_versions}"
383 self.where(sql_conds,
384 user_uuids: user_uuids,
385 permission_link_classes: ['permission', 'resources'])
388 def save_with_unique_name!
393 conn = ActiveRecord::Base.connection
394 conn.exec_query 'SAVEPOINT save_with_unique_name'
397 rescue ActiveRecord::RecordNotUnique => rn
398 raise if max_retries == 0
401 conn.exec_query 'ROLLBACK TO SAVEPOINT save_with_unique_name'
403 # Dig into the error to determine if it is specifically calling out a
404 # (owner_uuid, name) uniqueness violation. In this specific case, and
405 # the client requested a unique name with ensure_unique_name==true,
406 # update the name field and try to save again. Loop as necessary to
407 # discover a unique name. It is necessary to handle name choosing at
408 # this level (as opposed to the client) to ensure that record creation
409 # never fails due to a race condition.
411 raise unless err.is_a?(PG::UniqueViolation)
413 # Unfortunately ActiveRecord doesn't abstract out any of the
414 # necessary information to figure out if this the error is actually
415 # the specific case where we want to apply the ensure_unique_name
416 # behavior, so the following code is specialized to Postgres.
417 detail = err.result.error_field(PG::Result::PG_DIAG_MESSAGE_DETAIL)
418 raise unless /^Key \(owner_uuid, name\)=\([a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15}, .*?\) already exists\./.match detail
420 new_name = "#{name_was} (#{db_current_time.utc.iso8601(3)})"
422 # If the database is fast enough to do two attempts in the
423 # same millisecond, we need to wait to ensure we try a
424 # different timestamp on each attempt.
426 new_name = "#{name_was} (#{db_current_time.utc.iso8601(3)})"
429 self[:name] = new_name
430 if uuid_was.nil? && !uuid.nil?
432 if self.is_a? Collection
433 # Reset so that is assigned to the new UUID
434 self[:current_version_uuid] = nil
437 conn.exec_query 'SAVEPOINT save_with_unique_name'
440 conn.exec_query 'RELEASE SAVEPOINT save_with_unique_name'
446 if self.owner_uuid.nil?
447 return current_user.uuid
449 owner_class = ArvadosModel.resource_class_for_uuid(self.owner_uuid)
450 if owner_class == User
453 owner_class.find_by_uuid(self.owner_uuid).user_owner_uuid
457 def logged_attributes
458 attributes.except(*Rails.configuration.AuditLogs.UnloggedAttributes.stringify_keys.keys)
461 def self.full_text_searchable_columns
462 self.columns.select do |col|
463 [:string, :text, :jsonb].include?(col.type)
467 def self.full_text_coalesce
468 full_text_searchable_columns.collect do |column|
469 is_jsonb = self.columns.select{|x|x.name == column}[0].type == :jsonb
470 cast = (is_jsonb || serialized_attributes[column]) ? '::text' : ''
471 "coalesce(#{column}#{cast},'')"
475 def self.full_text_trgm
476 "(#{full_text_coalesce.join(" || ' ' || ")})"
479 def self.full_text_tsvector
480 parts = full_text_searchable_columns.collect do |column|
481 is_jsonb = self.columns.select{|x|x.name == column}[0].type == :jsonb
482 cast = (is_jsonb || serialized_attributes[column]) ? '::text' : ''
483 "coalesce(#{column}#{cast},'')"
485 "to_tsvector('english', substr(#{parts.join(" || ' ' || ")}, 0, 8000))"
488 def self.apply_filters query, filters
489 ft = record_filters filters, self
490 if not ft[:cond_out].any?
493 ft[:joins].each do |t|
494 query = query.joins(t)
496 query.where('(' + ft[:cond_out].join(') AND (') + ')',
502 def self.deep_sort_hash(x)
504 x.sort.collect do |k, v|
505 [k, deep_sort_hash(v)]
508 x.collect { |v| deep_sort_hash(v) }
514 def ensure_ownership_path_leads_to_user
515 if new_record? or owner_uuid_changed?
516 uuid_in_path = {owner_uuid => true, uuid => true}
518 while (owner_class = ArvadosModel::resource_class_for_uuid(x)) != User
521 # Test for cycles with the new version, not the DB contents
523 elsif !owner_class.respond_to? :find_by_uuid
524 raise ActiveRecord::RecordNotFound.new
526 x = owner_class.find_by_uuid(x).owner_uuid
528 rescue ActiveRecord::RecordNotFound => e
529 errors.add :owner_uuid, "is not owned by any user: #{e}"
534 errors.add :owner_uuid, "would create an ownership cycle"
536 errors.add :owner_uuid, "has an ownership cycle"
540 uuid_in_path[x] = true
546 def set_default_owner
547 if new_record? and current_user and respond_to? :owner_uuid=
548 self.owner_uuid ||= current_user.uuid
552 def ensure_owner_uuid_is_permitted
553 raise PermissionDeniedError if !current_user
555 if self.owner_uuid.nil?
556 errors.add :owner_uuid, "cannot be nil"
557 raise PermissionDeniedError
560 rsc_class = ArvadosModel::resource_class_for_uuid owner_uuid
561 unless rsc_class == User or rsc_class == Group
562 errors.add :owner_uuid, "must be set to User or Group"
563 raise PermissionDeniedError
566 if new_record? || owner_uuid_changed?
567 # Permission on owner_uuid_was is needed to move an existing
568 # object away from its previous owner (which implies permission
569 # to modify this object itself, so we don't need to check that
570 # separately). Permission on the new owner_uuid is also needed.
571 [['old', owner_uuid_was],
573 ].each do |which, check_uuid|
575 # old_owner_uuid is nil? New record, no need to check.
576 elsif !current_user.can?(write: check_uuid)
577 logger.warn "User #{current_user.uuid} tried to set ownership of #{self.class.to_s} #{self.uuid} but does not have permission to write #{which} owner_uuid #{check_uuid}"
578 errors.add :owner_uuid, "cannot be set or changed without write permission on #{which} owner"
579 raise PermissionDeniedError
580 elsif rsc_class == Group && Group.find_by_uuid(owner_uuid).group_class != "project"
581 errors.add :owner_uuid, "must be a project"
582 raise PermissionDeniedError
586 # If the object already existed and we're not changing
587 # owner_uuid, we only need write permission on the object
589 if !current_user.can?(write: self.uuid)
590 logger.warn "User #{current_user.uuid} tried to modify #{self.class.to_s} #{self.uuid} without write permission"
591 errors.add :uuid, " #{uuid} is not writable by #{current_user.uuid}"
592 raise PermissionDeniedError
599 def ensure_permission_to_save
600 unless (new_record? ? permission_to_create : permission_to_update)
601 raise PermissionDeniedError
605 def permission_to_create
606 current_user.andand.is_active
609 def permission_to_update
611 logger.warn "Anonymous user tried to update #{self.class.to_s} #{self.uuid_was}"
614 if !current_user.is_active
615 logger.warn "Inactive user #{current_user.uuid} tried to update #{self.class.to_s} #{self.uuid_was}"
618 return true if current_user.is_admin
619 if self.uuid_changed?
620 logger.warn "User #{current_user.uuid} tried to change uuid of #{self.class.to_s} #{self.uuid_was} to #{self.uuid}"
626 def ensure_permission_to_destroy
627 raise PermissionDeniedError unless permission_to_destroy
630 def permission_to_destroy
631 if [system_user_uuid, system_group_uuid, anonymous_group_uuid,
632 anonymous_user_uuid, public_project_uuid].include? uuid
639 def maybe_update_modified_by_fields
640 update_modified_by_fields if self.changed? or self.new_record?
644 def update_modified_by_fields
645 current_time = db_current_time
646 self.created_at ||= created_at_was || current_time
647 self.updated_at = current_time
648 self.owner_uuid ||= current_default_owner if self.respond_to? :owner_uuid=
649 if !anonymous_updater
650 self.modified_by_user_uuid = current_user ? current_user.uuid : nil
653 self.modified_at = current_time
655 self.modified_by_client_uuid = current_api_client ? current_api_client.uuid : nil
659 def self.has_nonstring_keys? x
662 return true if !(k.is_a?(String) || k.is_a?(Symbol)) || has_nonstring_keys?(v)
666 return true if has_nonstring_keys?(v)
672 def self.where_serialized(colname, value, md5: false)
673 colsql = colname.to_s
675 colsql = "md5(#{colsql})"
678 # rails4 stores as null, rails3 stored as serialized [] or {}
679 sql = "#{colsql} is null or #{colsql} IN (?)"
682 sql = "#{colsql} IN (?)"
683 sorted = deep_sort_hash(value)
685 params = [sorted.to_yaml, SafeJSON.dump(sorted)]
687 params = params.map { |x| Digest::MD5.hexdigest(x) }
693 Hash => HashSerializer,
694 Array => ArraySerializer,
697 def self.serialize(colname, type)
698 coder = Serializer[type]
699 @serialized_attributes ||= {}
700 @serialized_attributes[colname.to_s] = coder
701 super(colname, coder)
704 def self.serialized_attributes
705 @serialized_attributes ||= {}
708 def serialized_attributes
709 self.class.serialized_attributes
712 def foreign_key_attributes
713 attributes.keys.select { |a| a.match(/_uuid$/) }
716 def skip_uuid_read_permission_check
717 %w(modified_by_client_uuid)
720 def skip_uuid_existence_check
724 def normalize_collection_uuids
725 foreign_key_attributes.each do |attr|
726 attr_value = send attr
727 if attr_value.is_a? String and
728 attr_value.match(/^[0-9a-f]{32,}(\+[@\w]+)*$/)
730 send "#{attr}=", Collection.normalize_uuid(attr_value)
732 # TODO: abort instead of silently accepting unnormalizable value?
738 @@prefixes_hash = nil
739 def self.uuid_prefixes
740 unless @@prefixes_hash
742 Rails.application.eager_load!
743 ActiveRecord::Base.descendants.reject(&:abstract_class?).each do |k|
744 if k.respond_to?(:uuid_prefix)
745 @@prefixes_hash[k.uuid_prefix] = k
752 def self.uuid_like_pattern
753 "#{Rails.configuration.ClusterID}-#{uuid_prefix}-_______________"
757 %r/[a-z0-9]{5}-#{uuid_prefix}-[a-z0-9]{15}/
760 def check_readable_uuid attr, attr_value
761 return if attr_value.nil?
762 if (r = ArvadosModel::resource_class_for_uuid attr_value)
763 unless skip_uuid_read_permission_check.include? attr
764 r = r.readable_by(current_user)
766 if r.where(uuid: attr_value).count == 0
767 errors.add(attr, "'#{attr_value}' not found")
770 # Not a valid uuid or PDH, but that (currently) is not an error.
774 def ensure_valid_uuids
775 specials = [system_user_uuid]
777 foreign_key_attributes.each do |attr|
778 if new_record? or send (attr + "_changed?")
779 next if skip_uuid_existence_check.include? attr
780 attr_value = send attr
781 next if specials.include? attr_value
782 check_readable_uuid attr, attr_value
787 def ensure_filesystem_compatible_name
788 if name == "." || name == ".."
789 errors.add(:name, "cannot be '.' or '..'")
790 elsif Rails.configuration.Collections.ForwardSlashNameSubstitution == "" && !name.nil? && name.index('/')
791 errors.add(:name, "cannot contain a '/' character")
804 def self.readable_by (*u)
809 [{:uuid => u[:uuid]}]
813 def self.resource_class_for_uuid(uuid)
814 if uuid.is_a? ArvadosModel
817 unless uuid.is_a? String
821 uuid.match HasUuid::UUID_REGEX do |re|
822 return uuid_prefixes[re[1]] if uuid_prefixes[re[1]]
825 if uuid.match(/.+@.+/)
832 # ArvadosModel.find_by_uuid needs extra magic to allow it to return
833 # an object in any class.
834 def self.find_by_uuid uuid
835 if self == ArvadosModel
836 # If called directly as ArvadosModel.find_by_uuid rather than via subclass,
837 # delegate to the appropriate subclass based on the given uuid.
838 self.resource_class_for_uuid(uuid).find_by_uuid(uuid)
844 def is_audit_logging_enabled?
845 return !(Rails.configuration.AuditLogs.MaxAge.to_i == 0 &&
846 Rails.configuration.AuditLogs.MaxDeleteBatch.to_i > 0)
849 def schedule_restoring_changes
850 # This will be checked at log_start_state, to reset any (virtual) changes
851 # produced by the act of reading a serialized attribute.
852 @fresh_from_database = true
856 if is_audit_logging_enabled?
857 @old_attributes = Marshal.load(Marshal.dump(attributes))
858 @old_logged_attributes = Marshal.load(Marshal.dump(logged_attributes))
859 if @fresh_from_database
860 # This instance was created from reading a database record. Attributes
861 # haven't been changed, but those serialized attributes will be reported
862 # as unpersisted, so we restore them to avoid issues with lock!() and
865 @fresh_from_database = nil
870 def log_change(event_type)
871 if is_audit_logging_enabled?
872 log = Log.new(event_type: event_type).fill_object(self)
880 if is_audit_logging_enabled?
881 log_change('create') do |log|
882 log.fill_properties('old', nil, nil)
889 if is_audit_logging_enabled?
890 log_change('update') do |log|
891 log.fill_properties('old', etag(@old_attributes), @old_logged_attributes)
898 if is_audit_logging_enabled?
899 log_change('delete') do |log|
900 log.fill_properties('old', etag(@old_attributes), @old_logged_attributes)