X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/d5524135b1495b919de332df4f952926664961f5..d8e3a67d508e9a5f5c01884259c0e75a140f64e9:/services/api/app/controllers/application_controller.rb diff --git a/services/api/app/controllers/application_controller.rb b/services/api/app/controllers/application_controller.rb index 776f7e190e..c39bdde4b8 100644 --- a/services/api/app/controllers/application_controller.rb +++ b/services/api/app/controllers/application_controller.rb @@ -1,3 +1,10 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +require 'safe_json' +require 'request_error' + module ApiTemplateOverride def allowed_to_render?(fieldset, field, model, options) return false if !super @@ -16,37 +23,39 @@ end require 'load_param' class ApplicationController < ActionController::Base - include CurrentApiClient include ThemesForRails::ActionController + include CurrentApiClient include LoadParam + include DbCurrentTime respond_to :json protect_from_forgery ERROR_ACTIONS = [:render_error, :render_not_found] - before_filter :disable_api_methods - before_filter :set_cors_headers - before_filter :respond_with_json_by_default - before_filter :remote_ip - before_filter :load_read_auths - before_filter :require_auth_scope, except: ERROR_ACTIONS - - before_filter :catch_redirect_hint - before_filter(:find_object_by_uuid, + around_action :set_current_request_id + before_action :disable_api_methods + before_action :set_cors_headers + before_action :respond_with_json_by_default + before_action :remote_ip + before_action :load_read_auths + before_action :require_auth_scope, except: ERROR_ACTIONS + + before_action :catch_redirect_hint + before_action :load_required_parameters + before_action(:find_object_by_uuid, except: [:index, :create] + ERROR_ACTIONS) - before_filter :load_required_parameters - before_filter :load_limit_offset_order_params, only: [:index, :contents] - before_filter :load_where_param, only: [:index, :contents] - before_filter :load_filters_param, only: [:index, :contents] - before_filter :find_objects_for_index, :only => :index - before_filter :reload_object_before_update, :only => :update - before_filter(:render_404_if_no_object, + before_action(:set_nullable_attrs_to_null, only: [:update, :create]) + before_action :load_limit_offset_order_params, only: [:index, :contents] + before_action :load_where_param, only: [:index, :contents] + before_action :load_filters_param, only: [:index, :contents] + before_action :find_objects_for_index, :only => :index + before_action :reload_object_before_update, :only => :update + before_action(:render_404_if_no_object, except: [:index, :create] + ERROR_ACTIONS) + before_action :only_admin_can_bypass_federation - theme :select_theme - - attr_accessor :resource_attrs + attr_writer :resource_attrs begin rescue_from(Exception, @@ -54,21 +63,35 @@ class ApplicationController < ActionController::Base :with => :render_error) rescue_from(ActiveRecord::RecordNotFound, ActionController::RoutingError, - ActionController::UnknownController, AbstractController::ActionNotFound, :with => :render_not_found) end + def initialize *args + super + @object = nil + @objects = nil + @offset = nil + @limit = nil + @select = nil + @distinct = nil + @response_resource_name = nil + @attrs = nil + @extra_included = nil + end + def default_url_options - if Rails.configuration.host - {:host => Rails.configuration.host} - else - {} + options = {} + if Rails.configuration.Services.Controller.ExternalURL != URI("") + exturl = Rails.configuration.Services.Controller.ExternalURL + options[:host] = exturl.host + options[:port] = exturl.port + options[:protocol] = exturl.scheme end + options end def index - @objects.uniq!(&:id) if @select.nil? or @select.include? "id" if params[:eager] and params[:eager] != '0' and params[:eager] != 0 and params[:eager] != '' @objects.each(&:eager_load_associations) end @@ -82,41 +105,12 @@ class ApplicationController < ActionController::Base def create @object = model_class.new resource_attrs - if @object.respond_to? :name and params[:ensure_unique_name] - # Record the original name. See below. - name_stem = @object.name - counter = 1 + if @object.respond_to?(:name) && params[:ensure_unique_name] + @object.save_with_unique_name! + else + @object.save! end - begin - @object.save! - rescue ActiveRecord::RecordNotUnique => rn - raise unless params[:ensure_unique_name] - - # Dig into the error to determine if it is specifically calling out a - # (owner_uuid, name) uniqueness violation. In this specific case, and - # the client requested a unique name with ensure_unique_name==true, - # update the name field and try to save again. Loop as necessary to - # discover a unique name. It is necessary to handle name choosing at - # this level (as opposed to the client) to ensure that record creation - # never fails due to a race condition. - raise unless rn.original_exception.is_a? PG::UniqueViolation - - # Unfortunately ActiveRecord doesn't abstract out any of the - # necessary information to figure out if this the error is actually - # the specific case where we want to apply the ensure_unique_name - # behavior, so the following code is specialized to Postgres. - err = rn.original_exception - detail = err.result.error_field(PG::Result::PG_DIAG_MESSAGE_DETAIL) - raise unless /^Key \(owner_uuid, name\)=\([a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15}, .*?\) already exists\./.match detail - - # OK, this exception really is just a unique name constraint - # violation, and we've been asked to ensure_unique_name. - counter += 1 - @object.uuid = nil - @object.name = "#{name_stem} (#{counter})" - redo - end while false show end @@ -145,10 +139,21 @@ class ApplicationController < ActionController::Base render_not_found "Object not found" if !@object end + def only_admin_can_bypass_federation + unless !params[:bypass_federation] || current_user.andand.is_admin + send_error("The bypass_federation parameter is only permitted when current user is admin", status: 403) + end + end + def render_error(e) logger.error e.inspect if e.respond_to? :backtrace and e.backtrace - logger.error e.backtrace.collect { |x| x + "\n" }.join('') + # This will be cleared by lograge after adding it to the log. + # Usually lograge would get the exceptions, but in our case we're catching + # all of them with exception handlers that cannot re-raise them because they + # don't get propagated. + Thread.current[:exception] = e.inspect + Thread.current[:backtrace] = e.backtrace.collect { |x| x + "\n" }.join('') end if (@object.respond_to? :errors and @object.errors.andand.full_messages.andand.any?) @@ -166,8 +171,23 @@ class ApplicationController < ActionController::Base send_error("Path not found", status: 404) end + def render_accepted + send_json ({accepted: true}), status: 202 + end + protected + def bool_param(pname) + if params.include?(pname) + if params[pname].is_a?(Boolean) + return params[pname] + else + logger.warn "Warning: received non-boolean value #{params[pname].inspect} for boolean parameter #{pname} on #{self.class.inspect}, treating as false." + end + end + false + end + def send_error(*args) if args.last.is_a? Hash err = args.pop @@ -175,6 +195,9 @@ class ApplicationController < ActionController::Base err = {} end err[:errors] ||= args + err[:errors].map! do |err| + err += " (#{request.request_id})" + end err[:error_token] = [Time.now.utc.to_i, "%08x" % rand(16 ** 8)].join("+") status = err.delete(:status) || 422 logger.error "Error #{err[:error_token]}: #{status}" @@ -185,23 +208,17 @@ class ApplicationController < ActionController::Base # The obvious render(json: ...) forces a slow JSON encoder. See # #3021 and commit logs. Might be fixed in Rails 4.1. render({ - text: Oj.dump(response, mode: :compat).html_safe, + plain: SafeJSON.dump(response).html_safe, content_type: 'application/json' }.merge opts) end - def self.limit_index_columns_read - # This method returns a list of column names. - # If an index request reads that column from the database, - # find_objects_for_index will only fetch objects until it reads - # max_index_database_read bytes of data from those columns. - [] - end - def find_objects_for_index - @objects ||= model_class.readable_by(*@read_users) + @objects ||= model_class.readable_by(*@read_users, { + :include_trash => (bool_param(:include_trash) || 'untrash' == action_name), + :include_old_versions => bool_param(:include_old_versions) + }) apply_where_limit_order_params - limit_database_read if (action_name == "index") end def apply_filters model_class=nil @@ -246,7 +263,7 @@ class ApplicationController < ActionController::Base conditions[0] << " and #{ar_table_name}.#{attr} in (?)" conditions << value end - elsif value.is_a? String or value.is_a? Fixnum or value == true or value == false + elsif value.is_a? String or value.is_a? Integer or value == true or value == false conditions[0] << " and #{ar_table_name}.#{attr}=?" conditions << value elsif value.is_a? Hash @@ -287,31 +304,37 @@ class ApplicationController < ActionController::Base @objects = @objects.order(@orders.join ", ") if @orders.any? @objects = @objects.limit(@limit) @objects = @objects.offset(@offset) - @objects = @objects.uniq(@distinct) if not @distinct.nil? + @objects = @objects.distinct(@distinct) if not @distinct.nil? end - def limit_database_read - limit_columns = self.class.limit_index_columns_read + # limit_database_read ensures @objects (which must be an + # ActiveRelation) does not return too many results to fit in memory, + # by previewing the results and calling @objects.limit() if + # necessary. + def limit_database_read(model_class:) + return if @limit == 0 || @limit == 1 + model_class ||= self.model_class + limit_columns = model_class.limit_index_columns_read limit_columns &= model_class.columns_for_attributes(@select) if @select return if limit_columns.empty? model_class.transaction do limit_query = @objects. - except(:select). + except(:select, :distinct). select("(%s) as read_length" % - limit_columns.map { |s| "octet_length(#{s})" }.join(" + ")) + limit_columns.map { |s| "octet_length(#{model_class.table_name}.#{s})" }.join(" + ")) new_limit = 0 read_total = 0 limit_query.each do |record| new_limit += 1 read_total += record.read_length.to_i - if read_total >= Rails.configuration.max_index_database_read + if read_total >= Rails.configuration.API.MaxIndexDatabaseRead new_limit -= 1 if new_limit > 1 + @limit = new_limit break elsif new_limit >= @limit break end end - @limit = new_limit @objects = @objects.limit(@limit) # Force @objects to run its query inside this transaction. @objects.each { |_| break } @@ -321,10 +344,12 @@ class ApplicationController < ActionController::Base def resource_attrs return @attrs if @attrs @attrs = params[resource_name] - if @attrs.is_a? String + if @attrs.nil? + @attrs = {} + elsif @attrs.is_a? String @attrs = Oj.strict_load @attrs, symbol_keys: true end - unless @attrs.is_a? Hash + unless [Hash, ActionController::Parameters].include? @attrs.class message = "No #{resource_name}" if resource_name.index('_') message << " (or #{resource_name.camelcase(:lower)})" @@ -335,7 +360,7 @@ class ApplicationController < ActionController::Base %w(created_at modified_by_client_uuid modified_by_user_uuid modified_at).each do |x| @attrs.delete x.to_sym end - @attrs = @attrs.symbolize_keys if @attrs.is_a? HashWithIndifferentAccess + @attrs = @attrs.symbolize_keys if @attrs.is_a? ActiveSupport::HashWithIndifferentAccess @attrs end @@ -349,23 +374,30 @@ class ApplicationController < ActionController::Base # If there are too many reader tokens, assume the request is malicious # and ignore it. if request.get? and params[:reader_tokens] and - params[:reader_tokens].size < 100 + params[:reader_tokens].size < 100 + secrets = params[:reader_tokens].map { |t| + if t.is_a? String and t.starts_with? "v2/" + t.split("/")[2] + else + t + end + } @read_auths += ApiClientAuthorization .includes(:user) .where('api_token IN (?) AND (expires_at IS NULL OR expires_at > CURRENT_TIMESTAMP)', - params[:reader_tokens]) - .all + secrets) + .to_a end @read_auths.select! { |auth| auth.scopes_allow_request? request } - @read_users = @read_auths.map { |auth| auth.user }.uniq + @read_users = @read_auths.map(&:user).uniq end def require_login if not current_user respond_to do |format| format.json { send_error("Not logged in", status: 401) } - format.html { redirect_to '/auth/joshid' } + format.html { redirect_to '/login' } end false end @@ -378,7 +410,7 @@ class ApplicationController < ActionController::Base end def require_auth_scope - if @read_auths.empty? + unless current_user && @read_auths.any? { |auth| auth.user.andand.uuid == current_user.uuid } if require_login != false send_error("Forbidden", status: 403) end @@ -386,9 +418,21 @@ class ApplicationController < ActionController::Base end end + def set_current_request_id + Rails.logger.tagged(request.request_id) do + yield + end + end + + def append_info_to_payload(payload) + super + payload[:request_id] = response.headers['X-Request-Id'] + payload[:client_ipaddr] = @remote_ip + payload[:client_auth] = current_api_client_authorization.andand.uuid || nil + end + def disable_api_methods - if Rails.configuration.disable_api_methods. - include?(controller_name + "." + action_name) + if Rails.configuration.API.DisabledAPIs[controller_name + "." + action_name] send_error("Disabled", status: 404) end end @@ -396,13 +440,13 @@ class ApplicationController < ActionController::Base def set_cors_headers response.headers['Access-Control-Allow-Origin'] = '*' response.headers['Access-Control-Allow-Methods'] = 'GET, HEAD, PUT, POST, DELETE' - response.headers['Access-Control-Allow-Headers'] = 'Authorization' + response.headers['Access-Control-Allow-Headers'] = 'Authorization, Content-Type' response.headers['Access-Control-Max-Age'] = '86486400' end def respond_with_json_by_default - html_index = request.accepts.index(Mime::HTML) - if html_index.nil? or request.accepts[0...html_index].include?(Mime::JSON) + html_index = request.accepts.index(Mime[:html]) + if html_index.nil? or request.accepts[0...html_index].include?(Mime[:json]) request.format = :json end end @@ -420,7 +464,7 @@ class ApplicationController < ActionController::Base end def find_object_by_uuid - if params[:id] and params[:id].match /\D/ + if params[:id] and params[:id].match(/\D/) params[:uuid] = params.delete :id end @where = { uuid: params[:uuid] } @@ -433,6 +477,29 @@ class ApplicationController < ActionController::Base @object = @objects.first end + def nullable_attributes + [] + end + + # Go code may send empty values (ie: empty string instead of NULL) that + # should be translated to NULL on the database. + def set_nullable_attrs_to_null + nullify_attrs(resource_attrs.to_hash).each do |k, v| + resource_attrs[k] = v + end + end + + def nullify_attrs(a = {}) + new_attrs = a.to_hash.symbolize_keys + (new_attrs.keys & nullable_attributes).each do |attr| + val = new_attrs[attr] + if (val.class == Integer && val == 0) || (val.class == String && val == "") + new_attrs[attr] = nil + end + end + return new_attrs + end + def reload_object_before_update # This is necessary to prevent an ActiveRecord::ReadOnlyRecord # error when updating an object which was retrieved using a join. @@ -442,21 +509,31 @@ class ApplicationController < ActionController::Base end def load_json_value(hash, key, must_be_class=nil) - if hash[key].is_a? String - hash[key] = Oj.strict_load(hash[key], symbol_keys: false) - if must_be_class and !hash[key].is_a? must_be_class - raise TypeError.new("parameter #{key.to_s} must be a #{must_be_class.to_s}") - end + return if hash[key].nil? + + val = hash[key] + if val.is_a? ActionController::Parameters + val = val.to_unsafe_hash + elsif val.is_a? String + val = SafeJSON.load(val) + hash[key] = val + end + # When assigning a Hash to an ActionController::Parameters and then + # retrieve it, we get another ActionController::Parameters instead of + # a Hash. This doesn't happen with other types. This is why 'val' is + # being used to do type checking below. + if must_be_class and !val.is_a? must_be_class + raise TypeError.new("parameter #{key.to_s} must be a #{must_be_class.to_s}") end end def self.accept_attribute_as_json(attr, must_be_class=nil) - before_filter lambda { accept_attribute_as_json attr, must_be_class } + before_action lambda { accept_attribute_as_json attr, must_be_class } end accept_attribute_as_json :properties, Hash accept_attribute_as_json :info, Hash def accept_attribute_as_json(attr, must_be_class) - if params[resource_name] and resource_attrs.is_a? Hash + if params[resource_name] and [Hash, ActionController::Parameters].include?(resource_attrs.class) if resource_attrs[attr].is_a? Hash # Convert symbol keys to strings (in hashes provided by # resource_attrs) @@ -469,11 +546,14 @@ class ApplicationController < ActionController::Base end def self.accept_param_as_json(key, must_be_class=nil) - prepend_before_filter lambda { load_json_value(params, key, must_be_class) } + prepend_before_action lambda { load_json_value(params, key, must_be_class) } end accept_param_as_json :reader_tokens, Array - def object_list + def object_list(model_class:) + if @objects.respond_to?(:except) + limit_database_read(model_class: model_class) + end list = { :kind => "arvados##{(@response_resource_name || resource_name).camelize(:lower)}List", :etag => "", @@ -482,21 +562,30 @@ class ApplicationController < ActionController::Base :limit => @limit, :items => @objects.as_api_response(nil, {select: @select}) } - if @objects.respond_to? :except - list[:items_available] = @objects. - except(:limit).except(:offset). - count(:id, distinct: true) + if @extra_included + list[:included] = @extra_included.as_api_response(nil, {select: @select}) + end + case params[:count] + when nil, '', 'exact' + if @objects.respond_to? :except + list[:items_available] = @objects. + except(:limit).except(:offset). + count(@distinct ? :id : '*') + end + when 'none' + else + raise ArgumentError.new("count parameter must be 'exact' or 'none'") end list end def render_list - send_json object_list + send_json object_list(model_class: self.model_class) end def remote_ip # Caveat: this is highly dependent on the proxy setup. YMMV. - if request.headers.has_key?('HTTP_X_REAL_IP') then + if request.headers.key?('HTTP_X_REAL_IP') then # We're behind a reverse proxy @remote_ip = request.headers['HTTP_X_REAL_IP'] else @@ -514,7 +603,7 @@ class ApplicationController < ActionController::Base # Make sure params[key] is either true or false -- not a # string, not nil, etc. if not params.include?(key) - params[key] = info[:default] + params[key] = info[:default] || false elsif [false, 'false', '0', 0].include? params[key] params[key] = false elsif [true, 'true', '1', 1].include? params[key] @@ -535,10 +624,20 @@ class ApplicationController < ActionController::Base location: "query", required: false, default: false - } + }, + cluster_id: { + type: 'string', + description: "Create object on a remote federated cluster instead of the current one.", + location: "query", + required: false, + }, } end + def self._update_requires_parameters + {} + end + def self._index_requires_parameters { filters: { type: 'array', required: false }, @@ -548,6 +647,18 @@ class ApplicationController < ActionController::Base distinct: { type: 'boolean', required: false }, limit: { type: 'integer', required: false, default: DEFAULT_LIMIT }, offset: { type: 'integer', required: false, default: 0 }, + count: { type: 'string', required: false, default: 'exact' }, + cluster_id: { + type: 'string', + description: "List objects on a remote federated cluster instead of the current one.", + location: "query", + required: false, + }, + bypass_federation: { + type: 'boolean', + required: false, + description: 'bypass federation behavior, list items from local instance database only' + } } end @@ -567,10 +678,6 @@ class ApplicationController < ActionController::Base } end end - super *opts - end - - def select_theme - return Rails.configuration.arvados_theme + super(*opts) end end