From: Peter Amstutz Date: Wed, 3 Apr 2019 20:41:58 +0000 (-0400) Subject: 13996: Can now use database info from config.yml X-Git-Tag: 1.4.0~65^2~14 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/6e9fcde0422b33d081e2985975e3104eb2434957 13996: Can now use database info from config.yml Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- diff --git a/services/api/config/initializers/load_config.rb b/services/api/config/arvados_config.rb similarity index 85% rename from services/api/config/initializers/load_config.rb rename to services/api/config/arvados_config.rb index 8bed5c6551..b0ac7c0be2 100644 --- a/services/api/config/initializers/load_config.rb +++ b/services/api/config/arvados_config.rb @@ -2,6 +2,19 @@ # # SPDX-License-Identifier: AGPL-3.0 +# +# Load Arvados configuration from /etc/arvados/config.yml, using defaults from config.defaults.yml +# +# Existing application.yml is migrated into the new config structure. +# Keys in the legacy application.yml take precedence. +# +# Use "bundle exec config:dump" to get the complete active configuration +# +# Use "bundle exec config:migrate" to migrate application.yml and +# database.yml to config.yml. After adding the output of +# config:migrate to /etc/arvados/config.yml, you will be able to +# delete application.yml and database.yml. + require 'config_loader' begin @@ -36,7 +49,7 @@ $arvados_config = {} if confs clusters = confs["Clusters"].first $arvados_config["ClusterID"] = clusters[0] - $arvados_config.merge!(clusters[1]) + $arvados_config.deep_merge!(clusters[1]) end end end @@ -154,17 +167,17 @@ application_config = {} confs = YAML.load(yaml, deserialize_symbols: true) # Ignore empty YAML file: next if confs == false - application_config.merge!(confs['common'] || {}) - application_config.merge!(confs[::Rails.env.to_s] || {}) + application_config.deep_merge!(confs['common'] || {}) + application_config.deep_merge!(confs[::Rails.env.to_s] || {}) end end db_config = {} -path = "#{::Rails.root.to_s}/config/database.ymlx" +path = "#{::Rails.root.to_s}/config/database.yml" if File.exist? path yaml = ERB.new(IO.read path).result(binding) confs = YAML.load(yaml, deserialize_symbols: true) - db_config.merge!(confs[::Rails.env.to_s] || {}) + db_config.deep_merge!(confs[::Rails.env.to_s] || {}) end $remaining_config = arvcfg.migrate_config(application_config, $arvados_config) @@ -183,22 +196,29 @@ end arvcfg.coercion_and_check $arvados_config dbcfg.coercion_and_check $arvados_config +# +# Special case for test database, because the Arvados config.yml +# doesn't have a concept of multiple rails environments. +# +if ::Rails.env.to_s == "test" + $arvados_config["PostgreSQL"]["Connection"]["DBName"] = "arvados_test" +end + +dbhost = $arvados_config["PostgreSQL"]["Connection"]["Host"] +if $arvados_config["PostgreSQL"]["Connection"]["Post"] != 0 + dbhost += ":#{$arvados_config["PostgreSQL"]["Connection"]["Post"]}" +end + +# +# If DATABASE_URL is set, then ActiveRecord won't error out if database.yml doesn't exist. +# +# For config migration, we've previously populated the PostgreSQL +# section of the config from database.yml +# +ENV["DATABASE_URL"] = "postgresql://#{$arvados_config["PostgreSQL"]["Connection"]["User"]}:#{$arvados_config["PostgreSQL"]["Connection"]["Password"]}@#{dbhost}/#{$arvados_config["PostgreSQL"]["Connection"]["DBName"]}?template=#{$arvados_config["PostgreSQL"]["Connection"]["Template"]}&encoding=#{$arvados_config["PostgreSQL"]["Connection"]["client_encoding"]}&pool=#{$arvados_config["PostgreSQL"]["ConnectionPool"]}" + Server::Application.configure do ConfigLoader.copy_into_config $arvados_config, config ConfigLoader.copy_into_config $remaining_config, config config.secret_key_base = config.secret_token - - dbcfg = {} - dbcfg[::Rails.env.to_s] = { - adapter: 'postgresql', - template: $arvados_config["PostgreSQL"]["Connection"]["Template"], - encoding: $arvados_config["PostgreSQL"]["Connection"]["Encoding"], - database: $arvados_config["PostgreSQL"]["Connection"]["DBName"], - username: $arvados_config["PostgreSQL"]["Connection"]["User"], - password: $arvados_config["PostgreSQL"]["Connection"]["Password"], - host: $arvados_config["PostgreSQL"]["Connection"]["Host"], - port: $arvados_config["PostgreSQL"]["Connection"]["Port"], - pool: $arvados_config["PostgreSQL"]["ConnectionPool"] - } - Rails.application.config.database_configuration = dbcfg end diff --git a/services/api/config/environment.rb b/services/api/config/environment.rb index b82ba27f9a..fbca777363 100644 --- a/services/api/config/environment.rb +++ b/services/api/config/environment.rb @@ -5,6 +5,7 @@ # Load the rails application require_relative 'application' require 'josh_id' +require_relative 'arvados_config' # Initialize the rails application Rails.application.initialize! diff --git a/services/api/config/initializers/legacy_jobs_api.rb b/services/api/config/initializers/legacy_jobs_api.rb index 9ea6b28843..8f3b3cb5f8 100644 --- a/services/api/config/initializers/legacy_jobs_api.rb +++ b/services/api/config/initializers/legacy_jobs_api.rb @@ -5,7 +5,6 @@ # Config must be done before we files; otherwise they # won't be able to use Rails.configuration.* to initialize their # classes. -require_relative 'load_config.rb' require 'enable_jobs_api' diff --git a/services/api/config/initializers/preload_all_models.rb b/services/api/config/initializers/preload_all_models.rb index 0ab2b032a4..713c61fd75 100644 --- a/services/api/config/initializers/preload_all_models.rb +++ b/services/api/config/initializers/preload_all_models.rb @@ -7,7 +7,6 @@ # Config must be done before we load model class files; otherwise they # won't be able to use Rails.configuration.* to initialize their # classes. -require_relative 'load_config.rb' if Rails.env == 'development' Dir.foreach("#{Rails.root}/app/models") do |model_file| diff --git a/services/api/lib/tasks/config_diff.rake b/services/api/lib/tasks/config_diff.rake deleted file mode 100644 index 0fd6fdd46f..0000000000 --- a/services/api/lib/tasks/config_diff.rake +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright (C) The Arvados Authors. All rights reserved. -# -# SPDX-License-Identifier: AGPL-3.0 - -def diff_hash base, final - diffed = {} - base.each do |k,v| - bk = base[k] - fk = final[k] - if bk.is_a? Hash - d = diff_hash bk, fk - if d.length > 0 - diffed[k] = d - end - else - if bk.to_s != fk.to_s - diffed[k] = fk - end - end - end - diffed -end - -namespace :config do - desc 'Print configuration loaded from legacy application.yml as new Arvados configuration structure' - task diff: :environment do - diffed = diff_hash $base_arvados_config, $arvados_config - cfg = { "Clusters" => {}} - cfg["Clusters"][$arvados_config["ClusterID"]] = diffed.select {|k,v| k != "ClusterID"} - puts cfg.to_yaml - end -end diff --git a/services/api/test/unit/container_request_test.rb b/services/api/test/unit/container_request_test.rb index c8dcfef238..6f02333069 100644 --- a/services/api/test/unit/container_request_test.rb +++ b/services/api/test/unit/container_request_test.rb @@ -525,7 +525,6 @@ class ContainerRequestTest < ActiveSupport::TestCase 'ENOEXIST', 'arvados/apitestfixture:ENOEXIST', ].each do |img| - puts "RC", Rails.configuration.RemoteClusters test "container_image_for_container(#{img.inspect}) => 422" do set_user_from_auth :active assert_raises(ArvadosModel::UnresolvableContainerError) do