13996: Can now use database info from config.yml

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz@veritasgenetics.com>

diff --git a/services/api/config/initializers/load_config.rb b/services/api/config/arvados_config.rb
similarity index 85%
rename from services/api/config/initializers/load_config.rb
rename to services/api/config/arvados_config.rb
index 8bed5c6551d885b02af0219e6ced1bb85da0d7f5..b0ac7c0be2dbe9a68883aea96b8164d38a184c22 100644 (file)
--- a/services/api/config/initializers/load_config.rb
+++ b/services/api/config/arvados_config.rb
@@ -2,6 +2,19 @@
 #
 # SPDX-License-Identifier: AGPL-3.0
 
+#
+# Load Arvados configuration from /etc/arvados/config.yml, using defaults from config.defaults.yml
+#
+# Existing application.yml is migrated into the new config structure.
+# Keys in the legacy application.yml take precedence.
+#
+# Use "bundle exec config:dump" to get the complete active configuration
+#
+# Use "bundle exec config:migrate" to migrate application.yml and
+# database.yml to config.yml.  After adding the output of
+# config:migrate to /etc/arvados/config.yml, you will be able to
+# delete application.yml and database.yml.
+
 require 'config_loader'
 
 begin
@@ -36,7 +49,7 @@ $arvados_config = {}
     if confs
       clusters = confs["Clusters"].first
       $arvados_config["ClusterID"] = clusters[0]
-      $arvados_config.merge!(clusters[1])
+      $arvados_config.deep_merge!(clusters[1])
     end
   end
 end
@@ -154,17 +167,17 @@ application_config = {}
     confs = YAML.load(yaml, deserialize_symbols: true)
     # Ignore empty YAML file:
     next if confs == false
-    application_config.merge!(confs['common'] || {})
-    application_config.merge!(confs[::Rails.env.to_s] || {})
+    application_config.deep_merge!(confs['common'] || {})
+    application_config.deep_merge!(confs[::Rails.env.to_s] || {})
   end
 end
 
 db_config = {}
-path = "#{::Rails.root.to_s}/config/database.ymlx"
+path = "#{::Rails.root.to_s}/config/database.yml"
 if File.exist? path
   yaml = ERB.new(IO.read path).result(binding)
   confs = YAML.load(yaml, deserialize_symbols: true)
-  db_config.merge!(confs[::Rails.env.to_s] || {})
+  db_config.deep_merge!(confs[::Rails.env.to_s] || {})
 end
 
 $remaining_config = arvcfg.migrate_config(application_config, $arvados_config)
@@ -183,22 +196,29 @@ end
 arvcfg.coercion_and_check $arvados_config
 dbcfg.coercion_and_check $arvados_config
 
+#
+# Special case for test database, because the Arvados config.yml
+# doesn't have a concept of multiple rails environments.
+#
+if ::Rails.env.to_s == "test"
+  $arvados_config["PostgreSQL"]["Connection"]["DBName"] = "arvados_test"
+end
+
+dbhost = $arvados_config["PostgreSQL"]["Connection"]["Host"]
+if $arvados_config["PostgreSQL"]["Connection"]["Post"] != 0
+  dbhost += ":#{$arvados_config["PostgreSQL"]["Connection"]["Post"]}"
+end
+
+#
+# If DATABASE_URL is set, then ActiveRecord won't error out if database.yml doesn't exist.
+#
+# For config migration, we've previously populated the PostgreSQL
+# section of the config from database.yml
+#
+ENV["DATABASE_URL"] = "postgresql://#{$arvados_config["PostgreSQL"]["Connection"]["User"]}:#{$arvados_config["PostgreSQL"]["Connection"]["Password"]}@#{dbhost}/#{$arvados_config["PostgreSQL"]["Connection"]["DBName"]}?template=#{$arvados_config["PostgreSQL"]["Connection"]["Template"]}&encoding=#{$arvados_config["PostgreSQL"]["Connection"]["client_encoding"]}&pool=#{$arvados_config["PostgreSQL"]["ConnectionPool"]}"
+
 Server::Application.configure do
   ConfigLoader.copy_into_config $arvados_config, config
   ConfigLoader.copy_into_config $remaining_config, config
   config.secret_key_base = config.secret_token
-
-  dbcfg = {}
-  dbcfg[::Rails.env.to_s] = {
-    adapter: 'postgresql',
-    template: $arvados_config["PostgreSQL"]["Connection"]["Template"],
-    encoding: $arvados_config["PostgreSQL"]["Connection"]["Encoding"],
-    database: $arvados_config["PostgreSQL"]["Connection"]["DBName"],
-    username: $arvados_config["PostgreSQL"]["Connection"]["User"],
-    password: $arvados_config["PostgreSQL"]["Connection"]["Password"],
-    host: $arvados_config["PostgreSQL"]["Connection"]["Host"],
-    port: $arvados_config["PostgreSQL"]["Connection"]["Port"],
-    pool: $arvados_config["PostgreSQL"]["ConnectionPool"]
-  }
-  Rails.application.config.database_configuration = dbcfg
 end

diff --git a/services/api/config/environment.rb b/services/api/config/environment.rb
index b82ba27f9ae1bd390bdb71c0fd4f79c9c1fb4b70..fbca777363d391e344faf2cad77d8ccfa30bb283 100644 (file)
--- a/services/api/config/environment.rb
+++ b/services/api/config/environment.rb
@@ -5,6 +5,7 @@
 # Load the rails application
 require_relative 'application'
 require 'josh_id'
+require_relative 'arvados_config'
 
 # Initialize the rails application
 Rails.application.initialize!

diff --git a/services/api/config/initializers/legacy_jobs_api.rb b/services/api/config/initializers/legacy_jobs_api.rb
index 9ea6b2884374317e65c1cbc1027a1f0581b3a103..8f3b3cb5f8e951df55979a1f74adce8b847de652 100644 (file)
--- a/services/api/config/initializers/legacy_jobs_api.rb
+++ b/services/api/config/initializers/legacy_jobs_api.rb
@@ -5,7 +5,6 @@
 # Config must be done before we  files; otherwise they
 # won't be able to use Rails.configuration.* to initialize their
 # classes.
-require_relative 'load_config.rb'
 
 require 'enable_jobs_api'
 

diff --git a/services/api/config/initializers/preload_all_models.rb b/services/api/config/initializers/preload_all_models.rb
index 0ab2b032a4c0e4e7b55da23512ee62fd03c4ac88..713c61fd75a6e75794b31afa7a1d6ca3f6005a43 100644 (file)
--- a/services/api/config/initializers/preload_all_models.rb
+++ b/services/api/config/initializers/preload_all_models.rb
@@ -7,7 +7,6 @@
 # Config must be done before we load model class files; otherwise they
 # won't be able to use Rails.configuration.* to initialize their
 # classes.
-require_relative 'load_config.rb'
 
 if Rails.env == 'development'
   Dir.foreach("#{Rails.root}/app/models") do |model_file|

diff --git a/services/api/lib/tasks/config_diff.rake b/services/api/lib/tasks/config_diff.rake
deleted file mode 100644 (file)
index 0fd6fdd..0000000
--- a/services/api/lib/tasks/config_diff.rake
+++ /dev/null
@@ -1,32 +0,0 @@
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-def diff_hash base, final
-  diffed = {}
-  base.each do |k,v|
-    bk = base[k]
-    fk = final[k]
-    if bk.is_a? Hash
-      d = diff_hash bk, fk
-      if d.length > 0
-        diffed[k] = d
-      end
-    else
-      if bk.to_s != fk.to_s
-        diffed[k] = fk
-      end
-    end
-  end
-  diffed
-end
-
-namespace :config do
-  desc 'Print configuration loaded from legacy application.yml as new Arvados configuration structure'
-  task diff: :environment do
-    diffed = diff_hash $base_arvados_config, $arvados_config
-    cfg = { "Clusters" => {}}
-    cfg["Clusters"][$arvados_config["ClusterID"]] = diffed.select {|k,v| k != "ClusterID"}
-    puts cfg.to_yaml
-  end
-end

diff --git a/services/api/test/unit/container_request_test.rb b/services/api/test/unit/container_request_test.rb
index c8dcfef238450157c1eb95d780be2deb08da7681..6f0233306925d8cb44653c41e6b3ac6b1a69c96d 100644 (file)
--- a/services/api/test/unit/container_request_test.rb
+++ b/services/api/test/unit/container_request_test.rb
@@ -525,7 +525,6 @@ class ContainerRequestTest < ActiveSupport::TestCase
    'ENOEXIST',
    'arvados/apitestfixture:ENOEXIST',
   ].each do |img|
-    puts "RC", Rails.configuration.RemoteClusters
     test "container_image_for_container(#{img.inspect}) => 422" do
       set_user_from_auth :active
       assert_raises(ArvadosModel::UnresolvableContainerError) do