13996: Refactor, create ConfigLoader class

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz@veritasgenetics.com>

diff --git a/lib/config/config.defaults.yml b/lib/config/config.defaults.yml
index 58c1c86dd9202a96c9be28c1c874958d2b9a67af..3da6dc803f2753712909f018e35bfa17722f6c43 100644 (file)
--- a/lib/config/config.defaults.yml
+++ b/lib/config/config.defaults.yml
@@ -63,6 +63,17 @@ Clusters:
         ExternalURL: ""
       Workbench2:
         ExternalURL: ""
+    PostgreSQL:
+      # max concurrent connections per arvados server daemon
+      ConnectionPool: 32
+      Connection:
+        # All parameters here are passed to the PG client library in a connection string;
+        # see https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-PARAMKEYWORDS
+        Host: ""
+        Port: 0
+        User: ""
+        Password: ""
+        DBName: ""
     API:
       # Maximum size (in bytes) allowed for a single API request.  This
       # limit is published in the discovery document for use by clients.

diff --git a/services/api/config/initializers/load_config.rb b/services/api/config/initializers/load_config.rb
index 96f6a7f04a89938ca0869a3c1a36f6198ac75d9e..8bed5c6551d885b02af0219e6ced1bb85da0d7f5 100644 (file)
--- a/services/api/config/initializers/load_config.rb
+++ b/services/api/config/initializers/load_config.rb
@@ -43,82 +43,84 @@ end
 
 $base_arvados_config = $arvados_config.deep_dup
 
-declare_config "ClusterID", NonemptyString, :uuid_prefix
-declare_config "ManagementToken", String, :ManagementToken
-declare_config "Git.Repositories", String, :git_repositories_dir
-declare_config "API.DisabledAPIs", Array, :disable_api_methods
-declare_config "API.MaxRequestSize", Integer, :max_request_size
-declare_config "API.MaxIndexDatabaseRead", Integer, :max_index_database_read
-declare_config "API.MaxItemsPerResponse", Integer, :max_items_per_response
-declare_config "API.AsyncPermissionsUpdateInterval", ActiveSupport::Duration, :async_permissions_update_interval
-declare_config "Users.AutoSetupNewUsers", Boolean, :auto_setup_new_users
-declare_config "Users.AutoSetupNewUsersWithVmUUID", String, :auto_setup_new_users_with_vm_uuid
-declare_config "Users.AutoSetupNewUsersWithRepository", Boolean, :auto_setup_new_users_with_repository
-declare_config "Users.AutoSetupUsernameBlacklist", Array, :auto_setup_name_blacklist
-declare_config "Users.NewUsersAreActive", Boolean, :new_users_are_active
-declare_config "Users.AutoAdminUserWithEmail", String, :auto_admin_user
-declare_config "Users.AutoAdminFirstUser", Boolean, :auto_admin_first_user
-declare_config "Users.UserProfileNotificationAddress", String, :user_profile_notification_address
-declare_config "Users.AdminNotifierEmailFrom", String, :admin_notifier_email_from
-declare_config "Users.EmailSubjectPrefix", String, :email_subject_prefix
-declare_config "Users.UserNotifierEmailFrom", String, :user_notifier_email_from
-declare_config "Users.NewUserNotificationRecipients", Array, :new_user_notification_recipients
-declare_config "Users.NewInactiveUserNotificationRecipients", Array, :new_inactive_user_notification_recipients
-declare_config "Login.ProviderAppSecret", NonemptyString, :sso_app_secret
-declare_config "Login.ProviderAppID", NonemptyString, :sso_app_id
-declare_config "TLS.Insecure", Boolean, :sso_insecure
-declare_config "Services.SSO.ExternalURL", NonemptyString, :sso_provider_url
-declare_config "AuditLogs.MaxAge", ActiveSupport::Duration, :max_audit_log_age
-declare_config "AuditLogs.MaxDeleteBatch", Integer, :max_audit_log_delete_batch
-declare_config "AuditLogs.UnloggedAttributes", Array, :unlogged_attributes
-declare_config "SystemLogs.MaxRequestLogParamsSize", Integer, :max_request_log_params_size
-declare_config "Collections.DefaultReplication", Integer, :default_collection_replication
-declare_config "Collections.DefaultTrashLifetime", ActiveSupport::Duration, :default_trash_lifetime
-declare_config "Collections.CollectionVersioning", Boolean, :collection_versioning
-declare_config "Collections.PreserveVersionIfIdle", ActiveSupport::Duration, :preserve_version_if_idle
-declare_config "Collections.TrashSweepInterval", ActiveSupport::Duration, :trash_sweep_interval
-declare_config "Collections.BlobSigningKey", NonemptyString, :blob_signing_key
-declare_config "Collections.BlobSigningTTL", Integer, :blob_signature_ttl
-declare_config "Collections.BlobSigning", Boolean, :permit_create_collection_with_unsigned_manifest
-declare_config "Containers.SupportedDockerImageFormats", Array, :docker_image_formats
-declare_config "Containers.LogReuseDecisions", Boolean, :log_reuse_decisions
-declare_config "Containers.DefaultKeepCacheRAM", Integer, :container_default_keep_cache_ram
-declare_config "Containers.MaxDispatchAttempts", Integer, :max_container_dispatch_attempts
-declare_config "Containers.MaxRetryAttempts", Integer, :container_count_max
-declare_config "Containers.UsePreemptibleInstances", Boolean, :preemptible_instances
-declare_config "Containers.MaxComputeVMs", Integer, :max_compute_nodes
-declare_config "Containers.Logging.LogBytesPerEvent", Integer, :crunch_log_bytes_per_event
-declare_config "Containers.Logging.LogSecondsBetweenEvents", ActiveSupport::Duration, :crunch_log_seconds_between_events
-declare_config "Containers.Logging.LogThrottlePeriod", ActiveSupport::Duration, :crunch_log_throttle_period
-declare_config "Containers.Logging.LogThrottleBytes", Integer, :crunch_log_throttle_bytes
-declare_config "Containers.Logging.LogThrottleLines", Integer, :crunch_log_throttle_lines
-declare_config "Containers.Logging.LimitLogBytesPerJob", Integer, :crunch_limit_log_bytes_per_job
-declare_config "Containers.Logging.LogPartialLineThrottlePeriod", ActiveSupport::Duration, :crunch_log_partial_line_throttle_period
-declare_config "Containers.Logging.LogUpdatePeriod", ActiveSupport::Duration, :crunch_log_update_period
-declare_config "Containers.Logging.LogUpdateSize", Integer, :crunch_log_update_size
-declare_config "Containers.Logging.MaxAge", ActiveSupport::Duration, :clean_container_log_rows_after
-declare_config "Containers.SLURM.Managed.DNSServerConfDir", Pathname, :dns_server_conf_dir
-declare_config "Containers.SLURM.Managed.DNSServerConfTemplate", Pathname, :dns_server_conf_template
-declare_config "Containers.SLURM.Managed.DNSServerReloadCommand", String, :dns_server_reload_command
-declare_config "Containers.SLURM.Managed.DNSServerUpdateCommand", String, :dns_server_update_command
-declare_config "Containers.SLURM.Managed.ComputeNodeDomain", String, :compute_node_domain
-declare_config "Containers.SLURM.Managed.ComputeNodeNameservers", Array, :compute_node_nameservers
-declare_config "Containers.SLURM.Managed.AssignNodeHostname", String, :assign_node_hostname
-declare_config "Containers.JobsAPI.Enable", String, :enable_legacy_jobs_api, ->(cfg, k, v) { set_cfg cfg, "Containers.JobsAPI.Enable", v.to_s }
-declare_config "Containers.JobsAPI.CrunchJobWrapper", String, :crunch_job_wrapper
-declare_config "Containers.JobsAPI.CrunchJobUser", String, :crunch_job_user
-declare_config "Containers.JobsAPI.CrunchRefreshTrigger", String, :crunch_refresh_trigger
-declare_config "Containers.JobsAPI.GitInternalDir", String, :git_internal_dir
-declare_config "Containers.JobsAPI.ReuseJobIfOutputsDiffer", Boolean, :reuse_job_if_outputs_differ
-declare_config "Containers.JobsAPI.DefaultDockerImage", String, :default_docker_image_for_jobs
-declare_config "Mail.MailchimpAPIKey", String, :mailchimp_api_key
-declare_config "Mail.MailchimpListID", String, :mailchimp_list_id
-declare_config "Services.Workbench1.ExternalURL", URI, :workbench_address
-declare_config "Services.Websocket.ExternalURL", URI, :websocket_address
-declare_config "Services.WebDAV.ExternalURL", URI, :keep_web_service_url
-declare_config "Services.GitHTTP.ExternalURL", URI, :git_repo_https_base
-declare_config "Services.GitSSH.ExternalURL", URI, :git_repo_ssh_base, ->(cfg, k, v) { set_cfg cfg, "Services.GitSSH.ExternalURL", "ssh://#{v}" }
-declare_config "RemoteClusters", Hash, :remote_hosts, ->(cfg, k, v) {
+arvcfg = ConfigLoader.new
+
+arvcfg.declare_config "ClusterID", NonemptyString, :uuid_prefix
+arvcfg.declare_config "ManagementToken", String, :ManagementToken
+arvcfg.declare_config "Git.Repositories", String, :git_repositories_dir
+arvcfg.declare_config "API.DisabledAPIs", Array, :disable_api_methods
+arvcfg.declare_config "API.MaxRequestSize", Integer, :max_request_size
+arvcfg.declare_config "API.MaxIndexDatabaseRead", Integer, :max_index_database_read
+arvcfg.declare_config "API.MaxItemsPerResponse", Integer, :max_items_per_response
+arvcfg.declare_config "API.AsyncPermissionsUpdateInterval", ActiveSupport::Duration, :async_permissions_update_interval
+arvcfg.declare_config "Users.AutoSetupNewUsers", Boolean, :auto_setup_new_users
+arvcfg.declare_config "Users.AutoSetupNewUsersWithVmUUID", String, :auto_setup_new_users_with_vm_uuid
+arvcfg.declare_config "Users.AutoSetupNewUsersWithRepository", Boolean, :auto_setup_new_users_with_repository
+arvcfg.declare_config "Users.AutoSetupUsernameBlacklist", Array, :auto_setup_name_blacklist
+arvcfg.declare_config "Users.NewUsersAreActive", Boolean, :new_users_are_active
+arvcfg.declare_config "Users.AutoAdminUserWithEmail", String, :auto_admin_user
+arvcfg.declare_config "Users.AutoAdminFirstUser", Boolean, :auto_admin_first_user
+arvcfg.declare_config "Users.UserProfileNotificationAddress", String, :user_profile_notification_address
+arvcfg.declare_config "Users.AdminNotifierEmailFrom", String, :admin_notifier_email_from
+arvcfg.declare_config "Users.EmailSubjectPrefix", String, :email_subject_prefix
+arvcfg.declare_config "Users.UserNotifierEmailFrom", String, :user_notifier_email_from
+arvcfg.declare_config "Users.NewUserNotificationRecipients", Array, :new_user_notification_recipients
+arvcfg.declare_config "Users.NewInactiveUserNotificationRecipients", Array, :new_inactive_user_notification_recipients
+arvcfg.declare_config "Login.ProviderAppSecret", NonemptyString, :sso_app_secret
+arvcfg.declare_config "Login.ProviderAppID", NonemptyString, :sso_app_id
+arvcfg.declare_config "TLS.Insecure", Boolean, :sso_insecure
+arvcfg.declare_config "Services.SSO.ExternalURL", NonemptyString, :sso_provider_url
+arvcfg.declare_config "AuditLogs.MaxAge", ActiveSupport::Duration, :max_audit_log_age
+arvcfg.declare_config "AuditLogs.MaxDeleteBatch", Integer, :max_audit_log_delete_batch
+arvcfg.declare_config "AuditLogs.UnloggedAttributes", Array, :unlogged_attributes
+arvcfg.declare_config "SystemLogs.MaxRequestLogParamsSize", Integer, :max_request_log_params_size
+arvcfg.declare_config "Collections.DefaultReplication", Integer, :default_collection_replication
+arvcfg.declare_config "Collections.DefaultTrashLifetime", ActiveSupport::Duration, :default_trash_lifetime
+arvcfg.declare_config "Collections.CollectionVersioning", Boolean, :collection_versioning
+arvcfg.declare_config "Collections.PreserveVersionIfIdle", ActiveSupport::Duration, :preserve_version_if_idle
+arvcfg.declare_config "Collections.TrashSweepInterval", ActiveSupport::Duration, :trash_sweep_interval
+arvcfg.declare_config "Collections.BlobSigningKey", NonemptyString, :blob_signing_key
+arvcfg.declare_config "Collections.BlobSigningTTL", Integer, :blob_signature_ttl
+arvcfg.declare_config "Collections.BlobSigning", Boolean, :permit_create_collection_with_unsigned_manifest
+arvcfg.declare_config "Containers.SupportedDockerImageFormats", Array, :docker_image_formats
+arvcfg.declare_config "Containers.LogReuseDecisions", Boolean, :log_reuse_decisions
+arvcfg.declare_config "Containers.DefaultKeepCacheRAM", Integer, :container_default_keep_cache_ram
+arvcfg.declare_config "Containers.MaxDispatchAttempts", Integer, :max_container_dispatch_attempts
+arvcfg.declare_config "Containers.MaxRetryAttempts", Integer, :container_count_max
+arvcfg.declare_config "Containers.UsePreemptibleInstances", Boolean, :preemptible_instances
+arvcfg.declare_config "Containers.MaxComputeVMs", Integer, :max_compute_nodes
+arvcfg.declare_config "Containers.Logging.LogBytesPerEvent", Integer, :crunch_log_bytes_per_event
+arvcfg.declare_config "Containers.Logging.LogSecondsBetweenEvents", ActiveSupport::Duration, :crunch_log_seconds_between_events
+arvcfg.declare_config "Containers.Logging.LogThrottlePeriod", ActiveSupport::Duration, :crunch_log_throttle_period
+arvcfg.declare_config "Containers.Logging.LogThrottleBytes", Integer, :crunch_log_throttle_bytes
+arvcfg.declare_config "Containers.Logging.LogThrottleLines", Integer, :crunch_log_throttle_lines
+arvcfg.declare_config "Containers.Logging.LimitLogBytesPerJob", Integer, :crunch_limit_log_bytes_per_job
+arvcfg.declare_config "Containers.Logging.LogPartialLineThrottlePeriod", ActiveSupport::Duration, :crunch_log_partial_line_throttle_period
+arvcfg.declare_config "Containers.Logging.LogUpdatePeriod", ActiveSupport::Duration, :crunch_log_update_period
+arvcfg.declare_config "Containers.Logging.LogUpdateSize", Integer, :crunch_log_update_size
+arvcfg.declare_config "Containers.Logging.MaxAge", ActiveSupport::Duration, :clean_container_log_rows_after
+arvcfg.declare_config "Containers.SLURM.Managed.DNSServerConfDir", Pathname, :dns_server_conf_dir
+arvcfg.declare_config "Containers.SLURM.Managed.DNSServerConfTemplate", Pathname, :dns_server_conf_template
+arvcfg.declare_config "Containers.SLURM.Managed.DNSServerReloadCommand", String, :dns_server_reload_command
+arvcfg.declare_config "Containers.SLURM.Managed.DNSServerUpdateCommand", String, :dns_server_update_command
+arvcfg.declare_config "Containers.SLURM.Managed.ComputeNodeDomain", String, :compute_node_domain
+arvcfg.declare_config "Containers.SLURM.Managed.ComputeNodeNameservers", Array, :compute_node_nameservers
+arvcfg.declare_config "Containers.SLURM.Managed.AssignNodeHostname", String, :assign_node_hostname
+arvcfg.declare_config "Containers.JobsAPI.Enable", String, :enable_legacy_jobs_api, ->(cfg, k, v) { ConfigLoader.set_cfg cfg, "Containers.JobsAPI.Enable", v.to_s }
+arvcfg.declare_config "Containers.JobsAPI.CrunchJobWrapper", String, :crunch_job_wrapper
+arvcfg.declare_config "Containers.JobsAPI.CrunchJobUser", String, :crunch_job_user
+arvcfg.declare_config "Containers.JobsAPI.CrunchRefreshTrigger", String, :crunch_refresh_trigger
+arvcfg.declare_config "Containers.JobsAPI.GitInternalDir", String, :git_internal_dir
+arvcfg.declare_config "Containers.JobsAPI.ReuseJobIfOutputsDiffer", Boolean, :reuse_job_if_outputs_differ
+arvcfg.declare_config "Containers.JobsAPI.DefaultDockerImage", String, :default_docker_image_for_jobs
+arvcfg.declare_config "Mail.MailchimpAPIKey", String, :mailchimp_api_key
+arvcfg.declare_config "Mail.MailchimpListID", String, :mailchimp_list_id
+arvcfg.declare_config "Services.Workbench1.ExternalURL", URI, :workbench_address
+arvcfg.declare_config "Services.Websocket.ExternalURL", URI, :websocket_address
+arvcfg.declare_config "Services.WebDAV.ExternalURL", URI, :keep_web_service_url
+arvcfg.declare_config "Services.GitHTTP.ExternalURL", URI, :git_repo_https_base
+arvcfg.declare_config "Services.GitSSH.ExternalURL", URI, :git_repo_ssh_base, ->(cfg, k, v) { ConfigLoader.set_cfg cfg, "Services.GitSSH.ExternalURL", "ssh://#{v}" }
+arvcfg.declare_config "RemoteClusters", Hash, :remote_hosts, ->(cfg, k, v) {
   h = {}
   v.each do |clusterid, host|
     h[clusterid] = {
@@ -129,9 +131,20 @@ declare_config "RemoteClusters", Hash, :remote_hosts, ->(cfg, k, v) {
       "ActivateUsers" => false
     }
   end
-  set_cfg cfg, "RemoteClusters", h
+  ConfigLoader.set_cfg cfg, "RemoteClusters", h
 }
-declare_config "RemoteClusters.*.Proxy", Boolean, :remote_hosts_via_dns
+arvcfg.declare_config "RemoteClusters.*.Proxy", Boolean, :remote_hosts_via_dns
+
+dbcfg = ConfigLoader.new
+
+dbcfg.declare_config "PostgreSQL.ConnectionPool", Integer, :pool
+dbcfg.declare_config "PostgreSQL.Connection.Host", String, :host
+dbcfg.declare_config "PostgreSQL.Connection.Port", Integer, :port
+dbcfg.declare_config "PostgreSQL.Connection.User", String, :username
+dbcfg.declare_config "PostgreSQL.Connection.Password", String, :password
+dbcfg.declare_config "PostgreSQL.Connection.DBName", String, :database
+dbcfg.declare_config "PostgreSQL.Connection.Template", String, :template
+dbcfg.declare_config "PostgreSQL.Connection.Encoding", String, :encoding
 
 application_config = {}
 %w(application.default application).each do |cfgfile|
@@ -146,7 +159,16 @@ application_config = {}
   end
 end
 
-$remaining_config = migrate_config application_config, $arvados_config
+db_config = {}
+path = "#{::Rails.root.to_s}/config/database.ymlx"
+if File.exist? path
+  yaml = ERB.new(IO.read path).result(binding)
+  confs = YAML.load(yaml, deserialize_symbols: true)
+  db_config.merge!(confs[::Rails.env.to_s] || {})
+end
+
+$remaining_config = arvcfg.migrate_config(application_config, $arvados_config)
+dbcfg.migrate_config(db_config, $arvados_config)
 
 if application_config[:auto_activate_users_from]
   application_config[:auto_activate_users_from].each do |cluster|
@@ -158,10 +180,25 @@ end
 
 # Checks for wrongly typed configuration items, and essential items
 # that can't be empty
-coercion_and_check $arvados_config
+arvcfg.coercion_and_check $arvados_config
+dbcfg.coercion_and_check $arvados_config
 
 Server::Application.configure do
-  copy_into_config $arvados_config, config
-  copy_into_config $remaining_config, config
+  ConfigLoader.copy_into_config $arvados_config, config
+  ConfigLoader.copy_into_config $remaining_config, config
   config.secret_key_base = config.secret_token
+
+  dbcfg = {}
+  dbcfg[::Rails.env.to_s] = {
+    adapter: 'postgresql',
+    template: $arvados_config["PostgreSQL"]["Connection"]["Template"],
+    encoding: $arvados_config["PostgreSQL"]["Connection"]["Encoding"],
+    database: $arvados_config["PostgreSQL"]["Connection"]["DBName"],
+    username: $arvados_config["PostgreSQL"]["Connection"]["User"],
+    password: $arvados_config["PostgreSQL"]["Connection"]["Password"],
+    host: $arvados_config["PostgreSQL"]["Connection"]["Host"],
+    port: $arvados_config["PostgreSQL"]["Connection"]["Port"],
+    pool: $arvados_config["PostgreSQL"]["ConnectionPool"]
+  }
+  Rails.application.config.database_configuration = dbcfg
 end

diff --git a/services/api/lib/config_loader.rb b/services/api/lib/config_loader.rb
index 8b31a62c4056382fe7bb7ba9cd3d152b8ac2517c..2d1ddd8b8672feb3fb8f0d7460667f9c5bf62002 100644 (file)
--- a/services/api/lib/config_loader.rb
+++ b/services/api/lib/config_loader.rb
@@ -40,31 +40,6 @@ module Psych
   end
 end
 
-def set_cfg cfg, k, v
-  # "foo.bar = baz" --> { cfg["foo"]["bar"] = baz }
-  ks = k.split '.'
-  k = ks.pop
-  ks.each do |kk|
-    cfg = cfg[kk]
-    if cfg.nil?
-      break
-    end
-  end
-  if !cfg.nil?
-    cfg[k] = v
-  end
-end
-
-$config_migrate_map = {}
-$config_types = {}
-def declare_config(assign_to, configtype, migrate_from=nil, migrate_fn=nil)
-  if migrate_from
-    $config_migrate_map[migrate_from] = migrate_fn || ->(cfg, k, v) {
-      set_cfg cfg, assign_to, v
-    }
-  end
-  $config_types[assign_to] = configtype
-end
 
 module Boolean; end
 class TrueClass; include Boolean; end
@@ -73,100 +48,132 @@ class FalseClass; include Boolean; end
 class NonemptyString < String
 end
 
-def parse_duration durstr
-  duration_re = /(\d+(\.\d+)?)(s|m|h)/
-  dursec = 0
-  while durstr != ""
-    mt = duration_re.match durstr
-    if !mt
-      raise "#{cfgkey} not a valid duration: '#{cfg[k]}', accepted suffixes are s, m, h"
+class ConfigLoader
+  def initialize
+    @config_migrate_map = {}
+    @config_types = {}
+  end
+
+  def declare_config(assign_to, configtype, migrate_from=nil, migrate_fn=nil)
+    if migrate_from
+      @config_migrate_map[migrate_from] = migrate_fn || ->(cfg, k, v) {
+        ConfigLoader.set_cfg cfg, assign_to, v
+      }
     end
-    multiplier = {s: 1, m: 60, h: 3600}
-    dursec += (Float(mt[1]) * multiplier[mt[3].to_sym])
-    durstr = durstr[mt[0].length..-1]
+    @config_types[assign_to] = configtype
   end
-  return dursec.seconds
-end
 
-def migrate_config from_config, to_config
-  remainders = {}
-  from_config.each do |k, v|
-    if $config_migrate_map[k.to_sym]
-      $config_migrate_map[k.to_sym].call to_config, k, v
-    else
-      remainders[k] = v
+
+  def migrate_config from_config, to_config
+    remainders = {}
+    from_config.each do |k, v|
+      if @config_migrate_map[k.to_sym]
+        @config_migrate_map[k.to_sym].call to_config, k, v
+      else
+        remainders[k] = v
+      end
     end
+    remainders
   end
-  remainders
-end
 
-def coercion_and_check check_cfg
-  $config_types.each do |cfgkey, cfgtype|
-    cfg = check_cfg
-    k = cfgkey
-    ks = k.split '.'
-    k = ks.pop
-    ks.each do |kk|
-      cfg = cfg[kk]
+  def coercion_and_check check_cfg
+    @config_types.each do |cfgkey, cfgtype|
+      cfg = check_cfg
+      k = cfgkey
+      ks = k.split '.'
+      k = ks.pop
+      ks.each do |kk|
+        cfg = cfg[kk]
+        if cfg.nil?
+          break
+        end
+      end
+
       if cfg.nil?
-        break
+        raise "missing #{cfgkey}"
       end
-    end
 
-    if cfg.nil?
-      raise "missing #{cfgkey}"
-    end
+      if cfgtype == String and !cfg[k]
+        cfg[k] = ""
+      end
 
-    if cfgtype == String and !cfg[k]
-      cfg[k] = ""
-    end
+      if cfgtype == String and cfg[k].is_a? Symbol
+        cfg[k] = cfg[k].to_s
+      end
 
-    if cfgtype == String and cfg[k].is_a? Symbol
-      cfg[k] = cfg[k].to_s
-    end
+      if cfgtype == Pathname and cfg[k].is_a? String
 
-    if cfgtype == Pathname and cfg[k].is_a? String
+        if cfg[k] == ""
+          cfg[k] = Pathname.new("")
+        else
+          cfg[k] = Pathname.new(cfg[k])
+          if !cfg[k].exist?
+            raise "#{cfgkey} path #{cfg[k]} does not exist"
+          end
+        end
+      end
 
-      if cfg[k] == ""
-        cfg[k] = Pathname.new("")
-      else
-        cfg[k] = Pathname.new(cfg[k])
-        if !cfg[k].exist?
-          raise "#{cfgkey} path #{cfg[k]} does not exist"
+      if cfgtype == NonemptyString
+        if (!cfg[k] || cfg[k] == "")
+          raise "#{cfgkey} cannot be empty"
+        end
+        if cfg[k].is_a? String
+          next
         end
       end
-    end
 
-    if cfgtype == NonemptyString
-      if (!cfg[k] || cfg[k] == "")
-        raise "#{cfgkey} cannot be empty"
+      if cfgtype == ActiveSupport::Duration
+        if cfg[k].is_a? Integer
+          cfg[k] = cfg[k].seconds
+        elsif cfg[k].is_a? String
+          cfg[k] = ConfigLoader.parse_duration cfg[k]
+        end
       end
-      if cfg[k].is_a? String
-        next
+
+      if cfgtype == URI
+        cfg[k] = URI(cfg[k])
       end
-    end
 
-    if cfgtype == ActiveSupport::Duration
-      if cfg[k].is_a? Integer
-        cfg[k] = cfg[k].seconds
-      elsif cfg[k].is_a? String
-        cfg[k] = parse_duration cfg[k]
+      if !cfg[k].is_a? cfgtype
+        raise "#{cfgkey} expected #{cfgtype} but was #{cfg[k].class}"
       end
     end
+  end
 
-    if cfgtype == URI
-      cfg[k] = URI(cfg[k])
+  def self.set_cfg cfg, k, v
+    # "foo.bar = baz" --> { cfg["foo"]["bar"] = baz }
+    ks = k.split '.'
+    k = ks.pop
+    ks.each do |kk|
+      cfg = cfg[kk]
+      if cfg.nil?
+        break
+      end
     end
-
-    if !cfg[k].is_a? cfgtype
-      raise "#{cfgkey} expected #{cfgtype} but was #{cfg[k].class}"
+    if !cfg.nil?
+      cfg[k] = v
     end
   end
 
-end
+  def self.parse_duration durstr
+    duration_re = /(\d+(\.\d+)?)(s|m|h)/
+    dursec = 0
+    while durstr != ""
+      mt = duration_re.match durstr
+      if !mt
+        raise "#{cfgkey} not a valid duration: '#{cfg[k]}', accepted suffixes are s, m, h"
+      end
+      multiplier = {s: 1, m: 60, h: 3600}
+      dursec += (Float(mt[1]) * multiplier[mt[3].to_sym])
+      durstr = durstr[mt[0].length..-1]
+    end
+    return dursec.seconds
+  end
 
-def copy_into_config src, dst
-  src.each do |k, v|
-    dst.send "#{k}=", Marshal.load(Marshal.dump v)
+  def self.copy_into_config src, dst
+    src.each do |k, v|
+      dst.send "#{k}=", Marshal.load(Marshal.dump v)
+    end
   end
+
 end

diff --git a/services/api/test/functional/arvados/v1/repositories_controller_test.rb b/services/api/test/functional/arvados/v1/repositories_controller_test.rb
index d92561fc102c142e023d2c37ad909083148459db..537fe525270333317cf6ef1fb77c53a6c035dce2 100644 (file)
--- a/services/api/test/functional/arvados/v1/repositories_controller_test.rb
+++ b/services/api/test/functional/arvados/v1/repositories_controller_test.rb
@@ -208,7 +208,7 @@ class Arvados::V1::RepositoriesControllerTest < ActionController::TestCase
     {cfg: "GitHTTP", cfgval: false, refute: /^http/ },
   ].each do |expect|
     test "set #{expect[:cfg]} to #{expect[:cfgval]}" do
-      set_cfg Rails.configuration.Services, expect[:cfg].to_s, expect[:cfgval]
+      ConfigLoader.set_cfg Rails.configuration.Services, expect[:cfg].to_s, expect[:cfgval]
       authorize_with :active
       get :index
       assert_response :success

diff --git a/services/api/test/test_helper.rb b/services/api/test/test_helper.rb
index e87a1c6d8a9daa353b01e683ef58e67eed9884c4..5747a85cf598965d20b563c918a304b01f9dce87 100644 (file)
--- a/services/api/test/test_helper.rb
+++ b/services/api/test/test_helper.rb
@@ -99,8 +99,8 @@ class ActiveSupport::TestCase
 
   def restore_configuration
     # Restore configuration settings changed during tests
-    copy_into_config $arvados_config, Rails.configuration
-    copy_into_config $remaining_config, Rails.configuration
+    ConfigLoader.copy_into_config $arvados_config, Rails.configuration
+    ConfigLoader.copy_into_config $remaining_config, Rails.configuration
   end
 
   def set_user_from_auth(auth_name)