services/api/config/initializers/load_config.rb

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: AGPL-3.0
   4
   5 begin
   6   # If secret_token.rb exists here, we need to load it first.
   7   require_relative 'secret_token.rb'
   8 rescue LoadError
   9   # Normally secret_token.rb is missing and the secret token is
  10   # configured by application.yml (i.e., here!) instead.
  11 end
  12
  13 if (File.exist?(File.expand_path '../omniauth.rb', __FILE__) and
  14     not defined? WARNED_OMNIAUTH_CONFIG)
  15   Rails.logger.warn <<-EOS
  16 DEPRECATED CONFIGURATION:
  17  Please move your SSO provider config into config/application.yml
  18  and delete config/initializers/omniauth.rb.
  19 EOS
  20   # Real values will be copied from globals by omniauth_init.rb. For
  21   # now, assign some strings so the generic *.yml config loader
  22   # doesn't overwrite them or complain that they're missing.
  23   Rails.configuration.Login["ProviderAppID"] = 'xxx'
  24   Rails.configuration.Login["ProviderAppSecret"] = 'xxx'
  25   Rails.configuration.Services["SSO"]["ExternalURL"] = '//xxx'
  26   WARNED_OMNIAUTH_CONFIG = true
  27 end
  28
  29 $arvados_config = {}
  30
  31 ["#{::Rails.root.to_s}/config/config.defaults.yml", "/etc/arvados/config.yml"].each do |path|
  32   if File.exist? path
  33     confs = YAML.load(IO.read(path), deserialize_symbols: false)
  34     if confs
  35       clusters = confs["Clusters"].first
  36       $arvados_config["ClusterID"] = clusters[0]
  37       $arvados_config.merge!(clusters[1])
  38     end
  39   end
  40 end
  41
  42 def set_cfg cfg, k, v
  43   # "foo.bar: baz" --> { config.foo.bar = baz }
  44   ks = k.split '.'
  45   k = ks.pop
  46   ks.each do |kk|
  47     cfg = cfg[kk]
  48     if cfg.nil?
  49       break
  50     end
  51   end
  52   if !cfg.nil?
  53     cfg[k] = v
  54   end
  55 end
  56
  57 $config_migrate_map = {}
  58 $config_types = {}
  59 def declare_config(assign_to, configtype, migrate_from=nil)
  60   if migrate_from
  61     $config_migrate_map[migrate_from] = ->(cfg, k, v) {
  62       set_cfg cfg, assign_to, v
  63     }
  64   end
  65   $config_types[assign_to] = configtype
  66 end
  67
  68 module Boolean; end
  69 class TrueClass; include Boolean; end
  70 class FalseClass; include Boolean; end
  71
  72 declare_config "ClusterID", String, :uuid_prefix
  73 declare_config "Git.Repositories", String, :git_repositories_dir
  74 declare_config "API.DisabledAPIs", Array, :disable_api_methods
  75 declare_config "API.MaxRequestSize", Integer, :max_request_size
  76 declare_config "API.MaxIndexDatabaseRead", Integer, :max_index_database_read
  77 declare_config "API.MaxItemsPerResponse", Integer, :max_items_per_response
  78 declare_config "API.AsyncPermissionsUpdateInterval", ActiveSupport::Duration, :async_permissions_update_interval
  79 declare_config "Users.AutoSetupNewUsers", Boolean, :auto_setup_new_users
  80 declare_config "Users.AutoSetupNewUsersWithVmUUID", String, :auto_setup_new_users_with_vm_uuid
  81 declare_config "Users.AutoSetupNewUsersWithRepository", Boolean, :auto_setup_new_users_with_repository
  82 declare_config "Users.AutoSetupUsernameBlacklist", Array, :auto_setup_name_blacklist
  83 declare_config "Users.NewUsersAreActive", Boolean, :new_users_are_active
  84 declare_config "Users.AutoAdminUserWithEmail", String, :auto_admin_user
  85 declare_config "Users.AutoAdminFirstUser", Boolean, :auto_admin_first_user
  86 declare_config "Users.UserProfileNotificationAddress", String, :user_profile_notification_address
  87 declare_config "Users.AdminNotifierEmailFrom", String, :admin_notifier_email_from
  88 declare_config "Users.EmailSubjectPrefix", String, :email_subject_prefix
  89 declare_config "Users.UserNotifierEmailFrom", String, :user_notifier_email_from
  90 declare_config "Users.NewUserNotificationRecipients", Array, :new_user_notification_recipients
  91 declare_config "Users.NewInactiveUserNotificationRecipients", Array, :new_inactive_user_notification_recipients
  92 declare_config "Login.ProviderAppSecret", String, :sso_app_secret
  93 declare_config "Login.ProviderAppID", String, :sso_app_id
  94 declare_config "TLS.Insecure", Boolean, :sso_insecure
  95 declare_config "Services.SSO.ExternalURL", String, :sso_provider_url
  96 declare_config "AuditLogs.MaxAge", ActiveSupport::Duration, :max_audit_log_age
  97 declare_config "AuditLogs.MaxDeleteBatch", Integer, :max_audit_log_delete_batch
  98 declare_config "AuditLogs.UnloggedAttributes", Array, :unlogged_attributes
  99 declare_config "SystemLogs.MaxRequestLogParamsSize", Integer, :max_request_log_params_size
 100 declare_config "Collections.DefaultReplication", Integer, :default_collection_replication
 101 declare_config "Collections.DefaultTrashLifetime", ActiveSupport::Duration, :default_trash_lifetime
 102 declare_config "Collections.CollectionVersioning", Boolean, :collection_versioning
 103 declare_config "Collections.PreserveVersionIfIdle", ActiveSupport::Duration, :preserve_version_if_idle
 104 declare_config "Collections.TrashSweepInterval", ActiveSupport::Duration, :trash_sweep_interval
 105 declare_config "Collections.BlobSigningKey", String, :blob_signing_key
 106 declare_config "Collections.BlobSigningTTL", Integer, :blob_signature_ttl
 107 declare_config "Collections.BlobSigning", Boolean, :permit_create_collection_with_unsigned_manifest
 108 declare_config "Containers.SupportedDockerImageFormats", Array, :docker_image_formats
 109 declare_config "Containers.LogReuseDecisions", Boolean, :log_reuse_decisions
 110 declare_config "Containers.DefaultKeepCacheRAM", Integer, :container_default_keep_cache_ram
 111 declare_config "Containers.MaxDispatchAttempts", Integer, :max_container_dispatch_attempts
 112 declare_config "Containers.MaxRetryAttempts", Integer, :container_count_max
 113 declare_config "Containers.UsePreemptibleInstances", Boolean, :preemptible_instances
 114 declare_config "Containers.MaxComputeVMs", Integer, :max_compute_nodes
 115 declare_config "Containers.Logging.LogBytesPerEvent", Integer, :crunch_log_bytes_per_event
 116 declare_config "Containers.Logging.LogSecondsBetweenEvents", ActiveSupport::Duration, :crunch_log_seconds_between_events
 117 declare_config "Containers.Logging.LogThrottlePeriod", ActiveSupport::Duration, :crunch_log_throttle_period
 118 declare_config "Containers.Logging.LogThrottleBytes", Integer, :crunch_log_throttle_bytes
 119 declare_config "Containers.Logging.LogThrottleLines", Integer, :crunch_log_throttle_lines
 120 declare_config "Containers.Logging.LimitLogBytesPerJob", Integer, :crunch_limit_log_bytes_per_job
 121 declare_config "Containers.Logging.LogPartialLineThrottlePeriod", ActiveSupport::Duration, :crunch_log_partial_line_throttle_period
 122 declare_config "Containers.Logging.LogUpdatePeriod", ActiveSupport::Duration, :crunch_log_update_period
 123 declare_config "Containers.Logging.LogUpdateSize", Integer, :crunch_log_update_size
 124 declare_config "Containers.Logging.MaxAge", ActiveSupport::Duration, :clean_container_log_rows_after
 125 declare_config "Containers.SLURM.Managed.DNSServerConfDir", String, :dns_server_conf_dir
 126 declare_config "Containers.SLURM.Managed.DNSServerConfTemplate", String, :dns_server_conf_template
 127 declare_config "Containers.SLURM.Managed.DNSServerReloadCommand", String, :dns_server_reload_command
 128 declare_config "Containers.SLURM.Managed.DNSServerUpdateCommand", String, :dns_server_update_command
 129 declare_config "Containers.SLURM.Managed.ComputeNodeDomain", String, :compute_node_domain
 130 declare_config "Containers.SLURM.Managed.ComputeNodeNameservers", Array, :compute_node_nameservers
 131 declare_config "Containers.SLURM.Managed.AssignNodeHostname", String, :assign_node_hostname
 132 declare_config "Containers.JobsAPI.Enable", String, :enable_legacy_jobs_api
 133 declare_config "Containers.JobsAPI.CrunchJobWrapper", String, :crunch_job_wrapper
 134 declare_config "Containers.JobsAPI.CrunchJobUser", String, :crunch_job_user
 135 declare_config "Containers.JobsAPI.CrunchRefreshTrigger", String, :crunch_refresh_trigger
 136 declare_config "Containers.JobsAPI.GitInternalDir", String, :git_internal_dir
 137 declare_config "Containers.JobsAPI.ReuseJobIfOutputsDiffer", Boolean, :reuse_job_if_outputs_differ
 138 declare_config "Containers.JobsAPI.DefaultDockerImage", String, :default_docker_image_for_jobs
 139 declare_config "Mail.MailchimpAPIKey", String, :mailchimp_api_key
 140 declare_config "Mail.MailchimpListID", String, :mailchimp_list_id
 141 declare_config "Services.Workbench1.ExternalURL", String, :workbench_address
 142 declare_config "Services.Websocket.ExternalURL", String, :websocket_address
 143 declare_config "Services.WebDAV.ExternalURL", String, :keep_web_service_url
 144 declare_config "Services.GitHTTP.ExternalURL", String, :git_repo_https_base
 145 declare_config "Services.GitSSH.ExternalURL", String, :git_repo_ssh_base
 146
 147 application_config = {}
 148 %w(application.default application).each do |cfgfile|
 149   path = "#{::Rails.root.to_s}/config/#{cfgfile}.yml"
 150   if File.exist? path
 151     yaml = ERB.new(IO.read path).result(binding)
 152     confs = YAML.load(yaml, deserialize_symbols: true)
 153     # Ignore empty YAML file:
 154     next if confs == false
 155     application_config.merge!(confs['common'] || {})
 156     application_config.merge!(confs[::Rails.env.to_s] || {})
 157   end
 158 end
 159
 160 application_config.each do |k, v|
 161   if $config_migrate_map[k.to_sym]
 162     $config_migrate_map[k.to_sym].call $arvados_config, k, v
 163   else
 164     set_cfg $arvados_config, k, v
 165   end
 166 end
 167
 168 $config_types.each do |cfgkey, cfgtype|
 169   cfg = $arvados_config
 170   k = cfgkey
 171   ks = k.split '.'
 172   k = ks.pop
 173   ks.each do |kk|
 174     cfg = cfg[kk]
 175     if cfg.nil?
 176       break
 177     end
 178   end
 179   if cfgtype == String and !cfg[k]
 180     cfg[k] = ""
 181   end
 182   if cfgtype == ActiveSupport::Duration
 183     if cfg[k].is_a? Integer
 184       cfg[k] = cfg[k].seconds
 185     elsif cfg[k].is_a? String
 186       # TODO handle suffixes
 187     end
 188   end
 189
 190   if cfg.nil?
 191     raise "missing #{cfgkey}"
 192   end
 193
 194   if !cfg[k].is_a? cfgtype
 195     raise "#{cfgkey} expected #{cfgtype} but was #{cfg[k].class}"
 196   end
 197 end
 198
 199 Server::Application.configure do
 200   nils = []
 201   $arvados_config.each do |k, v|
 202     cfg = config
 203     if cfg.respond_to?(k.to_sym) and !cfg.send(k).nil?
 204     # Config must have been set already in environments/*.rb.
 205     #
 206     # After config files have been migrated, this mechanism should
 207     # be deprecated, then removed.
 208     elsif v.nil?
 209       # Config variables are not allowed to be nil. Make a "naughty"
 210       # list, and present it below.
 211       nils << k
 212     else
 213       cfg.send "#{k}=", v
 214     end
 215   end
 216   if !nils.empty?
 217     raise <<EOS
 218 Refusing to start in #{::Rails.env.to_s} mode with missing configuration.
 219
 220 The following configuration settings must be specified in
 221 config/application.yml:
 222 * #{nils.join "\n* "}
 223
 224 EOS
 225   end
 226   config.secret_key_base = config.secret_token
 227 end