services/api/config/initializers/load_config.rb

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: AGPL-3.0
   4
   5 begin
   6   # If secret_token.rb exists here, we need to load it first.
   7   require_relative 'secret_token.rb'
   8 rescue LoadError
   9   # Normally secret_token.rb is missing and the secret token is
  10   # configured by application.yml (i.e., here!) instead.
  11 end
  12
  13 if (File.exist?(File.expand_path '../omniauth.rb', __FILE__) and
  14     not defined? WARNED_OMNIAUTH_CONFIG)
  15   Rails.logger.warn <<-EOS
  16 DEPRECATED CONFIGURATION:
  17  Please move your SSO provider config into config/application.yml
  18  and delete config/initializers/omniauth.rb.
  19 EOS
  20   # Real values will be copied from globals by omniauth_init.rb. For
  21   # now, assign some strings so the generic *.yml config loader
  22   # doesn't overwrite them or complain that they're missing.
  23   Rails.configuration.Login["ProviderAppID"] = 'xxx'
  24   Rails.configuration.Login["ProviderAppSecret"] = 'xxx'
  25   Rails.configuration.Services["SSO"]["ExternalURL"] = '//xxx'
  26   WARNED_OMNIAUTH_CONFIG = true
  27 end
  28
  29 $arvados_config = {}
  30
  31 ["#{::Rails.root.to_s}/config/config.defaults.yml", "/etc/arvados/config.yml"].each do |path|
  32   if File.exist? path
  33     confs = YAML.load(IO.read(path), deserialize_symbols: false)
  34     if confs
  35       clusters = confs["Clusters"].first
  36       $arvados_config["ClusterID"] = clusters[0]
  37       $arvados_config.merge!(clusters[1])
  38     end
  39   end
  40 end
  41
  42 def set_cfg cfg, k, v
  43   # "foo.bar: baz" --> { config.foo.bar = baz }
  44   ks = k.split '.'
  45   k = ks.pop
  46   ks.each do |kk|
  47     cfg = cfg[kk]
  48     if cfg.nil?
  49       break
  50     end
  51   end
  52   if !cfg.nil?
  53     cfg[k] = v
  54   end
  55 end
  56
  57 $config_migrate_map = {}
  58 $config_types = {}
  59 def declare_config(assign_to, configtype, migrate_from=nil, migrate_fn=nil)
  60   if migrate_from
  61     $config_migrate_map[migrate_from] = migrate_fn || ->(cfg, k, v) {
  62       set_cfg cfg, assign_to, v
  63     }
  64   end
  65   $config_types[assign_to] = configtype
  66 end
  67
  68 module Boolean; end
  69 class TrueClass; include Boolean; end
  70 class FalseClass; include Boolean; end
  71
  72 declare_config "ClusterID", String, :uuid_prefix
  73 declare_config "Git.Repositories", String, :git_repositories_dir
  74 declare_config "API.DisabledAPIs", Array, :disable_api_methods
  75 declare_config "API.MaxRequestSize", Integer, :max_request_size
  76 declare_config "API.MaxIndexDatabaseRead", Integer, :max_index_database_read
  77 declare_config "API.MaxItemsPerResponse", Integer, :max_items_per_response
  78 declare_config "API.AsyncPermissionsUpdateInterval", ActiveSupport::Duration, :async_permissions_update_interval
  79 declare_config "Users.AutoSetupNewUsers", Boolean, :auto_setup_new_users
  80 declare_config "Users.AutoSetupNewUsersWithVmUUID", String, :auto_setup_new_users_with_vm_uuid
  81 declare_config "Users.AutoSetupNewUsersWithRepository", Boolean, :auto_setup_new_users_with_repository
  82 declare_config "Users.AutoSetupUsernameBlacklist", Array, :auto_setup_name_blacklist
  83 declare_config "Users.NewUsersAreActive", Boolean, :new_users_are_active
  84 declare_config "Users.AutoAdminUserWithEmail", String, :auto_admin_user
  85 declare_config "Users.AutoAdminFirstUser", Boolean, :auto_admin_first_user
  86 declare_config "Users.UserProfileNotificationAddress", String, :user_profile_notification_address
  87 declare_config "Users.AdminNotifierEmailFrom", String, :admin_notifier_email_from
  88 declare_config "Users.EmailSubjectPrefix", String, :email_subject_prefix
  89 declare_config "Users.UserNotifierEmailFrom", String, :user_notifier_email_from
  90 declare_config "Users.NewUserNotificationRecipients", Array, :new_user_notification_recipients
  91 declare_config "Users.NewInactiveUserNotificationRecipients", Array, :new_inactive_user_notification_recipients
  92 declare_config "Login.ProviderAppSecret", String, :sso_app_secret
  93 declare_config "Login.ProviderAppID", String, :sso_app_id
  94 declare_config "TLS.Insecure", Boolean, :sso_insecure
  95 declare_config "Services.SSO.ExternalURL", String, :sso_provider_url
  96 declare_config "AuditLogs.MaxAge", ActiveSupport::Duration, :max_audit_log_age
  97 declare_config "AuditLogs.MaxDeleteBatch", Integer, :max_audit_log_delete_batch
  98 declare_config "AuditLogs.UnloggedAttributes", Array, :unlogged_attributes
  99 declare_config "SystemLogs.MaxRequestLogParamsSize", Integer, :max_request_log_params_size
 100 declare_config "Collections.DefaultReplication", Integer, :default_collection_replication
 101 declare_config "Collections.DefaultTrashLifetime", ActiveSupport::Duration, :default_trash_lifetime
 102 declare_config "Collections.CollectionVersioning", Boolean, :collection_versioning
 103 declare_config "Collections.PreserveVersionIfIdle", ActiveSupport::Duration, :preserve_version_if_idle
 104 declare_config "Collections.TrashSweepInterval", ActiveSupport::Duration, :trash_sweep_interval
 105 declare_config "Collections.BlobSigningKey", String, :blob_signing_key
 106 declare_config "Collections.BlobSigningTTL", Integer, :blob_signature_ttl
 107 declare_config "Collections.BlobSigning", Boolean, :permit_create_collection_with_unsigned_manifest
 108 declare_config "Containers.SupportedDockerImageFormats", Array, :docker_image_formats
 109 declare_config "Containers.LogReuseDecisions", Boolean, :log_reuse_decisions
 110 declare_config "Containers.DefaultKeepCacheRAM", Integer, :container_default_keep_cache_ram
 111 declare_config "Containers.MaxDispatchAttempts", Integer, :max_container_dispatch_attempts
 112 declare_config "Containers.MaxRetryAttempts", Integer, :container_count_max
 113 declare_config "Containers.UsePreemptibleInstances", Boolean, :preemptible_instances
 114 declare_config "Containers.MaxComputeVMs", Integer, :max_compute_nodes
 115 declare_config "Containers.Logging.LogBytesPerEvent", Integer, :crunch_log_bytes_per_event
 116 declare_config "Containers.Logging.LogSecondsBetweenEvents", ActiveSupport::Duration, :crunch_log_seconds_between_events
 117 declare_config "Containers.Logging.LogThrottlePeriod", ActiveSupport::Duration, :crunch_log_throttle_period
 118 declare_config "Containers.Logging.LogThrottleBytes", Integer, :crunch_log_throttle_bytes
 119 declare_config "Containers.Logging.LogThrottleLines", Integer, :crunch_log_throttle_lines
 120 declare_config "Containers.Logging.LimitLogBytesPerJob", Integer, :crunch_limit_log_bytes_per_job
 121 declare_config "Containers.Logging.LogPartialLineThrottlePeriod", ActiveSupport::Duration, :crunch_log_partial_line_throttle_period
 122 declare_config "Containers.Logging.LogUpdatePeriod", ActiveSupport::Duration, :crunch_log_update_period
 123 declare_config "Containers.Logging.LogUpdateSize", Integer, :crunch_log_update_size
 124 declare_config "Containers.Logging.MaxAge", ActiveSupport::Duration, :clean_container_log_rows_after
 125 declare_config "Containers.SLURM.Managed.DNSServerConfDir", String, :dns_server_conf_dir
 126 declare_config "Containers.SLURM.Managed.DNSServerConfTemplate", String, :dns_server_conf_template
 127 declare_config "Containers.SLURM.Managed.DNSServerReloadCommand", String, :dns_server_reload_command
 128 declare_config "Containers.SLURM.Managed.DNSServerUpdateCommand", String, :dns_server_update_command
 129 declare_config "Containers.SLURM.Managed.ComputeNodeDomain", String, :compute_node_domain
 130 declare_config "Containers.SLURM.Managed.ComputeNodeNameservers", Array, :compute_node_nameservers
 131 declare_config "Containers.SLURM.Managed.AssignNodeHostname", String, :assign_node_hostname
 132 declare_config "Containers.JobsAPI.Enable", String, :enable_legacy_jobs_api, ->(cfg, k, v) {
 133   set_cfg cfg, "Containers.JobsAPI.Enable", if v.is_a? Boolean then v.to_s else v end
 134 }
 135 declare_config "Containers.JobsAPI.CrunchJobWrapper", String, :crunch_job_wrapper
 136 declare_config "Containers.JobsAPI.CrunchJobUser", String, :crunch_job_user
 137 declare_config "Containers.JobsAPI.CrunchRefreshTrigger", String, :crunch_refresh_trigger
 138 declare_config "Containers.JobsAPI.GitInternalDir", String, :git_internal_dir
 139 declare_config "Containers.JobsAPI.ReuseJobIfOutputsDiffer", Boolean, :reuse_job_if_outputs_differ
 140 declare_config "Containers.JobsAPI.DefaultDockerImage", String, :default_docker_image_for_jobs
 141 declare_config "Mail.MailchimpAPIKey", String, :mailchimp_api_key
 142 declare_config "Mail.MailchimpListID", String, :mailchimp_list_id
 143 declare_config "Services.Workbench1.ExternalURL", String, :workbench_address
 144 declare_config "Services.Websocket.ExternalURL", String, :websocket_address
 145 declare_config "Services.WebDAV.ExternalURL", String, :keep_web_service_url
 146 declare_config "Services.GitHTTP.ExternalURL", String, :git_repo_https_base
 147 declare_config "Services.GitSSH.ExternalURL", String, :git_repo_ssh_base
 148
 149 application_config = {}
 150 %w(application.default application).each do |cfgfile|
 151   path = "#{::Rails.root.to_s}/config/#{cfgfile}.yml"
 152   if File.exist? path
 153     yaml = ERB.new(IO.read path).result(binding)
 154     confs = YAML.load(yaml, deserialize_symbols: true)
 155     # Ignore empty YAML file:
 156     next if confs == false
 157     application_config.merge!(confs['common'] || {})
 158     application_config.merge!(confs[::Rails.env.to_s] || {})
 159   end
 160 end
 161
 162 application_config.each do |k, v|
 163   if $config_migrate_map[k.to_sym]
 164     $config_migrate_map[k.to_sym].call $arvados_config, k, v
 165   else
 166     set_cfg $arvados_config, k, v
 167   end
 168 end
 169
 170 duration_re = /(\d+(\.\d+)?)(ms|s|m|h)/
 171
 172 $config_types.each do |cfgkey, cfgtype|
 173   cfg = $arvados_config
 174   k = cfgkey
 175   ks = k.split '.'
 176   k = ks.pop
 177   ks.each do |kk|
 178     cfg = cfg[kk]
 179     if cfg.nil?
 180       break
 181     end
 182   end
 183
 184   if cfg.nil?
 185     raise "missing #{cfgkey}"
 186   end
 187
 188   if cfgtype == String and !cfg[k]
 189     cfg[k] = ""
 190   end
 191   if cfgtype == ActiveSupport::Duration
 192     if cfg[k].is_a? Integer
 193       cfg[k] = cfg[k].seconds
 194     elsif cfg[k].is_a? String
 195       mt = duration_re.match cfg[k]
 196       if !mt
 197         raise "#{cfgkey} not a valid duration: '#{cfg[k]}', accepted suffixes are ms, s, m, h"
 198       end
 199       multiplier = {ms: 0.001, s: 1, m: 60, h: 3600}
 200       cfg[k] = (Float(mt[1]) * multiplier[mt[3].to_sym]).seconds
 201     end
 202   end
 203
 204   if !cfg[k].is_a? cfgtype
 205     raise "#{cfgkey} expected #{cfgtype} but was #{cfg[k].class}"
 206   end
 207 end
 208
 209 Server::Application.configure do
 210   nils = []
 211   $arvados_config.each do |k, v|
 212     cfg = config
 213     if cfg.respond_to?(k.to_sym) and !cfg.send(k).nil?
 214     # Config must have been set already in environments/*.rb.
 215     #
 216     # After config files have been migrated, this mechanism should
 217     # be deprecated, then removed.
 218     elsif v.nil?
 219       # Config variables are not allowed to be nil. Make a "naughty"
 220       # list, and present it below.
 221       nils << k
 222     else
 223       cfg.send "#{k}=", v
 224     end
 225   end
 226   if !nils.empty?
 227     raise <<EOS
 228 Refusing to start in #{::Rails.env.to_s} mode with missing configuration.
 229
 230 The following configuration settings must be specified in
 231 config/application.yml:
 232 * #{nils.join "\n* "}
 233
 234 EOS
 235   end
 236   config.secret_key_base = config.secret_token
 237 end