13996: Updating API server to use new config object WIP
authorPeter Amstutz <pamstutz@veritasgenetics.com>
Wed, 20 Mar 2019 21:33:51 +0000 (17:33 -0400)
committerPeter Amstutz <pamstutz@veritasgenetics.com>
Mon, 8 Apr 2019 15:05:14 +0000 (11:05 -0400)
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz@veritasgenetics.com>

14 files changed:
lib/config/config.defaults.yml
services/api/app/controllers/application_controller.rb
services/api/app/controllers/arvados/v1/groups_controller.rb
services/api/app/controllers/arvados/v1/schema_controller.rb
services/api/app/controllers/user_sessions_controller.rb
services/api/app/mailers/admin_notifier.rb
services/api/app/mailers/profile_notifier.rb
services/api/app/mailers/user_notifier.rb
services/api/app/models/arvados_model.rb
services/api/app/models/blob.rb
services/api/app/models/commit.rb
services/api/app/models/job.rb
services/api/app/models/repository.rb
services/api/app/models/user.rb

index ba9f7524ed7ffa1aab7eddbe846df4719bc909b4..53fc5d9cb1c8108d129967a1627f6c72616ac5b4 100644 (file)
@@ -329,42 +329,49 @@ Clusters:
           # (See http://ruby-doc.org/core-2.2.2/Kernel.html#method-i-format for more.)
           AssignNodeHostname: compute%<slot_number>d
 
-        JobsAPI:
-          # Enable the legacy Jobs API.
-          # auto -- (default) enable the Jobs API only if it has been used before
-          #         (i.e., there are job records in the database)
-          # true -- enable the Jobs API despite lack of existing records.
-          # false -- disable the Jobs API despite presence of existing records.
-          Enable: auto
-
-          # Git repositories must be readable by api server, or you won't be
-          # able to submit crunch jobs. To pass the test suites, put a clone
-          # of the arvados tree in {git_repositories_dir}/arvados.git or
-          # {git_repositories_dir}/arvados/.git
-          GitInternalDir: /var/lib/arvados/internal.git
-
-          # Docker image to be used when none found in runtime_constraints of a job
-          DefaultDockerImage: ""
-
-          # :none or :slurm_immediate
-          CrunchJobWrapper: :none
-
-          # username, or false = do not set uid when running jobs.
-          CrunchJobUser: crunch
-
-          # The web service must be able to create/write this file, and
-          # crunch-job must be able to stat() it.
-          CrunchRefreshTrigger: /tmp/crunch_refresh_trigger
-
-          # Control job reuse behavior when two completed jobs match the
-          # search criteria and have different outputs.
-          #
-          # If true, in case of a conflict, reuse the earliest job (this is
-          # similar to container reuse behavior).
-          #
-          # If false, in case of a conflict, do not reuse any completed job,
-          # but do reuse an already-running job if available (this is the
-          # original job reuse behavior, and is still the default).
-          ReuseJobIfOutputsDiffer: false
-
-      Mail: {}
+      JobsAPI:
+        # Enable the legacy Jobs API.
+        # auto -- (default) enable the Jobs API only if it has been used before
+        #         (i.e., there are job records in the database)
+        # true -- enable the Jobs API despite lack of existing records.
+        # false -- disable the Jobs API despite presence of existing records.
+        Enable: auto
+
+        # Git repositories must be readable by api server, or you won't be
+        # able to submit crunch jobs. To pass the test suites, put a clone
+        # of the arvados tree in {git_repositories_dir}/arvados.git or
+        # {git_repositories_dir}/arvados/.git
+        GitInternalDir: /var/lib/arvados/internal.git
+
+        # Docker image to be used when none found in runtime_constraints of a job
+        DefaultDockerImage: ""
+
+        # :none or :slurm_immediate
+        CrunchJobWrapper: :none
+
+        # username, or false = do not set uid when running jobs.
+        CrunchJobUser: crunch
+
+        # The web service must be able to create/write this file, and
+        # crunch-job must be able to stat() it.
+        CrunchRefreshTrigger: /tmp/crunch_refresh_trigger
+
+        # Control job reuse behavior when two completed jobs match the
+        # search criteria and have different outputs.
+        #
+        # If true, in case of a conflict, reuse the earliest job (this is
+        # similar to container reuse behavior).
+        #
+        # If false, in case of a conflict, do not reuse any completed job,
+        # but do reuse an already-running job if available (this is the
+        # original job reuse behavior, and is still the default).
+        ReuseJobIfOutputsDiffer: false
+
+      Mail:
+        MailchimpAPIKey:            # api-server/mailchimp_api_key
+        MailchimpListID:            # api-server/mailchimp_list_id
+        SendUserSetupNotificationEmail:  # workbench/send_user_setup_notification_email
+        IssueReporterEmailFrom:     # workbench/issue_reporter_email_from
+        IssueReporterEmailTo:       # workbench/issue_reporter_email_to
+        SupportEmailAddress:        # workbench/support_email_address
+        EmailFrom:                  # workbench/email_from
index b613e97a10133adf5be8160057d66b31ed3a84a6..78fea32b235de5915bb069b83177b50b2a551c9a 100644 (file)
@@ -306,7 +306,7 @@ class ApplicationController < ActionController::Base
       limit_query.each do |record|
         new_limit += 1
         read_total += record.read_length.to_i
-        if read_total >= Rails.configuration.max_index_database_read
+        if read_total >= Rails.configuration.API["MaxIndexDatabaseRead"]
           new_limit -= 1 if new_limit > 1
           @limit = new_limit
           break
@@ -419,7 +419,7 @@ class ApplicationController < ActionController::Base
   end
 
   def disable_api_methods
-    if Rails.configuration.disable_api_methods.
+    if Rails.configuration.API["DisabledAPIs"].
         include?(controller_name + "." + action_name)
       send_error("Disabled", status: 404)
     end
index f7db1ef121f606b597772ce695aadb859f0a3e31..825f9950dd10deb892310c82299fbef35ea45b02 100644 (file)
@@ -191,7 +191,7 @@ class Arvados::V1::GroupsController < ApplicationController
 
     table_names = Hash[klasses.collect { |k| [k, k.table_name] }]
 
-    disabled_methods = Rails.configuration.disable_api_methods
+    disabled_methods = Rails.configuration.API["DisabledAPIs"]
     avail_klasses = table_names.select{|k, t| !disabled_methods.include?(t+'.index')}
     klasses = avail_klasses.keys
 
index 8ff2a97c467d0f09f867e0992bcb4167c9953e7d..95e10498b2ede9417b633843d40d2bbd9539b0e7 100644 (file)
@@ -39,7 +39,7 @@ class Arvados::V1::SchemaController < ApplicationController
         title: "Arvados API",
         description: "The API to interact with Arvados.",
         documentationLink: "http://doc.arvados.org/api/index.html",
-        defaultCollectionReplication: Rails.configuration.default_collection_replication,
+        defaultCollectionReplication: Rails.configuration.Collections["DefaultReplication"],
         protocol: "rest",
         baseUrl: root_url + "arvados/v1/",
         basePath: "/arvados/v1/",
@@ -70,7 +70,7 @@ class Arvados::V1::SchemaController < ApplicationController
                 when false
                   ''
                 when true
-                  'https://git.%s.arvadosapi.com/' % Rails.configuration.uuid_prefix
+                  'https://git.%s.arvadosapi.com/' % Rails.configuration.ClusterID
                 else
                   Rails.application.config.git_repo_https_base
                 end,
@@ -405,7 +405,7 @@ class Arvados::V1::SchemaController < ApplicationController
           end
         end
       end
-      Rails.configuration.disable_api_methods.each do |method|
+      Rails.configuration.API["DisabledAPIs"].each do |method|
         ctrl, action = method.split('.', 2)
         discovery[:resources][ctrl][:methods].delete(action.to_sym)
       end
index 237156f1161ec53d53322cb3c9271c39ab945805..c5d3ae74f34be746b34bd79b7f49ae491ca8b90d 100644 (file)
@@ -52,7 +52,7 @@ class UserSessionsController < ApplicationController
                       :first_name => omniauth['info']['first_name'],
                       :last_name => omniauth['info']['last_name'],
                       :identity_url => omniauth['info']['identity_url'],
-                      :is_active => Rails.configuration.new_users_are_active,
+                      :is_active => Rails.configuration.Users["NewUsersAreActive"],
                       :owner_uuid => system_user_uuid)
       if omniauth['info']['username']
         user.set_initial_username(requested: omniauth['info']['username'])
index 87a5699f49a2061a37fd2ca3117fc0bc23ce750e..e454d64e039802c2d1b8a902448bbe08a54740cd 100644 (file)
@@ -5,32 +5,32 @@
 class AdminNotifier < ActionMailer::Base
   include AbstractController::Callbacks
 
-  default from: Rails.configuration.admin_notifier_email_from
+  default from: Rails.configuration.Users["AdminNotifierEmailFrom"]
 
   def new_user(user)
     @user = user
-    if not Rails.configuration.new_user_notification_recipients.empty? then
-      @recipients = Rails.configuration.new_user_notification_recipients
+    if not Rails.configuration.Users["NewUserNotificationRecipients"].empty? then
+      @recipients = Rails.configuration.Users["NewUserNotificationRecipients"]
       logger.info "Sending mail to #{@recipients} about new user #{@user.uuid} (#{@user.full_name} <#{@user.email}>)"
 
       add_to_subject = ''
-      if Rails.configuration.auto_setup_new_users
+      if Rails.configuration.Users["AutoSetupNewUsers"]
         add_to_subject = @user.is_invited ? ' and setup' : ', but not setup'
       end
 
       mail(to: @recipients,
-           subject: "#{Rails.configuration.email_subject_prefix}New user created#{add_to_subject} notification"
+           subject: "#{Rails.configuration.Users["EmailSubjectPrefix"]}New user created#{add_to_subject} notification"
           )
     end
   end
 
   def new_inactive_user(user)
     @user = user
-    if not Rails.configuration.new_inactive_user_notification_recipients.empty? then
-      @recipients = Rails.configuration.new_inactive_user_notification_recipients
+    if not Rails.configuration.Users["NewInactiveUserNotificationRecipients"].empty? then
+      @recipients = Rails.configuration.Users["NewInactiveUserNotificationRecipients"]
       logger.info "Sending mail to #{@recipients} about new user #{@user.uuid} (#{@user.full_name} <#{@user.email}>)"
       mail(to: @recipients,
-           subject: "#{Rails.configuration.email_subject_prefix}New inactive user notification"
+           subject: "#{Rails.configuration.Users["EmailSubjectPrefix"]}New inactive user notification"
           )
     end
   end
index 8c0c5ec863bccf5b91893cc7ef3bfbf5c3c42edd..a23d5f34509eb0394e27bf912d78fd9490a3d09e 100644 (file)
@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0
 
 class ProfileNotifier < ActionMailer::Base
-  default from: Rails.configuration.admin_notifier_email_from
+  default from: Rails.configuration.Users["AdminNotifierEmailFrom"]
 
   def profile_created(user, address)
     @user = user
index 5fb7036bf2596acb8425c2e6695b8a63ad259da8..dbde7a973a4e38879f3e50a7c55d2924455af223 100644 (file)
@@ -5,7 +5,7 @@
 class UserNotifier < ActionMailer::Base
   include AbstractController::Callbacks
 
-  default from: Rails.configuration.user_notifier_email_from
+  default from: Rails.configuration.Users["UserNotifierEmailFrom"]
 
   def account_is_setup(user)
     @user = user
index e619abe8c800781b17fc3c2c07f8fa69008ee6cf..00f60e6fba77999ea45758276eac80667bf6215e 100644 (file)
@@ -411,7 +411,7 @@ class ArvadosModel < ApplicationRecord
   end
 
   def logged_attributes
-    attributes.except(*Rails.configuration.unlogged_attributes)
+    attributes.except(*Rails.configuration.AuditLogs["UnloggedAttributes"])
   end
 
   def self.full_text_searchable_columns
@@ -735,7 +735,7 @@ class ArvadosModel < ApplicationRecord
   end
 
   def self.uuid_like_pattern
-    "#{Rails.configuration.uuid_prefix}-#{uuid_prefix}-_______________"
+    "#{Rails.configuration.ClusterID}-#{uuid_prefix}-_______________"
   end
 
   def self.uuid_regex
@@ -814,8 +814,8 @@ class ArvadosModel < ApplicationRecord
   end
 
   def is_audit_logging_enabled?
-    return !(Rails.configuration.max_audit_log_age.to_i == 0 &&
-             Rails.configuration.max_audit_log_delete_batch.to_i > 0)
+    return !(Rails.configuration.AuditLogs["MaxAge"].to_i == 0 &&
+             Rails.configuration.AuditLogs["MaxDeleteBatch"].to_i > 0)
   end
 
   def log_start_state
index 55a257856c989faa18bc70dfe075cc7d534b90e1..500a6627908b7e869bb6b18352fa22980066495b 100644 (file)
@@ -51,15 +51,15 @@ class Blob
       timestamp = opts[:expire]
     else
       timestamp = db_current_time.to_i +
-        (opts[:ttl] || Rails.configuration.blob_signature_ttl)
+        (opts[:ttl] || Rails.configuration.Collections["BlobSigningTTL"])
     end
     timestamp_hex = timestamp.to_s(16)
     # => "53163cb4"
-    blob_signature_ttl = Rails.configuration.blob_signature_ttl.to_s(16)
+    blob_signature_ttl = Rails.configuration.Collections["BlobSigningTTL"].to_s(16)
 
     # Generate a signature.
     signature =
-      generate_signature((opts[:key] or Rails.configuration.blob_signing_key),
+      generate_signature((opts[:key] or Rails.configuration.Collections["BlobSigningKey"]),
                          blob_hash, opts[:api_token], timestamp_hex, blob_signature_ttl)
 
     blob_locator + '+A' + signature + '@' + timestamp_hex
@@ -103,10 +103,10 @@ class Blob
     if timestamp.to_i(16) < (opts[:now] or db_current_time.to_i)
       raise Blob::InvalidSignatureError.new 'Signature expiry time has passed.'
     end
-    blob_signature_ttl = Rails.configuration.blob_signature_ttl.to_s(16)
+    blob_signature_ttl = Rails.configuration.Collections["BlobSigningTTL"].to_s(16)
 
     my_signature =
-      generate_signature((opts[:key] or Rails.configuration.blob_signing_key),
+      generate_signature((opts[:key] or Rails.configuration.Collections["BlobSigningKey"]),
                          blob_hash, opts[:api_token], timestamp, blob_signature_ttl)
 
     if my_signature != given_signature
index 921c690cd00f78f6fc2b46bbace23fff89992db8..ed3c5cd354091de038aada837713b931c8b9f757 100644 (file)
@@ -148,7 +148,7 @@ class Commit < ActiveRecord::Base
     unless src_gitdir
       raise ArgumentError.new "no local repository for #{repo_name}"
     end
-    dst_gitdir = Rails.configuration.git_internal_dir
+    dst_gitdir = Rails.configuration.Containers["JobsAPI"]["GitInternalDir"]
 
     begin
       commit_in_dst = must_git(dst_gitdir, "log -n1 --format=%H #{sha1.shellescape}^{commit}").strip
index 420386cdc2ac0b4cfa423bb423e9c6053ec8b4ba..deaa8ff71d027f529437577f08a80a8b8ba87731 100644 (file)
@@ -287,7 +287,7 @@ class Job < ArvadosModel
         log_reuse_info { "job #{j.uuid} has nil output" }
       elsif j.log.nil?
         log_reuse_info { "job #{j.uuid} has nil log" }
-      elsif Rails.configuration.reuse_job_if_outputs_differ
+      elsif Rails.configuration.Containers["JobsAPI"]["ReuseJobIfOutputsDiffer"]
         if !Collection.readable_by(current_user).find_by_portable_data_hash(j.output)
           # Ignore: keep looking for an incomplete job or one whose
           # output is readable.
@@ -493,7 +493,7 @@ class Job < ArvadosModel
   def find_docker_image_locator
     if runtime_constraints.is_a? Hash
       runtime_constraints['docker_image'] ||=
-        Rails.configuration.default_docker_image_for_jobs
+        Rails.configuration.Containers["JobsAPI"]["DefaultDockerImage"]
     end
 
     resolve_runtime_constraint("docker_image",
@@ -569,7 +569,7 @@ class Job < ArvadosModel
 
   def trigger_crunch_dispatch_if_cancelled
     if @need_crunch_dispatch_trigger
-      File.open(Rails.configuration.crunch_refresh_trigger, 'wb') do
+      File.open(Rails.configuration.Containers["JobsAPI"]["CrunchRefreshTrigger"], 'wb') do
         # That's all, just create/touch a file for crunch-job to see.
       end
     end
index 48655156c4c096fa406d3add1dcffa77b1707354..bde9d51d2af846efa333f29a6c41e7e2af783c69 100644 (file)
@@ -49,7 +49,7 @@ class Repository < ArvadosModel
     # prefers bare repositories over checkouts.
     [["%s.git"], ["%s", ".git"]].each do |repo_base, *join_args|
       [:uuid, :name].each do |path_attr|
-        git_dir = File.join(Rails.configuration.git_repositories_dir,
+        git_dir = File.join(Rails.configuration.Containers["Git"]["Repositories"],
                             repo_base % send(path_attr), *join_args)
         return git_dir if File.exist?(git_dir)
       end
@@ -108,8 +108,8 @@ class Repository < ArvadosModel
   def _clone_url config_var, default_base_fmt
     configured_base = Rails.configuration.send config_var
     return nil if configured_base == false
-    prefix = new_record? ? Rails.configuration.uuid_prefix : uuid[0,5]
-    if prefix == Rails.configuration.uuid_prefix and configured_base != true
+    prefix = new_record? ? Rails.configuration.ClusterID : uuid[0,5]
+    if prefix == Rails.configuration.ClusterID and configured_base != true
       base = configured_base
     else
       base = default_base_fmt % prefix
index de85cc5a8e75e32852407f9c8388777df1168119..49d3afe7be0d246a20a39c7a89fe43d8c499e518 100644 (file)
@@ -34,7 +34,7 @@ class User < ArvadosModel
   after_create :add_system_group_permission_link
   after_create :invalidate_permissions_cache
   after_create :auto_setup_new_user, :if => Proc.new { |user|
-    Rails.configuration.auto_setup_new_users and
+    Rails.configuration.Users["AutoSetupNewUsers"] and
     (user.uuid != system_user_uuid) and
     (user.uuid != anonymous_user_uuid)
   }
@@ -81,7 +81,7 @@ class User < ArvadosModel
 
   def is_invited
     !!(self.is_active ||
-       Rails.configuration.new_users_are_active ||
+       Rails.configuration.Users["NewUsersAreActive"] ||
        self.groups_i_can(:read).select { |x| x.match(/-f+$/) }.first)
   end
 
@@ -358,15 +358,15 @@ class User < ArvadosModel
     current_user.andand.is_admin or
       (self == current_user &&
        self.redirect_to_user_uuid.nil? &&
-       self.is_active == Rails.configuration.new_users_are_active)
+       self.is_active == Rails.configuration.Users["NewUsersAreActive"])
   end
 
   def check_auto_admin
     return if self.uuid.end_with?('anonymouspublic')
     if (User.where("email = ?",self.email).where(:is_admin => true).count == 0 and
-        Rails.configuration.auto_admin_user and self.email == Rails.configuration.auto_admin_user) or
+        Rails.configuration.Users["AutoAdminUserWithEmail"] and self.email == Rails.configuration.Users["AutoAdminUserWithEmail"]) or
        (User.where("uuid not like '%-000000000000000'").where(:is_admin => true).count == 0 and
-        Rails.configuration.auto_admin_first_user)
+        Rails.configuration.Users["AutoAdminFirstUser"])
       self.is_admin = true
       self.is_active = true
     end
@@ -381,7 +381,7 @@ class User < ArvadosModel
     quoted_name = self.class.connection.quote_string(basename)
     next_username = basename
     next_suffix = 1
-    while Rails.configuration.auto_setup_name_blacklist.include?(next_username)
+    while Rails.configuration.Users["AutoSetupUsernameBlacklist"].include?(next_username)
       next_suffix += 1
       next_username = "%s%i" % [basename, next_suffix]
     end
@@ -563,10 +563,10 @@ class User < ArvadosModel
   def auto_setup_new_user
     setup(openid_prefix: Rails.configuration.default_openid_prefix)
     if username
-      create_vm_login_permission_link(Rails.configuration.auto_setup_new_users_with_vm_uuid,
+      create_vm_login_permission_link(Rails.configuration.Users["AutoSetupNewUsersWithVmUUID"],
                                       username)
       repo_name = "#{username}/#{username}"
-      if Rails.configuration.auto_setup_new_users_with_repository and
+      if Rails.configuration.Users["AutoSetupNewUsersWithRepository"] and
           Repository.where(name: repo_name).first.nil?
         repo = Repository.create!(name: repo_name, owner_uuid: uuid)
         Link.create!(tail_uuid: uuid, head_uuid: repo.uuid,
@@ -579,7 +579,7 @@ class User < ArvadosModel
   def send_profile_created_notification
     if self.prefs_changed?
       if self.prefs_was.andand.empty? || !self.prefs_was.andand['profile']
-        profile_notification_address = Rails.configuration.user_profile_notification_address
+        profile_notification_address = Rails.configuration.Users["UserProfileNotificationAddress"]
         ProfileNotifier.profile_created(self, profile_notification_address).deliver_now if profile_notification_address
       end
     end