14812: Get defaults and config from arvados-server
authorPeter Amstutz <pamstutz@veritasgenetics.com>
Mon, 1 Jul 2019 18:12:44 +0000 (14:12 -0400)
committerPeter Amstutz <pamstutz@veritasgenetics.com>
Mon, 1 Jul 2019 18:12:44 +0000 (14:12 -0400)
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz@veritasgenetics.com>

apps/workbench/config/arvados_config.rb
build/run-tests.sh
cmd/arvados-server/cmd.go
lib/config/cmd.go
lib/config/generated_config.go

index c942884c1fd3ff8ea2aeeed9a6c0966b89b53379..d6fdedc10e5ef3714612f147f963dfa1a7311b76 100644 (file)
@@ -17,6 +17,7 @@
 
 require 'config_loader'
 require 'config_validators'
+require 'open3'
 
 begin
   # If secret_token.rb exists here, we need to load it first.
@@ -26,34 +27,29 @@ rescue LoadError
   # configured by application.yml (i.e., here!) instead.
 end
 
-# Load the defaults
-$arvados_config_defaults = ConfigLoader.load "#{::Rails.root.to_s}/config/config.default.yml"
-if $arvados_config_defaults.empty?
-  raise "Missing #{::Rails.root.to_s}/config/config.default.yml"
-end
-
-def remove_sample_entries(h)
-  return unless h.is_a? Hash
-  h.delete("SAMPLE")
-  h.each { |k, v| remove_sample_entries(v) }
+# Load the defaults, used by config:migrate and fallback loading
+# legacy application.yml
+Open3.popen2("arvados-server", "config-defaults") do |stdin, stdout, status_thread|
+  confs = YAML.load(stdout, deserialize_symbols: false)
+  clusterID, clusterConfig = confs["Clusters"].first
+  $arvados_config_defaults = clusterConfig
+  $arvados_config_defaults["ClusterID"] = clusterID
 end
-remove_sample_entries($arvados_config_defaults)
-
-clusterID, clusterConfig = $arvados_config_defaults["Clusters"].first
-$arvados_config_defaults = clusterConfig
-$arvados_config_defaults["ClusterID"] = clusterID
-
-# Initialize the global config with the defaults
-$arvados_config_global = $arvados_config_defaults.deep_dup
 
 # Load the global config file
-confs = ConfigLoader.load "/etc/arvados/config.yml"
-if !confs.empty?
-  clusterID, clusterConfig = confs["Clusters"].first
-  $arvados_config_global["ClusterID"] = clusterID
-
-  # Copy the cluster config over the defaults
-  $arvados_config_global.deep_merge!(clusterConfig)
+Open3.popen2("arvados-server", "config-dump") do |stdin, stdout, status_thread|
+  confs = YAML.load(stdout, deserialize_symbols: false)
+  if confs && !confs.empty?
+    # config-dump merges defaults with user configuration, so every
+    # key should be set.
+    clusterID, clusterConfig = confs["Clusters"].first
+    $arvados_config_global = clusterConfig
+    $arvados_config_global["ClusterID"] = clusterID
+  else
+    # config-dump failed, assume we will be loading from legacy
+    # application.yml, initialize with defaults.
+    $arvados_config_global = $arvados_config_defaults.deep_dup
+  end
 end
 
 # Now make a copy
index b9dcd777fa55a0b37b9e03ad15c9f81bfd0cf642..1b9092d3beb3f5fb8972a4c87f825aabd9dfdf41 100755 (executable)
@@ -616,6 +616,7 @@ initialize() {
     export R_LIBS
 
     export GOPATH
+    export PATH=$PATH:$GOPATH/bin
 
     # Jenkins config requires that glob tmp/*.log match something. Ensure
     # that happens even if we don't end up running services that set up
index 2506bd2c98892579c974c93f925673277c4c3172..dd34eff7d44855826cdaaf13efa59b6091a79060 100644 (file)
@@ -21,11 +21,12 @@ var (
                "-version":  cmd.Version(version),
                "--version": cmd.Version(version),
 
-               "cloudtest":      cloudtest.Command,
-               "config-check":   config.CheckCommand,
-               "config-dump":    config.DumpCommand,
-               "controller":     controller.Command,
-               "dispatch-cloud": dispatchcloud.Command,
+               "cloudtest":       cloudtest.Command,
+               "config-check":    config.CheckCommand,
+               "config-dump":     config.DumpCommand,
+               "config-defaults": config.DumpDefaultsCommand,
+               "controller":      controller.Command,
+               "dispatch-cloud":  dispatchcloud.Command,
        })
 )
 
index a41e4b0331548f977d69b3ce993795c51e28ea1d..b08ca0deca4dcd728b7e75678bbc88ffec1b4e9f 100644 (file)
@@ -146,3 +146,34 @@ func (pl *plainLogger) Warnf(format string, args ...interface{}) {
        pl.used = true
        fmt.Fprintf(pl.w, format+"\n", args...)
 }
+
+var DumpDefaultsCommand defaultsCommand
+
+type defaultsCommand struct{}
+
+func (defaultsCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+       var err error
+       defer func() {
+               if err != nil {
+                       fmt.Fprintf(stderr, "%s\n", err)
+               }
+       }()
+
+       var src map[string]interface{}
+       err = yaml.Unmarshal(DefaultYAML, &src)
+       if err != nil {
+               err = fmt.Errorf("loading default config data: %s", err)
+               return 1
+       }
+       removeSampleKeys(src)
+
+       out, err := yaml.Marshal(src)
+       if err != nil {
+               return 1
+       }
+       _, err = stdout.Write(out)
+       if err != nil {
+               return 1
+       }
+       return 0
+}
index 5e5222d11e28b9265d1270befb30cf4d92c9feea..87bc64e5807c536253c9a49729dc5712cb09d9fe 100644 (file)
@@ -66,6 +66,14 @@ Clusters:
         ExternalURL: ""
       WebShell:
         InternalURLs: {}
+        # ShellInABox service endpoint URL for a given VM.  If empty, do not
+        # offer web shell logins.
+        #
+        # E.g., using a path-based proxy server to forward connections to shell hosts:
+        # https://webshell.uuid_prefix.arvadosapi.com
+        #
+        # E.g., using a name-based proxy server to forward connections to shell hosts:
+        # https://*.webshell.uuid_prefix.arvadosapi.com
         ExternalURL: ""
       Workbench1:
         InternalURLs: {}
@@ -176,6 +184,11 @@ Clusters:
       NewUserNotificationRecipients: []
       NewInactiveUserNotificationRecipients: []
 
+      # Set anonymous_user_token to enable anonymous user access. You can get
+      # the token by running "bundle exec ./script/get_anonymous_user_token.rb"
+      # in the directory where your API server is running.
+      AnonymousUserToken: ""
+
     AuditLogs:
       # Time to keep audit logs, in seconds. (An audit log is a row added
       # to the "logs" table in the PostgreSQL database each time an
@@ -619,7 +632,7 @@ Clusters:
     Mail:
       MailchimpAPIKey: ""
       MailchimpListID: ""
-      SendUserSetupNotificationEmail: ""
+      SendUserSetupNotificationEmail: true
       IssueReporterEmailFrom: ""
       IssueReporterEmailTo: ""
       SupportEmailAddress: ""
@@ -657,16 +670,20 @@ Clusters:
       ArvadosDocsite: https://doc.arvados.org
       ArvadosPublicDataDocURL: https://playground.arvados.org/projects/public
       ShowUserAgreementInline: false
-      SecretToken: ""
       SecretKeyBase: ""
       RepositoryCache: /var/www/arvados-workbench/current/tmp/git
       UserProfileFormFields:
-        SAMPLE:
-          Type: text
-          FormFieldTitle: ""
-          FormFieldDescription: ""
-          Required: true
+        SAMPLE:
+          Type: text
+            FormFieldTitle: ""
+            FormFieldDescription: ""
+            Required: true
       UserProfileFormMessage: 'Welcome to Arvados. All <span style="color:red">required fields</span> must be completed before you can proceed.'
+
+      # Mimetypes of applications for which the view icon
+      # would be enabled in a collection's show page.
+      # It is sufficient to list only applications here.
+      # No need to list text and image types.
       ApplicationMimetypesWithViewIcon:
         cwl: {}
         fasta: {}
@@ -692,9 +709,10 @@ Clusters:
       RunningJobLogRecordsToFetch: 2000
       ShowRecentCollectionsOnDashboard: true
       ShowUserNotifications: true
-      MultiSiteSearch: false
+      MultiSiteSearch: ""
       Repositories: true
       SiteName: Arvados Workbench
+      TrustAllContent: false
 
       # Workbench2 configs
       VocabularyURL: ""