15000: Ensure all config defaults are captured in config struct.
authorTom Clegg <tclegg@veritasgenetics.com>
Thu, 6 Jun 2019 17:44:23 +0000 (13:44 -0400)
committerTom Clegg <tclegg@veritasgenetics.com>
Fri, 7 Jun 2019 16:38:07 +0000 (12:38 -0400)
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg@veritasgenetics.com>

lib/config/cmd_test.go
lib/config/config.default.yml
lib/config/export.go
lib/config/generated_config.go
lib/config/load_test.go
lib/controller/federation_test.go
lib/controller/handler_test.go
lib/controller/server_test.go
sdk/go/arvados/config.go

index 31a7544939e02fb235113cb722398af304d1b812..fdcb6cc7673dce492e7abbb5fa6873e339e7b3f5 100644 (file)
@@ -59,7 +59,7 @@ Clusters:
 `
        code := CheckCommand.RunCommand("arvados config-check", nil, bytes.NewBufferString(in), &stdout, &stderr)
        c.Check(code, check.Equals, 1)
-       c.Check(stdout.String(), check.Matches, `(?ms).*API:\n\- +.*MaxItemsPerResponse: 1000\n\+ +MaxItemsPerResponse: 1234\n.*`)
+       c.Check(stdout.String(), check.Matches, `(?ms).*\n\- +.*MaxItemsPerResponse: 1000\n\+ +MaxItemsPerResponse: 1234\n.*`)
 }
 
 func (s *CommandSuite) TestCheckUnknownKey(c *check.C) {
index 94cd8fcbf65d2181c918818f7ba4779408b281a0..a7b53489c0c705fc4ec059ad264be0bb30e04eaf 100644 (file)
@@ -35,11 +35,13 @@ Clusters:
         InternalURLs: {}
         ExternalURL: ""
       GitSSH:
+        InternalURLs: {}
         ExternalURL: ""
       DispatchCloud:
         InternalURLs: {}
         ExternalURL: "-"
       SSO:
+        InternalURLs: {}
         ExternalURL: ""
       Keepproxy:
         InternalURLs: {}
@@ -54,13 +56,16 @@ Clusters:
         InternalURLs: {}
         ExternalURL: "-"
       Composer:
+        InternalURLs: {}
         ExternalURL: ""
       WebShell:
+        InternalURLs: {}
         ExternalURL: ""
       Workbench1:
         InternalURLs: {}
         ExternalURL: ""
       Workbench2:
+        InternalURLs: {}
         ExternalURL: ""
       Nodemanager:
         InternalURLs: {}
@@ -113,7 +118,7 @@ Clusters:
       # Interval (seconds) between asynchronous permission view updates. Any
       # permission-updating API called with the 'async' parameter schedules a an
       # update on the permission view in the future, if not already scheduled.
-      AsyncPermissionsUpdateInterval: 20
+      AsyncPermissionsUpdateInterval: 20s
 
       # Maximum number of concurrent outgoing requests to make while
       # serving a single incoming multi-cluster (federated) request.
@@ -260,7 +265,7 @@ Clusters:
       # Interval (seconds) between trash sweeps. During a trash sweep,
       # collections are marked as trash if their trash_at time has
       # arrived, and deleted if their delete_at time has arrived.
-      TrashSweepInterval: 60
+      TrashSweepInterval: 60s
 
       # If true, enable collection versioning.
       # When a collection's preserve_version field is true or the current version
@@ -269,10 +274,10 @@ Clusters:
       # the current collection.
       CollectionVersioning: false
 
-      #   0 = auto-create a new version on every update.
-      #  -1 = never auto-create new versions.
-      # > 0 = auto-create a new version when older than the specified number of seconds.
-      PreserveVersionIfIdle: -1
+      #   0s = auto-create a new version on every update.
+      #  -1s = never auto-create new versions.
+      # > 0s = auto-create a new version when older than the specified number of seconds.
+      PreserveVersionIfIdle: -1s
 
     Login:
       # These settings are provided by your OAuth2 provider (e.g.,
@@ -336,12 +341,6 @@ Clusters:
       # scheduling parameter parameter set.
       UsePreemptibleInstances: false
 
-      # Include details about job reuse decisions in the server log. This
-      # causes additional database queries to run, so it should not be
-      # enabled unless you expect to examine the resulting logs for
-      # troubleshooting purposes.
-      LogReuseDecisions: false
-
       # PEM encoded SSH key (RSA, DSA, or ECDSA) used by the
       # (experimental) cloud dispatcher for executing containers on
       # worker VMs. Begins with "-----BEGIN RSA PRIVATE KEY-----\n"
@@ -366,8 +365,8 @@ Clusters:
         LogBytesPerEvent: 4096
         LogSecondsBetweenEvents: 1
 
-        # The sample period for throttling logs, in seconds.
-        LogThrottlePeriod: 60
+        # The sample period for throttling logs.
+        LogThrottlePeriod: 60s
 
         # Maximum number of bytes that job can log over crunch_log_throttle_period
         # before being silenced until the end of the period.
@@ -381,18 +380,18 @@ Clusters:
         # silenced by throttling are not counted against this total.
         LimitLogBytesPerJob: 67108864
 
-        LogPartialLineThrottlePeriod: 5
+        LogPartialLineThrottlePeriod: 5s
 
-        # Container logs are written to Keep and saved in a collection,
-        # which is updated periodically while the container runs.  This
-        # value sets the interval (given in seconds) between collection
-        # updates.
-        LogUpdatePeriod: 1800
+        # Container logs are written to Keep and saved in a
+        # collection, which is updated periodically while the
+        # container runs.  This value sets the interval between
+        # collection updates.
+        LogUpdatePeriod: 30m
 
         # The log collection is also updated when the specified amount of
         # log data (given in bytes) is produced in less than one update
         # period.
-        LogUpdateSize: 33554432
+        LogUpdateSize: 32MiB
 
       SLURM:
         Managed:
@@ -613,8 +612,20 @@ Clusters:
         Insecure: false
         ActivateUsers: false
       SAMPLE:
+        # API endpoint host or host:port; default is {id}.arvadosapi.com
         Host: sample.arvadosapi.com
+
+        # Perform a proxy request when a local client requests an
+        # object belonging to this remote.
         Proxy: false
+
+        # Default "https". Can be set to "http" for testing.
         Scheme: https
+
+        # Disable TLS verify. Can be set to true for testing.
         Insecure: false
+
+        # When users present tokens issued by this remote cluster, and
+        # their accounts are active on the remote cluster, activate
+        # them on this cluster too.
         ActivateUsers: false
index 98263c2b7b6b794dbc493eedcab5c75cbd698383..3c485e2b42c471053d15f3d97b20e3b72661855c 100644 (file)
@@ -55,62 +55,143 @@ func ExportJSON(w io.Writer, cluster *arvados.Cluster) error {
 // exists.
 var whitelist = map[string]bool{
        // | sort -t'"' -k2,2
-       "API":                                      true,
-       "API.MaxItemsPerResponse":                  true,
-       "API.MaxRequestAmplification":              true,
-       "API.RequestTimeout":                       true,
-       "Containers":                               true,
-       "Containers.CloudVMs":                      true,
-       "Containers.CloudVMs.BootProbeCommand":     true,
-       "Containers.CloudVMs.Driver":               true,
-       "Containers.CloudVMs.DriverParameters":     false,
-       "Containers.CloudVMs.Enable":               true,
-       "Containers.CloudVMs.ImageID":              true,
-       "Containers.CloudVMs.MaxCloudOpsPerSecond": true,
-       "Containers.CloudVMs.MaxProbesPerSecond":   true,
-       "Containers.CloudVMs.PollInterval":         true,
-       "Containers.CloudVMs.ProbeInterval":        true,
-       "Containers.CloudVMs.ResourceTags":         true,
-       "Containers.CloudVMs.ResourceTags.*":       true,
-       "Containers.CloudVMs.SSHPort":              true,
-       "Containers.CloudVMs.SyncInterval":         true,
-       "Containers.CloudVMs.TagKeyPrefix":         true,
-       "Containers.CloudVMs.TimeoutBooting":       true,
-       "Containers.CloudVMs.TimeoutIdle":          true,
-       "Containers.CloudVMs.TimeoutProbe":         true,
-       "Containers.CloudVMs.TimeoutShutdown":      true,
-       "Containers.CloudVMs.TimeoutSignal":        true,
-       "Containers.CloudVMs.TimeoutTERM":          true,
-       "Containers.DispatchPrivateKey":            true,
-       "Containers.StaleLockTimeout":              true,
-       "InstanceTypes":                            true,
-       "InstanceTypes.*":                          true,
-       "InstanceTypes.*.*":                        true,
-       "ManagementToken":                          false,
-       "PostgreSQL":                               true,
-       "PostgreSQL.Connection":                    false,
-       "PostgreSQL.ConnectionPool":                true,
-       "RemoteClusters":                           true,
-       "RemoteClusters.*":                         true,
-       "RemoteClusters.*.Host":                    true,
-       "RemoteClusters.*.Insecure":                true,
-       "RemoteClusters.*.Proxy":                   true,
-       "RemoteClusters.*.Scheme":                  true,
-       "Services":                                 true,
-       "Services.*":                               true,
-       "Services.*.ExternalURL":                   true,
-       "Services.*.InternalURLs":                  true,
-       "Services.*.InternalURLs.*":                true,
-       "Services.*.InternalURLs.*.*":              true,
-       "SystemLogs":                               true,
-       "SystemLogs.Format":                        true,
-       "SystemLogs.LogLevel":                      true,
-       "SystemLogs.MaxRequestLogParamsSize":       true,
-       "SystemRootToken":                          false,
-       "TLS":                                      true,
-       "TLS.Certificate":                          true,
-       "TLS.Insecure":                             true,
-       "TLS.Key":                                  false,
+       "API":                                             true,
+       "API.AsyncPermissionsUpdateInterval":              true,
+       "API.DisabledAPIs":                                true,
+       "API.MaxIndexDatabaseRead":                        true,
+       "API.MaxItemsPerResponse":                         true,
+       "API.MaxRequestAmplification":                     true,
+       "API.MaxRequestSize":                              true,
+       "API.RailsSessionSecretToken":                     false,
+       "API.RequestTimeout":                              true,
+       "AuditLogs":                                       true,
+       "AuditLogs.MaxAge":                                true,
+       "AuditLogs.MaxDeleteBatch":                        true,
+       "AuditLogs.UnloggedAttributes":                    true,
+       "Collections":                                     true,
+       "Collections.BlobSigning":                         true,
+       "Collections.BlobSigningKey":                      false,
+       "Collections.BlobSigningTTL":                      true,
+       "Collections.CollectionVersioning":                true,
+       "Collections.DefaultReplication":                  true,
+       "Collections.DefaultTrashLifetime":                true,
+       "Collections.PreserveVersionIfIdle":               true,
+       "Collections.TrashSweepInterval":                  true,
+       "Containers":                                      true,
+       "Containers.CloudVMs":                             true,
+       "Containers.CloudVMs.BootProbeCommand":            true,
+       "Containers.CloudVMs.Driver":                      true,
+       "Containers.CloudVMs.DriverParameters":            false,
+       "Containers.CloudVMs.Enable":                      true,
+       "Containers.CloudVMs.ImageID":                     true,
+       "Containers.CloudVMs.MaxCloudOpsPerSecond":        true,
+       "Containers.CloudVMs.MaxProbesPerSecond":          true,
+       "Containers.CloudVMs.PollInterval":                true,
+       "Containers.CloudVMs.ProbeInterval":               true,
+       "Containers.CloudVMs.ResourceTags":                true,
+       "Containers.CloudVMs.ResourceTags.*":              true,
+       "Containers.CloudVMs.SSHPort":                     true,
+       "Containers.CloudVMs.SyncInterval":                true,
+       "Containers.CloudVMs.TagKeyPrefix":                true,
+       "Containers.CloudVMs.TimeoutBooting":              true,
+       "Containers.CloudVMs.TimeoutIdle":                 true,
+       "Containers.CloudVMs.TimeoutProbe":                true,
+       "Containers.CloudVMs.TimeoutShutdown":             true,
+       "Containers.CloudVMs.TimeoutSignal":               true,
+       "Containers.CloudVMs.TimeoutTERM":                 true,
+       "Containers.DefaultKeepCacheRAM":                  true,
+       "Containers.DispatchPrivateKey":                   true,
+       "Containers.JobsAPI":                              true,
+       "Containers.JobsAPI.CrunchJobUser":                true,
+       "Containers.JobsAPI.CrunchJobWrapper":             true,
+       "Containers.JobsAPI.CrunchRefreshTrigger":         true,
+       "Containers.JobsAPI.DefaultDockerImage":           true,
+       "Containers.JobsAPI.Enable":                       true,
+       "Containers.JobsAPI.GitInternalDir":               true,
+       "Containers.JobsAPI.ReuseJobIfOutputsDiffer":      true,
+       "Containers.Logging":                              true,
+       "Containers.Logging.LimitLogBytesPerJob":          true,
+       "Containers.Logging.LogBytesPerEvent":             true,
+       "Containers.Logging.LogPartialLineThrottlePeriod": true,
+       "Containers.Logging.LogSecondsBetweenEvents":      true,
+       "Containers.Logging.LogThrottleBytes":             true,
+       "Containers.Logging.LogThrottleLines":             true,
+       "Containers.Logging.LogThrottlePeriod":            true,
+       "Containers.Logging.LogUpdatePeriod":              true,
+       "Containers.Logging.LogUpdateSize":                true,
+       "Containers.Logging.MaxAge":                       true,
+       "Containers.LogReuseDecisions":                    true,
+       "Containers.MaxComputeVMs":                        true,
+       "Containers.MaxDispatchAttempts":                  true,
+       "Containers.MaxRetryAttempts":                     true,
+       "Containers.SLURM":                                true,
+       "Containers.SLURM.Managed":                        true,
+       "Containers.SLURM.Managed.AssignNodeHostname":     true,
+       "Containers.SLURM.Managed.ComputeNodeDomain":      false,
+       "Containers.SLURM.Managed.ComputeNodeNameservers": false,
+       "Containers.SLURM.Managed.DNSServerConfDir":       true,
+       "Containers.SLURM.Managed.DNSServerConfTemplate":  true,
+       "Containers.SLURM.Managed.DNSServerReloadCommand": false,
+       "Containers.SLURM.Managed.DNSServerUpdateCommand": false,
+       "Containers.StaleLockTimeout":                     true,
+       "Containers.SupportedDockerImageFormats":          true,
+       "Containers.UsePreemptibleInstances":              true,
+       "Git":                                             true,
+       "Git.Repositories":                                true,
+       "InstanceTypes":                                   true,
+       "InstanceTypes.*":                                 true,
+       "InstanceTypes.*.*":                               true,
+       "Login":                                           true,
+       "Login.ProviderAppID":                             false,
+       "Login.ProviderAppSecret":                         false,
+       "Mail":                                            true,
+       "Mail.EmailFrom":                                  true,
+       "Mail.IssueReporterEmailFrom":                     true,
+       "Mail.IssueReporterEmailTo":                       true,
+       "Mail.MailchimpAPIKey":                            false,
+       "Mail.MailchimpListID":                            false,
+       "Mail.SendUserSetupNotificationEmail":             true,
+       "Mail.SupportEmailAddress":                        true,
+       "ManagementToken":                                 false,
+       "PostgreSQL":                                      true,
+       "PostgreSQL.Connection":                           false,
+       "PostgreSQL.ConnectionPool":                       true,
+       "RemoteClusters":                                  true,
+       "RemoteClusters.*":                                true,
+       "RemoteClusters.*.ActivateUsers":                  true,
+       "RemoteClusters.*.Host":                           true,
+       "RemoteClusters.*.Insecure":                       true,
+       "RemoteClusters.*.Proxy":                          true,
+       "RemoteClusters.*.Scheme":                         true,
+       "Services":                                        true,
+       "Services.*":                                      true,
+       "Services.*.ExternalURL":                          true,
+       "Services.*.InternalURLs":                         true,
+       "Services.*.InternalURLs.*":                       true,
+       "Services.*.InternalURLs.*.*":                     true,
+       "SystemLogs":                                      true,
+       "SystemLogs.Format":                               true,
+       "SystemLogs.LogLevel":                             true,
+       "SystemLogs.MaxRequestLogParamsSize":              true,
+       "SystemRootToken":                                 false,
+       "TLS":                                             true,
+       "TLS.Certificate":                                 true,
+       "TLS.Insecure":                                    true,
+       "TLS.Key":                                         false,
+       "Users":                                           true,
+       "Users.AdminNotifierEmailFrom":                    true,
+       "Users.AutoAdminFirstUser":                        false,
+       "Users.AutoAdminUserWithEmail":                    false,
+       "Users.AutoSetupNewUsers":                         true,
+       "Users.AutoSetupNewUsersWithRepository":           true,
+       "Users.AutoSetupNewUsersWithVmUUID":               true,
+       "Users.AutoSetupUsernameBlacklist":                false,
+       "Users.EmailSubjectPrefix":                        true,
+       "Users.NewInactiveUserNotificationRecipients":     false,
+       "Users.NewUserNotificationRecipients":             false,
+       "Users.NewUsersAreActive":                         true,
+       "Users.UserNotifierEmailFrom":                     true,
+       "Users.UserProfileNotificationAddress":            true,
 }
 
 func redactUnsafe(m map[string]interface{}, mPrefix, lookupPrefix string) error {
index 3492615e9959f48f6e3134f733194e93af043e93..f9cd7bcd154692c1379daab150f67daf4aa8f489 100644 (file)
@@ -41,11 +41,13 @@ Clusters:
         InternalURLs: {}
         ExternalURL: ""
       GitSSH:
+        InternalURLs: {}
         ExternalURL: ""
       DispatchCloud:
         InternalURLs: {}
         ExternalURL: "-"
       SSO:
+        InternalURLs: {}
         ExternalURL: ""
       Keepproxy:
         InternalURLs: {}
@@ -60,13 +62,16 @@ Clusters:
         InternalURLs: {}
         ExternalURL: "-"
       Composer:
+        InternalURLs: {}
         ExternalURL: ""
       WebShell:
+        InternalURLs: {}
         ExternalURL: ""
       Workbench1:
         InternalURLs: {}
         ExternalURL: ""
       Workbench2:
+        InternalURLs: {}
         ExternalURL: ""
       Nodemanager:
         InternalURLs: {}
@@ -119,7 +124,7 @@ Clusters:
       # Interval (seconds) between asynchronous permission view updates. Any
       # permission-updating API called with the 'async' parameter schedules a an
       # update on the permission view in the future, if not already scheduled.
-      AsyncPermissionsUpdateInterval: 20
+      AsyncPermissionsUpdateInterval: 20s
 
       # Maximum number of concurrent outgoing requests to make while
       # serving a single incoming multi-cluster (federated) request.
@@ -266,7 +271,7 @@ Clusters:
       # Interval (seconds) between trash sweeps. During a trash sweep,
       # collections are marked as trash if their trash_at time has
       # arrived, and deleted if their delete_at time has arrived.
-      TrashSweepInterval: 60
+      TrashSweepInterval: 60s
 
       # If true, enable collection versioning.
       # When a collection's preserve_version field is true or the current version
@@ -275,10 +280,10 @@ Clusters:
       # the current collection.
       CollectionVersioning: false
 
-      #   0 = auto-create a new version on every update.
-      #  -1 = never auto-create new versions.
-      # > 0 = auto-create a new version when older than the specified number of seconds.
-      PreserveVersionIfIdle: -1
+      #   0s = auto-create a new version on every update.
+      #  -1s = never auto-create new versions.
+      # > 0s = auto-create a new version when older than the specified number of seconds.
+      PreserveVersionIfIdle: -1s
 
     Login:
       # These settings are provided by your OAuth2 provider (e.g.,
@@ -342,12 +347,6 @@ Clusters:
       # scheduling parameter parameter set.
       UsePreemptibleInstances: false
 
-      # Include details about job reuse decisions in the server log. This
-      # causes additional database queries to run, so it should not be
-      # enabled unless you expect to examine the resulting logs for
-      # troubleshooting purposes.
-      LogReuseDecisions: false
-
       # PEM encoded SSH key (RSA, DSA, or ECDSA) used by the
       # (experimental) cloud dispatcher for executing containers on
       # worker VMs. Begins with "-----BEGIN RSA PRIVATE KEY-----\n"
@@ -372,8 +371,8 @@ Clusters:
         LogBytesPerEvent: 4096
         LogSecondsBetweenEvents: 1
 
-        # The sample period for throttling logs, in seconds.
-        LogThrottlePeriod: 60
+        # The sample period for throttling logs.
+        LogThrottlePeriod: 60s
 
         # Maximum number of bytes that job can log over crunch_log_throttle_period
         # before being silenced until the end of the period.
@@ -387,18 +386,18 @@ Clusters:
         # silenced by throttling are not counted against this total.
         LimitLogBytesPerJob: 67108864
 
-        LogPartialLineThrottlePeriod: 5
+        LogPartialLineThrottlePeriod: 5s
 
-        # Container logs are written to Keep and saved in a collection,
-        # which is updated periodically while the container runs.  This
-        # value sets the interval (given in seconds) between collection
-        # updates.
-        LogUpdatePeriod: 1800
+        # Container logs are written to Keep and saved in a
+        # collection, which is updated periodically while the
+        # container runs.  This value sets the interval between
+        # collection updates.
+        LogUpdatePeriod: 30m
 
         # The log collection is also updated when the specified amount of
         # log data (given in bytes) is produced in less than one update
         # period.
-        LogUpdateSize: 33554432
+        LogUpdateSize: 32MiB
 
       SLURM:
         Managed:
@@ -619,9 +618,21 @@ Clusters:
         Insecure: false
         ActivateUsers: false
       SAMPLE:
+        # API endpoint host or host:port; default is {id}.arvadosapi.com
         Host: sample.arvadosapi.com
+
+        # Perform a proxy request when a local client requests an
+        # object belonging to this remote.
         Proxy: false
+
+        # Default "https". Can be set to "http" for testing.
         Scheme: https
+
+        # Disable TLS verify. Can be set to true for testing.
         Insecure: false
+
+        # When users present tokens issued by this remote cluster, and
+        # their accounts are active on the remote cluster, activate
+        # them on this cluster too.
         ActivateUsers: false
 `)
index 6ce81bb5f9826b2374d9b7e77de9f1468e9e28c5..6b014476b6d9f31d6cf23197bcde6fac2fda0bc3 100644 (file)
@@ -97,6 +97,24 @@ Clusters:
        c.Check(logs, check.HasLen, 2)
 }
 
+func (s *LoadSuite) TestNoUnrecognizedKeysInDefaultConfig(c *check.C) {
+       var logbuf bytes.Buffer
+       logger := logrus.New()
+       logger.Out = &logbuf
+       var supplied map[string]interface{}
+       yaml.Unmarshal(DefaultYAML, &supplied)
+       cfg, err := Load(bytes.NewBuffer(DefaultYAML), logger)
+       c.Assert(err, check.IsNil)
+       var loaded map[string]interface{}
+       buf, err := yaml.Marshal(cfg)
+       c.Assert(err, check.IsNil)
+       err = yaml.Unmarshal(buf, &loaded)
+       c.Assert(err, check.IsNil)
+
+       logExtraKeys(logger, loaded, supplied, "")
+       c.Check(logbuf.String(), check.Equals, "")
+}
+
 func (s *LoadSuite) TestNoWarningsForDumpedConfig(c *check.C) {
        var logbuf bytes.Buffer
        logger := logrus.New()
index 1c859cfc515d142a0289610e402e725e07bfebb1..7d8e7a4334ae98bd727ed62725d1acea09746ae9 100644 (file)
@@ -57,12 +57,10 @@ func (s *FederationSuite) SetUpTest(c *check.C) {
        cluster := &arvados.Cluster{
                ClusterID:  "zhome",
                PostgreSQL: integrationTestCluster().PostgreSQL,
-               TLS:        arvados.TLS{Insecure: true},
-               API: arvados.API{
-                       MaxItemsPerResponse:     1000,
-                       MaxRequestAmplification: 4,
-               },
        }
+       cluster.TLS.Insecure = true
+       cluster.API.MaxItemsPerResponse = 1000
+       cluster.API.MaxRequestAmplification = 4
        arvadostest.SetServiceURL(&cluster.Services.RailsAPI, "http://localhost:1/")
        arvadostest.SetServiceURL(&cluster.Services.Controller, "http://localhost:/")
        s.testHandler = &Handler{Cluster: cluster}
index 06a5a84113dd36222a38f9e8cd971f0f46eac0f7..cb69f2f6bba0730e9bcaa83737a969f340c86cf5 100644 (file)
@@ -42,8 +42,8 @@ func (s *HandlerSuite) SetUpTest(c *check.C) {
        s.cluster = &arvados.Cluster{
                ClusterID:  "zzzzz",
                PostgreSQL: integrationTestCluster().PostgreSQL,
-               TLS:        arvados.TLS{Insecure: true},
        }
+       s.cluster.TLS.Insecure = true
        arvadostest.SetServiceURL(&s.cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
        arvadostest.SetServiceURL(&s.cluster.Services.Controller, "http://localhost:/")
        s.handler = newHandler(s.ctx, s.cluster, "")
index a398af97b21884ae896f675b1c2ab00a59ae55d4..ae7f138b1b6862ab43022ed91b0fbdd360b3dc36 100644 (file)
@@ -36,8 +36,8 @@ func newServerFromIntegrationTestEnv(c *check.C) *httpserver.Server {
        handler := &Handler{Cluster: &arvados.Cluster{
                ClusterID:  "zzzzz",
                PostgreSQL: integrationTestCluster().PostgreSQL,
-               TLS:        arvados.TLS{Insecure: true},
        }}
+       handler.Cluster.TLS.Insecure = true
        arvadostest.SetServiceURL(&handler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
        arvadostest.SetServiceURL(&handler.Cluster.Services.Controller, "http://localhost:/")
 
index d96bf25173a949dc0d95cb49f9ba639295c019b4..fb01f25f8c7eb3948c68f9ef17497972dfd949be 100644 (file)
@@ -50,12 +50,6 @@ func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
        }
 }
 
-type API struct {
-       MaxItemsPerResponse     int
-       MaxRequestAmplification int
-       RequestTimeout          Duration
-}
-
 type Cluster struct {
        ClusterID       string `json:"-"`
        ManagementToken string
@@ -65,28 +59,98 @@ type Cluster struct {
        Containers      ContainersConfig
        RemoteClusters  map[string]RemoteCluster
        PostgreSQL      PostgreSQL
-       API             API
-       SystemLogs      SystemLogs
-       TLS             TLS
+
+       API struct {
+               AsyncPermissionsUpdateInterval Duration
+               DisabledAPIs                   []string
+               MaxIndexDatabaseRead           int
+               MaxItemsPerResponse            int
+               MaxRequestAmplification        int
+               MaxRequestSize                 int
+               RailsSessionSecretToken        string
+               RequestTimeout                 Duration
+       }
+       AuditLogs struct {
+               MaxAge             Duration
+               MaxDeleteBatch     int
+               UnloggedAttributes []string
+       }
+       Collections struct {
+               BlobSigning           bool
+               BlobSigningKey        string
+               DefaultReplication    int
+               BlobSigningTTL        Duration
+               DefaultTrashLifetime  Duration
+               TrashSweepInterval    Duration
+               CollectionVersioning  bool
+               PreserveVersionIfIdle Duration
+       }
+       Git struct {
+               Repositories string
+       }
+       Login struct {
+               ProviderAppSecret string
+               ProviderAppID     string
+       }
+       Mail struct {
+               MailchimpAPIKey                string
+               MailchimpListID                string
+               SendUserSetupNotificationEmail string
+               IssueReporterEmailFrom         string
+               IssueReporterEmailTo           string
+               SupportEmailAddress            string
+               EmailFrom                      string
+       }
+       SystemLogs struct {
+               LogLevel                string
+               Format                  string
+               MaxRequestLogParamsSize int
+       }
+       TLS struct {
+               Certificate string
+               Key         string
+               Insecure    bool
+       }
+       Users struct {
+               AdminNotifierEmailFrom                string
+               AutoAdminFirstUser                    bool
+               AutoAdminUserWithEmail                string
+               AutoSetupNewUsers                     bool
+               AutoSetupNewUsersWithRepository       bool
+               AutoSetupNewUsersWithVmUUID           string
+               AutoSetupUsernameBlacklist            []string
+               EmailSubjectPrefix                    string
+               NewInactiveUserNotificationRecipients []string
+               NewUserNotificationRecipients         []string
+               NewUsersAreActive                     bool
+               UserNotifierEmailFrom                 string
+               UserProfileNotificationAddress        string
+       }
 }
 
 type Services struct {
-       Controller    Service
-       DispatchCloud Service
-       Health        Service
-       Keepbalance   Service
-       Keepproxy     Service
-       Keepstore     Service
-       Nodemanager   Service
-       RailsAPI      Service
-       WebDAV        Service
-       Websocket     Service
-       Workbench1    Service
-       Workbench2    Service
+       Composer       Service
+       Controller     Service
+       DispatchCloud  Service
+       GitHTTP        Service
+       GitSSH         Service
+       Health         Service
+       Keepbalance    Service
+       Keepproxy      Service
+       Keepstore      Service
+       Nodemanager    Service
+       RailsAPI       Service
+       SSO            Service
+       WebDAVDownload Service
+       WebDAV         Service
+       WebShell       Service
+       Websocket      Service
+       Workbench1     Service
+       Workbench2     Service
 }
 
 type Service struct {
-       InternalURLs map[URL]ServiceInstance `json:",omitempty"`
+       InternalURLs map[URL]ServiceInstance
        ExternalURL  URL
 }
 
@@ -109,12 +173,6 @@ func (su URL) MarshalText() ([]byte, error) {
 
 type ServiceInstance struct{}
 
-type SystemLogs struct {
-       LogLevel                string
-       Format                  string
-       MaxRequestLogParamsSize int
-}
-
 type PostgreSQL struct {
        Connection     PostgreSQLConnection
        ConnectionPool int
@@ -123,15 +181,11 @@ type PostgreSQL struct {
 type PostgreSQLConnection map[string]string
 
 type RemoteCluster struct {
-       // API endpoint host or host:port; default is {id}.arvadosapi.com
-       Host string
-       // Perform a proxy request when a local client requests an
-       // object belonging to this remote.
-       Proxy bool
-       // Scheme, default "https". Can be set to "http" for testing.
-       Scheme string
-       // Disable TLS verify. Can be set to true for testing.
-       Insecure bool
+       Host          string
+       Proxy         bool
+       Scheme        string
+       Insecure      bool
+       ActivateUsers bool
 }
 
 type InstanceType struct {
@@ -147,9 +201,49 @@ type InstanceType struct {
 }
 
 type ContainersConfig struct {
-       CloudVMs           CloudVMsConfig
-       DispatchPrivateKey string
-       StaleLockTimeout   Duration
+       CloudVMs                    CloudVMsConfig
+       DefaultKeepCacheRAM         ByteSize
+       DispatchPrivateKey          string
+       LogReuseDecisions           bool
+       MaxComputeVMs               int
+       MaxDispatchAttempts         int
+       MaxRetryAttempts            int
+       StaleLockTimeout            Duration
+       SupportedDockerImageFormats []string
+       UsePreemptibleInstances     bool
+
+       JobsAPI struct {
+               Enable                  string
+               GitInternalDir          string
+               DefaultDockerImage      string
+               CrunchJobWrapper        string
+               CrunchJobUser           string
+               CrunchRefreshTrigger    string
+               ReuseJobIfOutputsDiffer bool
+       }
+       Logging struct {
+               MaxAge                       Duration
+               LogBytesPerEvent             int
+               LogSecondsBetweenEvents      int
+               LogThrottlePeriod            Duration
+               LogThrottleBytes             int
+               LogThrottleLines             int
+               LimitLogBytesPerJob          int
+               LogPartialLineThrottlePeriod Duration
+               LogUpdatePeriod              Duration
+               LogUpdateSize                ByteSize
+       }
+       SLURM struct {
+               Managed struct {
+                       DNSServerConfDir       string
+                       DNSServerConfTemplate  string
+                       DNSServerReloadCommand string
+                       DNSServerUpdateCommand string
+                       ComputeNodeDomain      string
+                       ComputeNodeNameservers []string
+                       AssignNodeHostname     string
+               }
+       }
 }
 
 type CloudVMsConfig struct {
@@ -269,9 +363,3 @@ func (svcs Services) Map() map[ServiceName]Service {
                ServiceNameKeepstore:     svcs.Keepstore,
        }
 }
-
-type TLS struct {
-       Certificate string
-       Key         string
-       Insecure    bool
-}