From 1cce0422dfc66a02e59f0c3a783562c90d0931d9 Mon Sep 17 00:00:00 2001 From: Eric Biagiotti Date: Fri, 2 Aug 2019 15:47:02 -0400 Subject: [PATCH] 14715: Adds keepproxy to cluster config loading Arvados-DCO-1.1-Signed-off-by: Eric Biagiotti --- lib/config/config.default.yml | 3 ++ lib/config/deprecated.go | 71 ++++++++++++++++++++++++++++++++++ lib/config/export.go | 1 + lib/config/generated_config.go | 3 ++ lib/config/load.go | 6 +++ sdk/go/arvados/config.go | 1 + 6 files changed, 85 insertions(+) diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml index 9ac4aeeb96..6da5344fa3 100644 --- a/lib/config/config.default.yml +++ b/lib/config/config.default.yml @@ -206,6 +206,9 @@ Clusters: WebsocketClientEventQueue: 64 WebsocketServerEventQueue: 4 + # Timeout on requests to internal Keep services. + KeepServiceRequestTimeout: 15s + Users: # Config parameters to automatically setup new users. If enabled, # this users will be able to self-activate. Enable this if you want diff --git a/lib/config/deprecated.go b/lib/config/deprecated.go index 12581ddff0..dcfdd8b747 100644 --- a/lib/config/deprecated.go +++ b/lib/config/deprecated.go @@ -326,3 +326,74 @@ func (ldr *Loader) loadOldWebsocketConfig(cfg *arvados.Config) error { cfg.Clusters[cluster.ClusterID] = *cluster return nil } + +type oldKeepProxyConfig struct { + Client *arvados.Client + Listen *string + DisableGet *bool + DisablePut *bool + DefaultReplicas *int + Timeout *arvados.Duration + PIDFile *string + Debug *bool + ManagementToken *string +} + +const defaultKeepproxyConfigPath = "/etc/arvados/keepproxy/keepproxy.yml" + +func (ldr *Loader) loadOldKeepproxyConfig(cfg *arvados.Config) error { + if ldr.KeepproxyPath == "" { + return nil + } + var oc oldKeepProxyConfig + err := ldr.loadOldConfigHelper("keepproxy", ldr.KeepproxyPath, &oc) + if os.IsNotExist(err) && ldr.KeepproxyPath == defaultKeepproxyConfigPath { + return nil + } else if err != nil { + return err + } + + cluster, err := cfg.GetCluster("") + if err != nil { + return err + } + + loadOldClientConfig(cluster, oc.Client) + + if oc.Listen != nil { + cluster.Services.Keepproxy.InternalURLs[arvados.URL{Host: *oc.Listen}] = arvados.ServiceInstance{} + } + if oc.DefaultReplicas != nil { + cluster.Collections.DefaultReplication = *oc.DefaultReplicas + } + if oc.Timeout != nil { + cluster.API.KeepServiceRequestTimeout = *oc.Timeout + } + if oc.Debug != nil { + if *oc.Debug && cluster.SystemLogs.LogLevel != "debug" { + cluster.SystemLogs.LogLevel = "debug" + } else if !*oc.Debug && cluster.SystemLogs.LogLevel != "info" { + cluster.SystemLogs.LogLevel = "info" + } + } + if oc.ManagementToken != nil { + cluster.ManagementToken = *oc.ManagementToken + } + + unsupportedEntry := func(cfgEntry string) error { + return fmt.Errorf("the keepproxy %s configuration option is no longer supported, please remove it from your configuration file", cfgEntry) + } + + if oc.DisableGet != nil && *oc.DisableGet { + return unsupportedEntry("DisableGet") + } + if oc.DisablePut != nil && *oc.DisablePut { + return unsupportedEntry("DisablePut") + } + if oc.PIDFile != nil && *oc.PIDFile != "" { + return unsupportedEntry("PIDFile") + } + + cfg.Clusters[cluster.ClusterID] = *cluster + return nil +} diff --git a/lib/config/export.go b/lib/config/export.go index b125d7dc91..82b48b36b0 100644 --- a/lib/config/export.go +++ b/lib/config/export.go @@ -72,6 +72,7 @@ var whitelist = map[string]bool{ "API.WebsocketClientEventQueue": false, "API.SendTimeout": true, "API.WebsocketServerEventQueue": false, + "API.KeepServiceRequestTimeout": false, "AuditLogs": false, "AuditLogs.MaxAge": false, "AuditLogs.MaxDeleteBatch": false, diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go index 602f30e1da..ffcdddfe2f 100644 --- a/lib/config/generated_config.go +++ b/lib/config/generated_config.go @@ -212,6 +212,9 @@ Clusters: WebsocketClientEventQueue: 64 WebsocketServerEventQueue: 4 + # Timeout on requests to internal Keep services. + KeepServiceRequestTimeout: 15s + Users: # Config parameters to automatically setup new users. If enabled, # this users will be able to self-activate. Enable this if you want diff --git a/lib/config/load.go b/lib/config/load.go index 33d31f71c9..309c0a615d 100644 --- a/lib/config/load.go +++ b/lib/config/load.go @@ -33,6 +33,7 @@ type Loader struct { KeepstorePath string CrunchDispatchSlurmPath string WebsocketPath string + KeepproxyPath string configdata []byte } @@ -62,6 +63,7 @@ func (ldr *Loader) SetupFlags(flagset *flag.FlagSet) { flagset.StringVar(&ldr.KeepstorePath, "legacy-keepstore-config", defaultKeepstoreConfigPath, "Legacy keepstore configuration `file`") flagset.StringVar(&ldr.CrunchDispatchSlurmPath, "legacy-crunch-dispatch-slurm-config", defaultCrunchDispatchSlurmConfigPath, "Legacy crunch-dispatch-slurm configuration `file`") flagset.StringVar(&ldr.WebsocketPath, "legacy-ws-config", defaultWebsocketConfigPath, "Legacy arvados-ws configuration `file`") + flagset.StringVar(&ldr.KeepproxyPath, "legacy-keepproxy-config", defaultKeepproxyConfigPath, "Legacy keepproxy configuration `file`") flagset.BoolVar(&ldr.SkipLegacy, "skip-legacy", false, "Don't load legacy config files") } @@ -133,6 +135,9 @@ func (ldr *Loader) MungeLegacyConfigArgs(lgr logrus.FieldLogger, args []string, if legacyConfigArg != "-legacy-ws-config" { ldr.WebsocketPath = "" } + if legacyConfigArg != "-legacy-keepproxy-config" { + ldr.WebsocketPath = "" + } return munged } @@ -232,6 +237,7 @@ func (ldr *Loader) Load() (*arvados.Config, error) { ldr.loadOldKeepstoreConfig(&cfg), ldr.loadOldCrunchDispatchSlurmConfig(&cfg), ldr.loadOldWebsocketConfig(&cfg), + ldr.loadOldKeepproxyConfig(&cfg), } { if err != nil { return nil, err diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go index f6b736d587..bc434a2030 100644 --- a/sdk/go/arvados/config.go +++ b/sdk/go/arvados/config.go @@ -79,6 +79,7 @@ type Cluster struct { SendTimeout Duration WebsocketClientEventQueue int WebsocketServerEventQueue int + KeepServiceRequestTimeout Duration } AuditLogs struct { MaxAge Duration -- 2.30.2