14715: Adds keepproxy to cluster config loading

Arvados-DCO-1.1-Signed-off-by: Eric Biagiotti <ebiagiotti@veritasgenetics.com>

diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index 9ac4aeeb9606461dd23f0f743e6a17a065662c76..6da5344fa3a55920894def50df20acc68d4bcd02 100644 (file)
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -206,6 +206,9 @@ Clusters:
       WebsocketClientEventQueue: 64
       WebsocketServerEventQueue: 4
 
+      # Timeout on requests to internal Keep services.
+      KeepServiceRequestTimeout: 15s
+
     Users:
       # Config parameters to automatically setup new users.  If enabled,
       # this users will be able to self-activate.  Enable this if you want

diff --git a/lib/config/deprecated.go b/lib/config/deprecated.go
index 12581ddff08123cb3026afa0d7ecf5f510311570..dcfdd8b747726ec84284dc82ad2d8af65416fa78 100644 (file)
--- a/lib/config/deprecated.go
+++ b/lib/config/deprecated.go
@@ -326,3 +326,74 @@ func (ldr *Loader) loadOldWebsocketConfig(cfg *arvados.Config) error {
        cfg.Clusters[cluster.ClusterID] = *cluster
        return nil
 }
+
+type oldKeepProxyConfig struct {
+       Client          *arvados.Client
+       Listen          *string
+       DisableGet      *bool
+       DisablePut      *bool
+       DefaultReplicas *int
+       Timeout         *arvados.Duration
+       PIDFile         *string
+       Debug           *bool
+       ManagementToken *string
+}
+
+const defaultKeepproxyConfigPath = "/etc/arvados/keepproxy/keepproxy.yml"
+
+func (ldr *Loader) loadOldKeepproxyConfig(cfg *arvados.Config) error {
+       if ldr.KeepproxyPath == "" {
+               return nil
+       }
+       var oc oldKeepProxyConfig
+       err := ldr.loadOldConfigHelper("keepproxy", ldr.KeepproxyPath, &oc)
+       if os.IsNotExist(err) && ldr.KeepproxyPath == defaultKeepproxyConfigPath {
+               return nil
+       } else if err != nil {
+               return err
+       }
+
+       cluster, err := cfg.GetCluster("")
+       if err != nil {
+               return err
+       }
+
+       loadOldClientConfig(cluster, oc.Client)
+
+       if oc.Listen != nil {
+               cluster.Services.Keepproxy.InternalURLs[arvados.URL{Host: *oc.Listen}] = arvados.ServiceInstance{}
+       }
+       if oc.DefaultReplicas != nil {
+               cluster.Collections.DefaultReplication = *oc.DefaultReplicas
+       }
+       if oc.Timeout != nil {
+               cluster.API.KeepServiceRequestTimeout = *oc.Timeout
+       }
+       if oc.Debug != nil {
+               if *oc.Debug && cluster.SystemLogs.LogLevel != "debug" {
+                       cluster.SystemLogs.LogLevel = "debug"
+               } else if !*oc.Debug && cluster.SystemLogs.LogLevel != "info" {
+                       cluster.SystemLogs.LogLevel = "info"
+               }
+       }
+       if oc.ManagementToken != nil {
+               cluster.ManagementToken = *oc.ManagementToken
+       }
+
+       unsupportedEntry := func(cfgEntry string) error {
+               return fmt.Errorf("the keepproxy %s configuration option is no longer supported, please remove it from your configuration file", cfgEntry)
+       }
+
+       if oc.DisableGet != nil && *oc.DisableGet {
+               return unsupportedEntry("DisableGet")
+       }
+       if oc.DisablePut != nil && *oc.DisablePut {
+               return unsupportedEntry("DisablePut")
+       }
+       if oc.PIDFile != nil && *oc.PIDFile != "" {
+               return unsupportedEntry("PIDFile")
+       }
+
+       cfg.Clusters[cluster.ClusterID] = *cluster
+       return nil
+}

diff --git a/lib/config/export.go b/lib/config/export.go
index b125d7dc917db54283ecf98eddf9daade25d29f8..82b48b36b0fc074dc82b551985e58a7d08f883bd 100644 (file)
--- a/lib/config/export.go
+++ b/lib/config/export.go
@@ -72,6 +72,7 @@ var whitelist = map[string]bool{
        "API.WebsocketClientEventQueue":                false,
        "API.SendTimeout":                              true,
        "API.WebsocketServerEventQueue":                false,
+       "API.KeepServiceRequestTimeout":                false,
        "AuditLogs":                                    false,
        "AuditLogs.MaxAge":                             false,
        "AuditLogs.MaxDeleteBatch":                     false,

diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go
index 602f30e1dae5480bb22ed39e3b0a9bf8c1e04e8f..ffcdddfe2fce5962babde94f1a47853174b14a2c 100644 (file)
--- a/lib/config/generated_config.go
+++ b/lib/config/generated_config.go
@@ -212,6 +212,9 @@ Clusters:
       WebsocketClientEventQueue: 64
       WebsocketServerEventQueue: 4
 
+      # Timeout on requests to internal Keep services.
+      KeepServiceRequestTimeout: 15s
+
     Users:
       # Config parameters to automatically setup new users.  If enabled,
       # this users will be able to self-activate.  Enable this if you want

diff --git a/lib/config/load.go b/lib/config/load.go
index 33d31f71c9172272cf901db29cc202c9338dd0bd..309c0a615dc98620ac86895c471f60db5c50807c 100644 (file)
--- a/lib/config/load.go
+++ b/lib/config/load.go
@@ -33,6 +33,7 @@ type Loader struct {
        KeepstorePath           string
        CrunchDispatchSlurmPath string
        WebsocketPath           string
+       KeepproxyPath           string
 
        configdata []byte
 }
@@ -62,6 +63,7 @@ func (ldr *Loader) SetupFlags(flagset *flag.FlagSet) {
        flagset.StringVar(&ldr.KeepstorePath, "legacy-keepstore-config", defaultKeepstoreConfigPath, "Legacy keepstore configuration `file`")
        flagset.StringVar(&ldr.CrunchDispatchSlurmPath, "legacy-crunch-dispatch-slurm-config", defaultCrunchDispatchSlurmConfigPath, "Legacy crunch-dispatch-slurm configuration `file`")
        flagset.StringVar(&ldr.WebsocketPath, "legacy-ws-config", defaultWebsocketConfigPath, "Legacy arvados-ws configuration `file`")
+       flagset.StringVar(&ldr.KeepproxyPath, "legacy-keepproxy-config", defaultKeepproxyConfigPath, "Legacy keepproxy configuration `file`")
        flagset.BoolVar(&ldr.SkipLegacy, "skip-legacy", false, "Don't load legacy config files")
 }
 
@@ -133,6 +135,9 @@ func (ldr *Loader) MungeLegacyConfigArgs(lgr logrus.FieldLogger, args []string,
        if legacyConfigArg != "-legacy-ws-config" {
                ldr.WebsocketPath = ""
        }
+       if legacyConfigArg != "-legacy-keepproxy-config" {
+               ldr.WebsocketPath = ""
+       }
 
        return munged
 }
@@ -232,6 +237,7 @@ func (ldr *Loader) Load() (*arvados.Config, error) {
                        ldr.loadOldKeepstoreConfig(&cfg),
                        ldr.loadOldCrunchDispatchSlurmConfig(&cfg),
                        ldr.loadOldWebsocketConfig(&cfg),
+                       ldr.loadOldKeepproxyConfig(&cfg),
                } {
                        if err != nil {
                                return nil, err

diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index f6b736d587c893bc8128875e5889ebfe0ad78be7..bc434a20304bbbd129ed64d76307469c46a43fbc 100644 (file)
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -79,6 +79,7 @@ type Cluster struct {
                SendTimeout                    Duration
                WebsocketClientEventQueue      int
                WebsocketServerEventQueue      int
+               KeepServiceRequestTimeout      Duration
        }
        AuditLogs struct {
                MaxAge             Duration