14717: Fix fallback behavior for component config vs main config

For backwards compatability, we need to be able to start with only a
component config and not the main config, and it is a fatal error if
it doesn't exist.

However, if there is a main config, and then it is okay if the
component config doesn't exist.

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz@veritasgenetics.com>

diff --git a/lib/config/deprecated.go b/lib/config/deprecated.go
index 4526706a3da4d60a0a87b3f6e4a10eae72567317..845e5113ff965243c1e8e7b41a0ad1aef9f9cc5b 100644 (file)
--- a/lib/config/deprecated.go
+++ b/lib/config/deprecated.go
@@ -127,10 +127,10 @@ func (ldr *Loader) loadOldConfigHelper(component, path string, target interface{
 }
 
 // update config using values from an old-style keepstore config file.
-func (ldr *Loader) loadOldKeepstoreConfig(cfg *arvados.Config) error {
+func (ldr *Loader) loadOldKeepstoreConfig(cfg *arvados.Config, required bool) error {
        var oc oldKeepstoreConfig
        err := ldr.loadOldConfigHelper("keepstore", ldr.KeepstorePath, &oc)
-       if os.IsNotExist(err) && ldr.KeepstorePath == defaultKeepstoreConfigPath {
+       if os.IsNotExist(err) && !required {
                return nil
        } else if err != nil {
                return err
@@ -197,10 +197,10 @@ func loadOldClientConfig(cluster *arvados.Cluster, client *arvados.Client) {
 }
 
 // update config using values from an crunch-dispatch-slurm config file.
-func (ldr *Loader) loadOldCrunchDispatchSlurmConfig(cfg *arvados.Config) error {
+func (ldr *Loader) loadOldCrunchDispatchSlurmConfig(cfg *arvados.Config, required bool) error {
        var oc oldCrunchDispatchSlurmConfig
        err := ldr.loadOldConfigHelper("crunch-dispatch-slurm", ldr.CrunchDispatchSlurmPath, &oc)
-       if os.IsNotExist(err) && ldr.CrunchDispatchSlurmPath == defaultCrunchDispatchSlurmConfigPath {
+       if os.IsNotExist(err) && !required {
                return nil
        } else if err != nil {
                return err
@@ -259,13 +259,13 @@ type oldWsConfig struct {
        ManagementToken *string
 }
 
-const defaultWebsocketsConfigPath = "/etc/arvados/ws/ws.yml"
+const defaultWebsocketConfigPath = "/etc/arvados/ws/ws.yml"
 
 // update config using values from an crunch-dispatch-slurm config file.
-func (ldr *Loader) loadOldWebsocketsConfig(cfg *arvados.Config) error {
+func (ldr *Loader) loadOldWebsocketConfig(cfg *arvados.Config, required bool) error {
        var oc oldWsConfig
-       err := ldr.loadOldConfigHelper("arvados-ws", ldr.WebsocketsPath, &oc)
-       if os.IsNotExist(err) && ldr.WebsocketsPath == defaultWebsocketsConfigPath {
+       err := ldr.loadOldConfigHelper("arvados-ws", ldr.WebsocketPath, &oc)
+       if os.IsNotExist(err) && !required {
                return nil
        } else if err != nil {
                return err
@@ -277,7 +277,6 @@ func (ldr *Loader) loadOldWebsocketsConfig(cfg *arvados.Config) error {
        }
 
        loadOldClientConfig(cluster, oc.Client)
-       fmt.Printf("Clllllllllllient %v %v", *oc.Client, cluster.Services.Controller.ExternalURL)
 
        if oc.Postgres != nil {
                cluster.PostgreSQL.Connection = *oc.Postgres
@@ -295,7 +294,7 @@ func (ldr *Loader) loadOldWebsocketsConfig(cfg *arvados.Config) error {
                cluster.SystemLogs.Format = *oc.LogFormat
        }
        if oc.PingTimeout != nil {
-               cluster.API.WebsocketKeepaliveTimeout = *oc.PingTimeout
+               cluster.API.SendTimeout = *oc.PingTimeout
        }
        if oc.ClientEventQueue != nil {
                cluster.API.WebsocketClientEventQueue = *oc.ClientEventQueue

diff --git a/lib/config/load.go b/lib/config/load.go
index 63b6ac7d986543304a0c156003dd64227ac4168c..f9ee6989d22842dd98133ed66b51875e806628a4 100644 (file)
--- a/lib/config/load.go
+++ b/lib/config/load.go
@@ -31,7 +31,13 @@ type Loader struct {
        Path                    string
        KeepstorePath           string
        CrunchDispatchSlurmPath string
-       WebsocketsPath          string
+       WebsocketPath           string
+
+       // Legacy config file for the current component (will be the
+       // same as one of the above files).  If set, not being able to
+       // load the 'main' config.yml will not be a fatal error, but
+       // the the legacy file will be required instead.
+       LegacyComponentConfig string
 
        configdata []byte
 }
@@ -60,7 +66,7 @@ func (ldr *Loader) SetupFlags(flagset *flag.FlagSet) {
        flagset.StringVar(&ldr.Path, "config", arvados.DefaultConfigFile, "Site configuration `file` (default may be overridden by setting an ARVADOS_CONFIG environment variable)")
        flagset.StringVar(&ldr.KeepstorePath, "legacy-keepstore-config", defaultKeepstoreConfigPath, "Legacy keepstore configuration `file`")
        flagset.StringVar(&ldr.CrunchDispatchSlurmPath, "legacy-crunch-dispatch-slurm-config", defaultCrunchDispatchSlurmConfigPath, "Legacy crunch-dispatch-slurm configuration `file`")
-       flagset.StringVar(&ldr.WebsocketsPath, "legacy-ws-config", defaultWebsocketsConfigPath, "Legacy arvados-ws configuration `file`")
+       flagset.StringVar(&ldr.WebsocketPath, "legacy-ws-config", defaultWebsocketConfigPath, "Legacy arvados-ws configuration `file`")
 }
 
 // MungeLegacyConfigArgs checks args for a -config flag whose argument
@@ -134,20 +140,19 @@ func (ldr *Loader) loadBytes(path string) ([]byte, error) {
        return ioutil.ReadAll(f)
 }
 
-func (ldr *Loader) LoadDefaults() (*arvados.Config, error) {
-       ldr.configdata = []byte(`Clusters: {zzzzz: {}}`)
-       defer func() { ldr.configdata = nil }()
-       return ldr.Load()
-}
-
 func (ldr *Loader) Load() (*arvados.Config, error) {
        if ldr.configdata == nil {
                buf, err := ldr.loadBytes(ldr.Path)
                if err != nil {
-                       return nil, err
+                       if ldr.LegacyComponentConfig != "" && os.IsNotExist(err) && !ldr.SkipDeprecated {
+                               buf = []byte(`Clusters: {zzzzz: {}}`)
+                       } else {
+                               return nil, err
+                       }
                }
                ldr.configdata = buf
        }
+       noConfigLoaded := bytes.Compare(ldr.configdata, []byte(`Clusters: {zzzzz: {}}`)) == 0
 
        // Load the config into a dummy map to get the cluster ID
        // keys, discarding the values; then set up defaults for each
@@ -213,10 +218,19 @@ func (ldr *Loader) Load() (*arvados.Config, error) {
                if err != nil {
                        return nil, err
                }
+               // legacy file is required when either:
+               // * a non-default location was specified
+               // * no primary config was loaded, and this is the
+               // legacy config file for the current component
                for _, err := range []error{
-                       ldr.loadOldKeepstoreConfig(&cfg),
-                       ldr.loadOldCrunchDispatchSlurmConfig(&cfg),
-                       ldr.loadOldWebsocketsConfig(&cfg),
+                       ldr.loadOldKeepstoreConfig(&cfg, (ldr.KeepstorePath != defaultKeepstoreConfigPath) ||
+                               (noConfigLoaded && ldr.LegacyComponentConfig == ldr.KeepstorePath)),
+
+                       ldr.loadOldCrunchDispatchSlurmConfig(&cfg, (ldr.CrunchDispatchSlurmPath != defaultCrunchDispatchSlurmConfigPath) ||
+                               (noConfigLoaded && ldr.LegacyComponentConfig == ldr.CrunchDispatchSlurmPath)),
+
+                       ldr.loadOldWebsocketConfig(&cfg, (ldr.WebsocketPath != defaultWebsocketConfigPath) ||
+                               (noConfigLoaded && ldr.LegacyComponentConfig == ldr.WebsocketPath)),
                } {
                        if err != nil {
                                return nil, err

diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index 9072b73192941847ddad15cc89325e6ed946d626..d0ab58ae426611b7a63c73c0f6ddea76106a9170 100644 (file)
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -76,7 +76,7 @@ type Cluster struct {
                MaxRequestSize                 int
                RailsSessionSecretToken        string
                RequestTimeout                 Duration
-               WebsocketKeepaliveTimeout      Duration
+               SendTimeout                    Duration
                WebsocketClientEventQueue      int
                WebsocketServerEventQueue      int
        }

diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
index 1a7ad6fac745067fa9da535ba3052321f062e9de..75e6146f530ac6f580c4789eb20f8e8b144e9286 100644 (file)
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
@@ -109,6 +109,7 @@ func (disp *Dispatcher) configure(prog string, args []string) error {
 
        disp.logger.Printf("crunch-dispatch-slurm %s started", version)
 
+       loader.LegacyComponentConfig = loader.CrunchDispatchSlurmPath
        cfg, err := loader.Load()
        if err != nil {
                return err

diff --git a/services/ws/main.go b/services/ws/main.go
index 0556c77d610d5dede9112b419803f06227992007..2ea1a987ddb0d4929074730d8a45472ae88b740c 100644 (file)
--- a/services/ws/main.go
+++ b/services/ws/main.go
@@ -36,6 +36,7 @@ func configure(log logrus.FieldLogger, args []string) *arvados.Cluster {
                return nil
        }
 
+       loader.LegacyComponentConfig = loader.WebsocketPath
        cfg, err := loader.Load()
        if err != nil {
                log.Fatal(err)

diff --git a/services/ws/router.go b/services/ws/router.go
index 5a5b7c53b2cc31a28182e832408168fb802bfcda..14dc63ec37b12d5524885f476a6912643847ce1c 100644 (file)
--- a/services/ws/router.go
+++ b/services/ws/router.go
@@ -54,7 +54,7 @@ type debugStatuser interface {
 
 func (rtr *router) setup() {
        rtr.handler = &handler{
-               PingTimeout: time.Duration(rtr.cluster.API.WebsocketKeepaliveTimeout),
+               PingTimeout: time.Duration(rtr.cluster.API.SendTimeout),
                QueueSize:   rtr.cluster.API.WebsocketClientEventQueue,
        }
        rtr.mux = http.NewServeMux()

diff --git a/services/ws/server_test.go b/services/ws/server_test.go
index 097889c987a753fcd3cd629216f5402b3f18a8c1..ca57f9faab5a1d1c10f32fba78ec3ee3cc68c957 100644 (file)
--- a/services/ws/server_test.go
+++ b/services/ws/server_test.go
@@ -36,7 +36,8 @@ func (s *serverSuite) SetUpTest(c *check.C) {
 
 func (*serverSuite) testConfig() (*arvados.Cluster, error) {
        ldr := config.NewLoader(nil, nil)
-       cfg, err := ldr.LoadDefaults()
+       ldr.LegacyComponentConfig = "ws-test"
+       cfg, err := ldr.Load()
        if err != nil {
                return nil, err
        }