14717: Remove LegacyComponentConfig behavior

run-tests.sh now requires a minimal config.yml with database information.

Remove database.yml from run-tests.sh

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz@veritasgenetics.com>

diff --git a/build/run-tests.sh b/build/run-tests.sh
index a46f2ec7660590cd82a3969e22ebecaceafe1dc2..626ea974a8fa50f6763149ab4fcb9d6720c7d893 100755 (executable)
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -628,10 +628,11 @@ initialize() {
     setup_ruby_environment
 
     if [[ -s "$CONFIGSRC/config.yml" ]] ; then
+       echo "Getting database configuration from $CONFIGSRC/config.yml"
        cp "$CONFIGSRC/config.yml" "$temp/test-config.yml"
-       export ARVADOS_CONFIG="$temp/test-config.yml"
     else
        if [[ -s /etc/arvados/config.yml ]] ; then
+           echo "Getting database configuration from /etc/arvados/config.yml"
            python > "$temp/test-config.yml" <<EOF
 import yaml
 import json
@@ -639,13 +640,11 @@ v = list(yaml.safe_load(open('/etc/arvados/config.yml'))['Clusters'].values())[0
 v['Connection']['dbname'] = 'arvados_test'
 print(json.dumps({"Clusters": { "zzzzz": {'PostgreSQL': v}}}))
 EOF
-           export ARVADOS_CONFIG="$temp/test-config.yml"
        else
-           if [[ ! -f "$WORKSPACE/services/api/config/database.yml" ]]; then
-               fatal "Please provide a database.yml file for the test suite"
-           fi
+           fatal "Please provide a config.yml file for the test suite in CONFIGSRC or /etc/arvados"
        fi
     fi
+    export ARVADOS_CONFIG="$temp/test-config.yml"
 
     echo "PATH is $PATH"
 }
@@ -951,19 +950,11 @@ install_services/api() {
     rm -f config/environments/test.rb
     cp config/environments/test.rb.example config/environments/test.rb
 
-    if [ -n "$CONFIGSRC" ]
-    then
-        for f in database.yml
-        do
-            cp "$CONFIGSRC/$f" config/ || fatal "$f"
-        done
-    fi
-
     # Clear out any lingering postgresql connections to the test
     # database, so that we can drop it. This assumes the current user
     # is a postgresql superuser.
     cd "$WORKSPACE/services/api" \
-        && test_database=$(python -c "import yaml; print yaml.safe_load(file('config/database.yml'))['test']['database']") \
+        && test_database=$(python -c "import yaml; print yaml.safe_load(file('$ARVADOS_CONFIG'))['Clusters']['zzzzz']['PostgreSQL']['Connection']['dbname']") \
         && psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.pid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null
 
     mkdir -p "$WORKSPACE/services/api/tmp/pids"

diff --git a/lib/config/deprecated.go b/lib/config/deprecated.go
index 845e5113ff965243c1e8e7b41a0ad1aef9f9cc5b..3e1ec7278bcf4662dafbddb0314a8f56432682ac 100644 (file)
--- a/lib/config/deprecated.go
+++ b/lib/config/deprecated.go
@@ -127,10 +127,10 @@ func (ldr *Loader) loadOldConfigHelper(component, path string, target interface{
 }
 
 // update config using values from an old-style keepstore config file.
-func (ldr *Loader) loadOldKeepstoreConfig(cfg *arvados.Config, required bool) error {
+func (ldr *Loader) loadOldKeepstoreConfig(cfg *arvados.Config) error {
        var oc oldKeepstoreConfig
        err := ldr.loadOldConfigHelper("keepstore", ldr.KeepstorePath, &oc)
-       if os.IsNotExist(err) && !required {
+       if os.IsNotExist(err) && (ldr.KeepstorePath == defaultKeepstoreConfigPath) {
                return nil
        } else if err != nil {
                return err
@@ -197,10 +197,10 @@ func loadOldClientConfig(cluster *arvados.Cluster, client *arvados.Client) {
 }
 
 // update config using values from an crunch-dispatch-slurm config file.
-func (ldr *Loader) loadOldCrunchDispatchSlurmConfig(cfg *arvados.Config, required bool) error {
+func (ldr *Loader) loadOldCrunchDispatchSlurmConfig(cfg *arvados.Config) error {
        var oc oldCrunchDispatchSlurmConfig
        err := ldr.loadOldConfigHelper("crunch-dispatch-slurm", ldr.CrunchDispatchSlurmPath, &oc)
-       if os.IsNotExist(err) && !required {
+       if os.IsNotExist(err) && (ldr.CrunchDispatchSlurmPath == defaultCrunchDispatchSlurmConfigPath) {
                return nil
        } else if err != nil {
                return err
@@ -262,10 +262,10 @@ type oldWsConfig struct {
 const defaultWebsocketConfigPath = "/etc/arvados/ws/ws.yml"
 
 // update config using values from an crunch-dispatch-slurm config file.
-func (ldr *Loader) loadOldWebsocketConfig(cfg *arvados.Config, required bool) error {
+func (ldr *Loader) loadOldWebsocketConfig(cfg *arvados.Config) error {
        var oc oldWsConfig
        err := ldr.loadOldConfigHelper("arvados-ws", ldr.WebsocketPath, &oc)
-       if os.IsNotExist(err) && !required {
+       if os.IsNotExist(err) && ldr.WebsocketPath == defaultWebsocketConfigPath {
                return nil
        } else if err != nil {
                return err

diff --git a/lib/config/load.go b/lib/config/load.go
index f9ee6989d22842dd98133ed66b51875e806628a4..2dacd5c26c423739cbb5e2238d6ef2c9cbbbc370 100644 (file)
--- a/lib/config/load.go
+++ b/lib/config/load.go
@@ -33,12 +33,6 @@ type Loader struct {
        CrunchDispatchSlurmPath string
        WebsocketPath           string
 
-       // Legacy config file for the current component (will be the
-       // same as one of the above files).  If set, not being able to
-       // load the 'main' config.yml will not be a fatal error, but
-       // the the legacy file will be required instead.
-       LegacyComponentConfig string
-
        configdata []byte
 }
 
@@ -144,15 +138,10 @@ func (ldr *Loader) Load() (*arvados.Config, error) {
        if ldr.configdata == nil {
                buf, err := ldr.loadBytes(ldr.Path)
                if err != nil {
-                       if ldr.LegacyComponentConfig != "" && os.IsNotExist(err) && !ldr.SkipDeprecated {
-                               buf = []byte(`Clusters: {zzzzz: {}}`)
-                       } else {
-                               return nil, err
-                       }
+                       return nil, err
                }
                ldr.configdata = buf
        }
-       noConfigLoaded := bytes.Compare(ldr.configdata, []byte(`Clusters: {zzzzz: {}}`)) == 0
 
        // Load the config into a dummy map to get the cluster ID
        // keys, discarding the values; then set up defaults for each
@@ -223,14 +212,9 @@ func (ldr *Loader) Load() (*arvados.Config, error) {
                // * no primary config was loaded, and this is the
                // legacy config file for the current component
                for _, err := range []error{
-                       ldr.loadOldKeepstoreConfig(&cfg, (ldr.KeepstorePath != defaultKeepstoreConfigPath) ||
-                               (noConfigLoaded && ldr.LegacyComponentConfig == ldr.KeepstorePath)),
-
-                       ldr.loadOldCrunchDispatchSlurmConfig(&cfg, (ldr.CrunchDispatchSlurmPath != defaultCrunchDispatchSlurmConfigPath) ||
-                               (noConfigLoaded && ldr.LegacyComponentConfig == ldr.CrunchDispatchSlurmPath)),
-
-                       ldr.loadOldWebsocketConfig(&cfg, (ldr.WebsocketPath != defaultWebsocketConfigPath) ||
-                               (noConfigLoaded && ldr.LegacyComponentConfig == ldr.WebsocketPath)),
+                       ldr.loadOldKeepstoreConfig(&cfg),
+                       ldr.loadOldCrunchDispatchSlurmConfig(&cfg),
+                       ldr.loadOldWebsocketConfig(&cfg),
                } {
                        if err != nil {
                                return nil, err

diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
index 75e6146f530ac6f580c4789eb20f8e8b144e9286..1a7ad6fac745067fa9da535ba3052321f062e9de 100644 (file)
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
@@ -109,7 +109,6 @@ func (disp *Dispatcher) configure(prog string, args []string) error {
 
        disp.logger.Printf("crunch-dispatch-slurm %s started", version)
 
-       loader.LegacyComponentConfig = loader.CrunchDispatchSlurmPath
        cfg, err := loader.Load()
        if err != nil {
                return err

diff --git a/services/ws/main.go b/services/ws/main.go
index 2ea1a987ddb0d4929074730d8a45472ae88b740c..0556c77d610d5dede9112b419803f06227992007 100644 (file)
--- a/services/ws/main.go
+++ b/services/ws/main.go
@@ -36,7 +36,6 @@ func configure(log logrus.FieldLogger, args []string) *arvados.Cluster {
                return nil
        }
 
-       loader.LegacyComponentConfig = loader.WebsocketPath
        cfg, err := loader.Load()
        if err != nil {
                log.Fatal(err)

diff --git a/services/ws/server_test.go b/services/ws/server_test.go
index 1d6231fe8c24c2ab841e43b14ed15a5eaac9b1c1..8b43cef3758398f3b44e9166aab6f76df3276707 100644 (file)
--- a/services/ws/server_test.go
+++ b/services/ws/server_test.go
@@ -36,7 +36,6 @@ func (s *serverSuite) SetUpTest(c *check.C) {
 
 func (*serverSuite) testConfig() (*arvados.Cluster, error) {
        ldr := config.NewLoader(nil, nil)
-       ldr.LegacyComponentConfig = "ws-test"
        cfg, err := ldr.Load()
        if err != nil {
                return nil, err