Merge remote-tracking branch 'origin/master' into 14714-keep-balance-config

refs #14714

Arvados-DCO-1.1-Signed-off-by: Eric Biagiotti <ebiagiotti@veritasgenetics.com>

diff --git a/lib/config/cmd_test.go b/lib/config/cmd_test.go
index 019d5edd0186f43aa3edfb3c92692c7131f25a85..fb1cba38b4857d4b253933158f3f8a64a002cb48 100644 (file)
--- a/lib/config/cmd_test.go
+++ b/lib/config/cmd_test.go
@@ -24,6 +24,12 @@ var (
 
 type CommandSuite struct{}
 
+func (s *CommandSuite) SetUpSuite(c *check.C) {
+       os.Unsetenv("ARVADOS_API_HOST")
+       os.Unsetenv("ARVADOS_API_HOST_INSECURE")
+       os.Unsetenv("ARVADOS_API_TOKEN")
+}
+
 func (s *CommandSuite) TestBadArg(c *check.C) {
        var stderr bytes.Buffer
        code := DumpCommand.RunCommand("arvados config-dump", []string{"-badarg"}, bytes.NewBuffer(nil), bytes.NewBuffer(nil), &stderr)

diff --git a/lib/config/deprecated.go b/lib/config/deprecated.go
index 593a9bdcbcaf51b19a40b91a0fd083a3def2f3ba..e6b06c1314668a9662a60c74b710642f33ec5f83 100644 (file)
--- a/lib/config/deprecated.go
+++ b/lib/config/deprecated.go
@@ -7,6 +7,7 @@ package config
 import (
        "fmt"
        "io/ioutil"
+       "net/url"
        "os"
        "strings"
 
@@ -549,3 +550,30 @@ func (ldr *Loader) loadOldKeepBalanceConfig(cfg *arvados.Config) error {
        cfg.Clusters[cluster.ClusterID] = *cluster
        return nil
 }
+
+func (ldr *Loader) loadOldEnvironmentVariables(cfg *arvados.Config) error {
+       if os.Getenv("ARVADOS_API_TOKEN") == "" && os.Getenv("ARVADOS_API_HOST") == "" {
+               return nil
+       }
+       cluster, err := cfg.GetCluster("")
+       if err != nil {
+               return err
+       }
+       if tok := os.Getenv("ARVADOS_API_TOKEN"); tok != "" && cluster.SystemRootToken == "" {
+               ldr.Logger.Warn("SystemRootToken missing from cluster config, falling back to ARVADOS_API_TOKEN environment variable")
+               cluster.SystemRootToken = tok
+       }
+       if apihost := os.Getenv("ARVADOS_API_HOST"); apihost != "" && cluster.Services.Controller.ExternalURL.Host == "" {
+               ldr.Logger.Warn("Services.Controller.ExternalURL missing from cluster config, falling back to ARVADOS_API_HOST(_INSECURE) environment variables")
+               u, err := url.Parse("https://" + apihost)
+               if err != nil {
+                       return fmt.Errorf("cannot parse ARVADOS_API_HOST: %s", err)
+               }
+               cluster.Services.Controller.ExternalURL = arvados.URL(*u)
+               if i := os.Getenv("ARVADOS_API_HOST_INSECURE"); i != "" && i != "0" {
+                       cluster.TLS.Insecure = true
+               }
+       }
+       cfg.Clusters[cluster.ClusterID] = *cluster
+       return nil
+}

diff --git a/lib/config/deprecated_keepstore.go b/lib/config/deprecated_keepstore.go
index 04e937cc5cbfbe8f77f84a7877f3d4b4ccfaadd9..29019296d64e46e1a894595668453d28045f6c92 100644 (file)
--- a/lib/config/deprecated_keepstore.go
+++ b/lib/config/deprecated_keepstore.go
@@ -227,6 +227,10 @@ func (ldr *Loader) loadOldKeepstoreConfig(cfg *arvados.Config) error {
                }
        }
 
+       if err := ldr.checkPendingKeepstoreMigrations(cluster); err != nil {
+               return err
+       }
+
        cfg.Clusters[cluster.ClusterID] = *cluster
        return nil
 }
@@ -580,7 +584,7 @@ func keepServiceIsMe(ks arvados.KeepService, hostname string, listen string) boo
 // been warned about in loadOldKeepstoreConfig() -- i.e., unmigrated
 // keepstore hosts other than the present host, and obsolete content
 // in the keep_services table.
-func (ldr *Loader) checkPendingKeepstoreMigrations(cluster arvados.Cluster) error {
+func (ldr *Loader) checkPendingKeepstoreMigrations(cluster *arvados.Cluster) error {
        if cluster.Services.Controller.ExternalURL.String() == "" {
                ldr.Logger.Debug("Services.Controller.ExternalURL not configured -- skipping check for pending keepstore config migrations")
                return nil
@@ -589,7 +593,7 @@ func (ldr *Loader) checkPendingKeepstoreMigrations(cluster arvados.Cluster) erro
                ldr.Logger.Debug("(Loader).SkipAPICalls == true -- skipping check for pending keepstore config migrations")
                return nil
        }
-       client, err := arvados.NewClientFromConfig(&cluster)
+       client, err := arvados.NewClientFromConfig(cluster)
        if err != nil {
                return err
        }

diff --git a/lib/config/deprecated_keepstore_test.go b/lib/config/deprecated_keepstore_test.go
index d1028e8bd0bfe35efccd632df719a2fdaedebe73..62f3f1e3fe5b2e59865d1b756f5edfaca5d32d49 100644 (file)
--- a/lib/config/deprecated_keepstore_test.go
+++ b/lib/config/deprecated_keepstore_test.go
@@ -30,6 +30,10 @@ type KeepstoreMigrationSuite struct {
 var _ = check.Suite(&KeepstoreMigrationSuite{})
 
 func (s *KeepstoreMigrationSuite) SetUpSuite(c *check.C) {
+       os.Setenv("ARVADOS_API_HOST", os.Getenv("ARVADOS_TEST_API_HOST"))
+       os.Setenv("ARVADOS_API_HOST_INSECURE", "1")
+       os.Setenv("ARVADOS_API_TOKEN", arvadostest.AdminToken)
+
        // We don't need the keepstore servers, but we do need
        // keep_services listings that point to localhost, rather than
        // the apiserver fixtures that point to fictional hosts

diff --git a/lib/config/load.go b/lib/config/load.go
index 8c335f4c7c00075ebad7830fab52513926b5c1dd..21d17227372d4d3f6776b2526581508d89c937ef 100644 (file)
--- a/lib/config/load.go
+++ b/lib/config/load.go
@@ -250,6 +250,7 @@ func (ldr *Loader) Load() (*arvados.Config, error) {
                // * no primary config was loaded, and this is the
                // legacy config file for the current component
                for _, err := range []error{
+                       ldr.loadOldEnvironmentVariables(&cfg),
                        ldr.loadOldKeepstoreConfig(&cfg),
                        ldr.loadOldKeepWebConfig(&cfg),
                        ldr.loadOldCrunchDispatchSlurmConfig(&cfg),
@@ -268,7 +269,6 @@ func (ldr *Loader) Load() (*arvados.Config, error) {
        for id, cc := range cfg.Clusters {
                for _, err = range []error{
                        checkKeyConflict(fmt.Sprintf("Clusters.%s.PostgreSQL.Connection", id), cc.PostgreSQL.Connection),
-                       ldr.checkPendingKeepstoreMigrations(cc),
                        ldr.checkEmptyKeepstores(cc),
                        ldr.checkUnlistedKeepstores(cc),
                } {

diff --git a/lib/config/load_test.go b/lib/config/load_test.go
index 17e0af7ba0bcf9526458641b9f89e8f92aae5a4e..2e521ab0932218cd2f2e1e7f7f976dbbdf3c26d4 100644 (file)
--- a/lib/config/load_test.go
+++ b/lib/config/load_test.go
@@ -46,6 +46,12 @@ func testLoader(c *check.C, configdata string, logdst io.Writer) *Loader {
 
 type LoadSuite struct{}
 
+func (s *LoadSuite) SetUpSuite(c *check.C) {
+       os.Unsetenv("ARVADOS_API_HOST")
+       os.Unsetenv("ARVADOS_API_HOST_INSECURE")
+       os.Unsetenv("ARVADOS_API_TOKEN")
+}
+
 func (s *LoadSuite) TestEmpty(c *check.C) {
        cfg, err := testLoader(c, "", nil).Load()
        c.Check(cfg, check.IsNil)

diff --git a/lib/service/cmd.go b/lib/service/cmd.go
index d5619b87b307c53e781ea77b78ba943b20f78735..ddff5f47a444739998086e6c7d2a748d13f40a8d 100644 (file)
--- a/lib/service/cmd.go
+++ b/lib/service/cmd.go
@@ -12,7 +12,6 @@ import (
        "io"
        "net"
        "net/http"
-       "net/url"
        "os"
        "strings"
 
@@ -110,23 +109,6 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout
        }
        ctx = context.WithValue(ctx, contextKeyURL{}, listenURL)
 
-       if cluster.SystemRootToken == "" {
-               logger.Warn("SystemRootToken missing from cluster config, falling back to ARVADOS_API_TOKEN environment variable")
-               cluster.SystemRootToken = os.Getenv("ARVADOS_API_TOKEN")
-       }
-       if cluster.Services.Controller.ExternalURL.Host == "" {
-               logger.Warn("Services.Controller.ExternalURL missing from cluster config, falling back to ARVADOS_API_HOST(_INSECURE) environment variables")
-               u, err := url.Parse("https://" + os.Getenv("ARVADOS_API_HOST"))
-               if err != nil {
-                       err = fmt.Errorf("ARVADOS_API_HOST: %s", err)
-                       return 1
-               }
-               cluster.Services.Controller.ExternalURL = arvados.URL(*u)
-               if i := os.Getenv("ARVADOS_API_HOST_INSECURE"); i != "" && i != "0" {
-                       cluster.TLS.Insecure = true
-               }
-       }
-
        reg := prometheus.NewRegistry()
        handler := c.newHandler(ctx, cluster, cluster.SystemRootToken, reg)
        if err = handler.CheckHealth(); err != nil {

diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index ccfa1c4825f7da33684d263f12ecf8b7387134eb..48aabbbe409a5c672d7917ae8b57b73973fd7bec 100644 (file)
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -640,6 +640,7 @@ def run_nginx():
          '-g', 'pid '+_pidfile('nginx')+';',
          '-c', conffile],
         env=env, stdin=open('/dev/null'), stdout=sys.stderr)
+    _wait_until_port_listens(nginxconf['CONTROLLERSSLPORT'])
 
 def setup_config():
     rails_api_port = find_available_port()