Merge remote-tracking branch 'origin/master' into 14714-keep-balance-config

refs #14714

Arvados-DCO-1.1-Signed-off-by: Eric Biagiotti <ebiagiotti@veritasgenetics.com>

diff --combined lib/config/deprecated.go
index 593a9bdcbcaf51b19a40b91a0fd083a3def2f3ba,d0e61dbca06454d3488b1f50473b55f67b2a66b1..e6b06c1314668a9662a60c74b710642f33ec5f83
--- 1/lib/config/deprecated.go
--- 2/lib/config/deprecated.go
+++ b/lib/config/deprecated.go
@@@ -7,6 -7,7 +7,7 @@@ package confi
  import (
        "fmt"
        "io/ioutil"
+       "net/url"
        "os"
        "strings"
  
@@@ -475,77 -476,29 +476,104 @@@ func (ldr *Loader) loadOldGitHttpdConfi
        return nil
  }
  
 +const defaultKeepBalanceConfigPath = "/etc/arvados/keep-balance/keep-balance.yml"
 +
 +type oldKeepBalanceConfig struct {
 +      Client              *arvados.Client
 +      Listen              *string
 +      KeepServiceTypes    *[]string
 +      KeepServiceList     *arvados.KeepServiceList
 +      RunPeriod           *arvados.Duration
 +      CollectionBatchSize *int
 +      CollectionBuffers   *int
 +      RequestTimeout      *arvados.Duration
 +      LostBlocksFile      *string
 +      ManagementToken     *string
 +}
 +
 +func (ldr *Loader) loadOldKeepBalanceConfig(cfg *arvados.Config) error {
 +      if ldr.KeepBalancePath == "" {
 +              return nil
 +      }
 +      var oc oldKeepBalanceConfig
 +      err := ldr.loadOldConfigHelper("keep-balance", ldr.KeepBalancePath, &oc)
 +      if os.IsNotExist(err) && ldr.KeepBalancePath == defaultKeepBalanceConfigPath {
 +              return nil
 +      } else if err != nil {
 +              return err
 +      }
 +
 +      cluster, err := cfg.GetCluster("")
 +      if err != nil {
 +              return err
 +      }
 +
 +      loadOldClientConfig(cluster, oc.Client)
 +
 +      if oc.Listen != nil {
 +              cluster.Services.Keepbalance.InternalURLs[arvados.URL{Host: *oc.Listen}] = arvados.ServiceInstance{}
 +      }
 +      if oc.ManagementToken != nil {
 +              cluster.ManagementToken = *oc.ManagementToken
 +      }
 +      if oc.RunPeriod != nil {
 +              cluster.Collections.BalancePeriod = *oc.RunPeriod
 +      }
 +      if oc.LostBlocksFile != nil {
 +              cluster.Collections.BlobMissingReport = *oc.LostBlocksFile
 +      }
 +      if oc.CollectionBatchSize != nil {
 +              cluster.Collections.BalanceCollectionBatch = *oc.CollectionBatchSize
 +      }
 +      if oc.CollectionBuffers != nil {
 +              cluster.Collections.BalanceCollectionBuffers = *oc.CollectionBuffers
 +      }
 +      if oc.RequestTimeout != nil {
 +              cluster.API.KeepServiceRequestTimeout = *oc.RequestTimeout
 +      }
 +
 +      msg := "The %s configuration option is no longer supported. Please remove it from your configuration file. Keep-balance will operate on all configured volumes."
 +
 +      // If the keep service type provided is "disk" silently ignore it, since
 +      // this is what ends up being done anyway.
 +      if oc.KeepServiceTypes != nil {
 +              numTypes := len(*oc.KeepServiceTypes)
 +              if numTypes != 0 && !(numTypes == 1 && (*oc.KeepServiceTypes)[0] == "disk") {
 +                      return fmt.Errorf(msg, "KeepServiceType")
 +              }
 +      }
 +
 +      if oc.KeepServiceList != nil {
 +              return fmt.Errorf(msg, "KeepServiceList")
 +      }
 +
 +      cfg.Clusters[cluster.ClusterID] = *cluster
 +      return nil
 +}
++
+ func (ldr *Loader) loadOldEnvironmentVariables(cfg *arvados.Config) error {
+       if os.Getenv("ARVADOS_API_TOKEN") == "" && os.Getenv("ARVADOS_API_HOST") == "" {
+               return nil
+       }
+       cluster, err := cfg.GetCluster("")
+       if err != nil {
+               return err
+       }
+       if tok := os.Getenv("ARVADOS_API_TOKEN"); tok != "" && cluster.SystemRootToken == "" {
+               ldr.Logger.Warn("SystemRootToken missing from cluster config, falling back to ARVADOS_API_TOKEN environment variable")
+               cluster.SystemRootToken = tok
+       }
+       if apihost := os.Getenv("ARVADOS_API_HOST"); apihost != "" && cluster.Services.Controller.ExternalURL.Host == "" {
+               ldr.Logger.Warn("Services.Controller.ExternalURL missing from cluster config, falling back to ARVADOS_API_HOST(_INSECURE) environment variables")
+               u, err := url.Parse("https://" + apihost)
+               if err != nil {
+                       return fmt.Errorf("cannot parse ARVADOS_API_HOST: %s", err)
+               }
+               cluster.Services.Controller.ExternalURL = arvados.URL(*u)
+               if i := os.Getenv("ARVADOS_API_HOST_INSECURE"); i != "" && i != "0" {
+                       cluster.TLS.Insecure = true
+               }
+       }
+       cfg.Clusters[cluster.ClusterID] = *cluster
+       return nil
+ }

diff --combined lib/config/load.go
index 8c335f4c7c00075ebad7830fab52513926b5c1dd,61d80a3c58f535733ad65b9053b55283c9d08510..21d17227372d4d3f6776b2526581508d89c937ef
--- 1/lib/config/load.go
--- 2/lib/config/load.go
+++ b/lib/config/load.go
@@@ -37,7 -37,6 +37,7 @@@ type Loader struct 
        WebsocketPath           string
        KeepproxyPath           string
        GitHttpdPath            string
 +      KeepBalancePath         string
  
        configdata []byte
  }
@@@ -70,7 -69,6 +70,7 @@@ func (ldr *Loader) SetupFlags(flagset *
        flagset.StringVar(&ldr.WebsocketPath, "legacy-ws-config", defaultWebsocketConfigPath, "Legacy arvados-ws configuration `file`")
        flagset.StringVar(&ldr.KeepproxyPath, "legacy-keepproxy-config", defaultKeepproxyConfigPath, "Legacy keepproxy configuration `file`")
        flagset.StringVar(&ldr.GitHttpdPath, "legacy-git-httpd-config", defaultGitHttpdConfigPath, "Legacy arv-git-httpd configuration `file`")
 +      flagset.StringVar(&ldr.KeepBalancePath, "legacy-keepbalance-config", defaultKeepBalanceConfigPath, "Legacy keep-balance configuration `file`")
        flagset.BoolVar(&ldr.SkipLegacy, "skip-legacy", false, "Don't load legacy config files")
  }
  
@@@ -151,9 -149,6 +151,9 @@@ func (ldr *Loader) MungeLegacyConfigArg
        if legacyConfigArg != "-legacy-git-httpd-config" {
                ldr.GitHttpdPath = ""
        }
 +      if legacyConfigArg != "-legacy-keepbalance-config" {
 +              ldr.KeepBalancePath = ""
 +      }
  
        return munged
  }
@@@ -250,13 -245,13 +250,14 @@@ func (ldr *Loader) Load() (*arvados.Con
                // * no primary config was loaded, and this is the
                // legacy config file for the current component
                for _, err := range []error{
+                       ldr.loadOldEnvironmentVariables(&cfg),
                        ldr.loadOldKeepstoreConfig(&cfg),
                        ldr.loadOldKeepWebConfig(&cfg),
                        ldr.loadOldCrunchDispatchSlurmConfig(&cfg),
                        ldr.loadOldWebsocketConfig(&cfg),
                        ldr.loadOldKeepproxyConfig(&cfg),
                        ldr.loadOldGitHttpdConfig(&cfg),
 +                      ldr.loadOldKeepBalanceConfig(&cfg),
                } {
                        if err != nil {
                                return nil, err
@@@ -268,7 -263,6 +269,6 @@@
        for id, cc := range cfg.Clusters {
                for _, err = range []error{
                        checkKeyConflict(fmt.Sprintf("Clusters.%s.PostgreSQL.Connection", id), cc.PostgreSQL.Connection),
-                       ldr.checkPendingKeepstoreMigrations(cc),
                        ldr.checkEmptyKeepstores(cc),
                        ldr.checkUnlistedKeepstores(cc),
                } {