package config
import (
+ "encoding/json"
"fmt"
"io/ioutil"
+ "net/url"
"os"
"strings"
- "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
"github.com/ghodss/yaml"
)
type deprCluster struct {
RequestLimits deprRequestLimits
NodeProfiles map[string]nodeProfile
+ Login struct {
+ GoogleClientID *string
+ GoogleClientSecret *string
+ GoogleAlternateEmailAddresses *bool
+ ProviderAppID *string
+ ProviderAppSecret *string
+ }
}
type deprecatedConfig struct {
Keepproxy systemServiceInstance `json:"keepproxy"`
Keepstore systemServiceInstance `json:"keepstore"`
Keepweb systemServiceInstance `json:"keep-web"`
- Nodemanager systemServiceInstance `json:"arvados-node-manager"`
DispatchCloud systemServiceInstance `json:"arvados-dispatch-cloud"`
RailsAPI systemServiceInstance `json:"arvados-api-server"`
Websocket systemServiceInstance `json:"arvados-ws"`
if dst, n := &cluster.API.MaxRequestAmplification, dcluster.RequestLimits.MultiClusterRequestConcurrency; n != nil && *n != *dst {
*dst = *n
}
+
+ // Google* moved to Google.*
+ if dst, n := &cluster.Login.Google.ClientID, dcluster.Login.GoogleClientID; n != nil && *n != *dst {
+ *dst = *n
+ if *n != "" {
+ // In old config, non-empty ClientID meant enable
+ cluster.Login.Google.Enable = true
+ }
+ }
+ if dst, n := &cluster.Login.Google.ClientSecret, dcluster.Login.GoogleClientSecret; n != nil && *n != *dst {
+ *dst = *n
+ }
+ if dst, n := &cluster.Login.Google.AlternateEmailAddresses, dcluster.Login.GoogleAlternateEmailAddresses; n != nil && *n != *dst {
+ *dst = *n
+ }
+
cfg.Clusters[id] = cluster
}
return nil
}
+func (ldr *Loader) applyDeprecatedVolumeDriverParameters(cfg *arvados.Config) error {
+ for clusterID, cluster := range cfg.Clusters {
+ for volID, vol := range cluster.Volumes {
+ if vol.Driver == "S3" {
+ var params struct {
+ AccessKey string `json:",omitempty"`
+ SecretKey string `json:",omitempty"`
+ AccessKeyID string
+ SecretAccessKey string
+ }
+ err := json.Unmarshal(vol.DriverParameters, ¶ms)
+ if err != nil {
+ return fmt.Errorf("error loading %s.Volumes.%s.DriverParameters: %w", clusterID, volID, err)
+ }
+ if params.AccessKey != "" || params.SecretKey != "" {
+ if params.AccessKeyID != "" || params.SecretAccessKey != "" {
+ return fmt.Errorf("cannot use old keys (AccessKey/SecretKey) and new keys (AccessKeyID/SecretAccessKey) at the same time in %s.Volumes.%s.DriverParameters -- you must remove the old config keys", clusterID, volID)
+ }
+ var allparams map[string]interface{}
+ err = json.Unmarshal(vol.DriverParameters, &allparams)
+ if err != nil {
+ return fmt.Errorf("error loading %s.Volumes.%s.DriverParameters: %w", clusterID, volID, err)
+ }
+ for k := range allparams {
+ if lk := strings.ToLower(k); lk == "accesskey" || lk == "secretkey" {
+ delete(allparams, k)
+ }
+ }
+ ldr.Logger.Warnf("using your old config keys %s.Volumes.%s.DriverParameters.AccessKey/SecretKey -- but you should rename them to AccessKeyID/SecretAccessKey", clusterID, volID)
+ allparams["AccessKeyID"] = params.AccessKey
+ allparams["SecretAccessKey"] = params.SecretKey
+ vol.DriverParameters, err = json.Marshal(allparams)
+ if err != nil {
+ return err
+ }
+ cluster.Volumes[volID] = vol
+ }
+ }
+ }
+ }
+ return nil
+}
+
func applyDeprecatedNodeProfile(hostname string, ssi systemServiceInstance, svc *arvados.Service) {
scheme := "https"
if !ssi.TLS {
if strings.HasPrefix(host, ":") {
host = hostname + host
}
- svc.InternalURLs[arvados.URL{Scheme: scheme, Host: host}] = arvados.ServiceInstance{}
+ svc.InternalURLs[arvados.URL{Scheme: scheme, Host: host, Path: "/"}] = arvados.ServiceInstance{}
}
func (ldr *Loader) loadOldConfigHelper(component, path string, target interface{}) error {
}
if client.APIHost != "" {
cluster.Services.Controller.ExternalURL.Host = client.APIHost
+ cluster.Services.Controller.ExternalURL.Path = "/"
}
if client.Scheme != "" {
cluster.Services.Controller.ExternalURL.Scheme = client.Scheme
cluster.PostgreSQL.ConnectionPool = *oc.PostgresPool
}
if oc.Listen != nil {
- cluster.Services.Websocket.InternalURLs[arvados.URL{Host: *oc.Listen}] = arvados.ServiceInstance{}
+ cluster.Services.Websocket.InternalURLs[arvados.URL{Host: *oc.Listen, Path: "/"}] = arvados.ServiceInstance{}
}
if oc.LogLevel != nil {
cluster.SystemLogs.LogLevel = *oc.LogLevel
loadOldClientConfig(cluster, oc.Client)
if oc.Listen != nil {
- cluster.Services.Keepproxy.InternalURLs[arvados.URL{Host: *oc.Listen}] = arvados.ServiceInstance{}
+ cluster.Services.Keepproxy.InternalURLs[arvados.URL{Host: *oc.Listen, Path: "/"}] = arvados.ServiceInstance{}
}
if oc.DefaultReplicas != nil {
cluster.Collections.DefaultReplication = *oc.DefaultReplicas
type oldKeepWebConfig struct {
Client *arvados.Client
- Listen string
+ Listen *string
- AnonymousTokens []string
- AttachmentOnlyHost string
- TrustAllContent bool
+ AnonymousTokens *[]string
+ AttachmentOnlyHost *string
+ TrustAllContent *bool
Cache struct {
- TTL arvados.Duration
- UUIDTTL arvados.Duration
- MaxCollectionEntries int
- MaxCollectionBytes int64
- MaxPermissionEntries int
- MaxUUIDEntries int
+ TTL *arvados.Duration
+ UUIDTTL *arvados.Duration
+ MaxCollectionEntries *int
+ MaxCollectionBytes *int64
+ MaxUUIDEntries *int
}
// Hack to support old command line flag, which is a bool
// meaning "get actual token from environment".
- deprecatedAllowAnonymous bool
+ deprecatedAllowAnonymous *bool
// Authorization token to be included in all health check requests.
- ManagementToken string
+ ManagementToken *string
}
func (ldr *Loader) loadOldKeepWebConfig(cfg *arvados.Config) error {
loadOldClientConfig(cluster, oc.Client)
- cluster.Services.WebDAV.InternalURLs[arvados.URL{Host: oc.Listen}] = arvados.ServiceInstance{}
- cluster.Services.WebDAVDownload.InternalURLs[arvados.URL{Host: oc.Listen}] = arvados.ServiceInstance{}
- cluster.Services.WebDAVDownload.ExternalURL = arvados.URL{Host: oc.AttachmentOnlyHost}
- cluster.TLS.Insecure = oc.Client.Insecure
- cluster.ManagementToken = oc.ManagementToken
- cluster.Collections.TrustAllContent = oc.TrustAllContent
- cluster.Collections.WebDAVCache.TTL = oc.Cache.TTL
- cluster.Collections.WebDAVCache.UUIDTTL = oc.Cache.UUIDTTL
- cluster.Collections.WebDAVCache.MaxCollectionEntries = oc.Cache.MaxCollectionEntries
- cluster.Collections.WebDAVCache.MaxCollectionBytes = oc.Cache.MaxCollectionBytes
- cluster.Collections.WebDAVCache.MaxPermissionEntries = oc.Cache.MaxPermissionEntries
- cluster.Collections.WebDAVCache.MaxUUIDEntries = oc.Cache.MaxUUIDEntries
- if len(oc.AnonymousTokens) > 0 {
- cluster.Users.AnonymousUserToken = oc.AnonymousTokens[0]
- if len(oc.AnonymousTokens) > 1 {
- ldr.Logger.Warn("More than 1 anonymous tokens configured, using only the first and discarding the rest.")
+ if oc.Listen != nil {
+ cluster.Services.WebDAV.InternalURLs[arvados.URL{Host: *oc.Listen, Path: "/"}] = arvados.ServiceInstance{}
+ cluster.Services.WebDAVDownload.InternalURLs[arvados.URL{Host: *oc.Listen, Path: "/"}] = arvados.ServiceInstance{}
+ }
+ if oc.AttachmentOnlyHost != nil {
+ cluster.Services.WebDAVDownload.ExternalURL = arvados.URL{Host: *oc.AttachmentOnlyHost, Path: "/"}
+ }
+ if oc.ManagementToken != nil {
+ cluster.ManagementToken = *oc.ManagementToken
+ }
+ if oc.TrustAllContent != nil {
+ cluster.Collections.TrustAllContent = *oc.TrustAllContent
+ }
+ if oc.Cache.TTL != nil {
+ cluster.Collections.WebDAVCache.TTL = *oc.Cache.TTL
+ }
+ if oc.Cache.UUIDTTL != nil {
+ cluster.Collections.WebDAVCache.UUIDTTL = *oc.Cache.UUIDTTL
+ }
+ if oc.Cache.MaxCollectionEntries != nil {
+ cluster.Collections.WebDAVCache.MaxCollectionEntries = *oc.Cache.MaxCollectionEntries
+ }
+ if oc.Cache.MaxCollectionBytes != nil {
+ cluster.Collections.WebDAVCache.MaxCollectionBytes = *oc.Cache.MaxCollectionBytes
+ }
+ if oc.Cache.MaxUUIDEntries != nil {
+ cluster.Collections.WebDAVCache.MaxUUIDEntries = *oc.Cache.MaxUUIDEntries
+ }
+ if oc.AnonymousTokens != nil {
+ if len(*oc.AnonymousTokens) > 0 {
+ cluster.Users.AnonymousUserToken = (*oc.AnonymousTokens)[0]
+ if len(*oc.AnonymousTokens) > 1 {
+ ldr.Logger.Warn("More than 1 anonymous tokens configured, using only the first and discarding the rest.")
+ }
}
}
type oldGitHttpdConfig struct {
Client *arvados.Client
- Listen string
- GitCommand string
- GitoliteHome string
- RepoRoot string
- ManagementToken string
+ Listen *string
+ GitCommand *string
+ GitoliteHome *string
+ RepoRoot *string
+ ManagementToken *string
}
func (ldr *Loader) loadOldGitHttpdConfig(cfg *arvados.Config) error {
return nil
}
var oc oldGitHttpdConfig
- err := ldr.loadOldConfigHelper("arv-git-httpd", ldr.GitHttpdPath, &oc)
+ err := ldr.loadOldConfigHelper("arvados-git-httpd", ldr.GitHttpdPath, &oc)
if os.IsNotExist(err) && ldr.GitHttpdPath == defaultGitHttpdConfigPath {
return nil
} else if err != nil {
loadOldClientConfig(cluster, oc.Client)
- cluster.Services.GitHTTP.InternalURLs[arvados.URL{Host: oc.Listen}] = arvados.ServiceInstance{}
- cluster.TLS.Insecure = oc.Client.Insecure
- cluster.ManagementToken = oc.ManagementToken
- cluster.Git.GitCommand = oc.GitCommand
- cluster.Git.GitoliteHome = oc.GitoliteHome
- cluster.Git.Repositories = oc.RepoRoot
+ if oc.Listen != nil {
+ cluster.Services.GitHTTP.InternalURLs[arvados.URL{Host: *oc.Listen}] = arvados.ServiceInstance{}
+ }
+ if oc.ManagementToken != nil {
+ cluster.ManagementToken = *oc.ManagementToken
+ }
+ if oc.GitCommand != nil {
+ cluster.Git.GitCommand = *oc.GitCommand
+ }
+ if oc.GitoliteHome != nil {
+ cluster.Git.GitoliteHome = *oc.GitoliteHome
+ }
+ if oc.RepoRoot != nil {
+ cluster.Git.Repositories = *oc.RepoRoot
+ }
+
+ cfg.Clusters[cluster.ClusterID] = *cluster
+ return nil
+}
+
+const defaultKeepBalanceConfigPath = "/etc/arvados/keep-balance/keep-balance.yml"
+
+type oldKeepBalanceConfig struct {
+ Client *arvados.Client
+ Listen *string
+ KeepServiceTypes *[]string
+ KeepServiceList *arvados.KeepServiceList
+ RunPeriod *arvados.Duration
+ CollectionBatchSize *int
+ CollectionBuffers *int
+ RequestTimeout *arvados.Duration
+ LostBlocksFile *string
+ ManagementToken *string
+}
+func (ldr *Loader) loadOldKeepBalanceConfig(cfg *arvados.Config) error {
+ if ldr.KeepBalancePath == "" {
+ return nil
+ }
+ var oc oldKeepBalanceConfig
+ err := ldr.loadOldConfigHelper("keep-balance", ldr.KeepBalancePath, &oc)
+ if os.IsNotExist(err) && ldr.KeepBalancePath == defaultKeepBalanceConfigPath {
+ return nil
+ } else if err != nil {
+ return err
+ }
+
+ cluster, err := cfg.GetCluster("")
+ if err != nil {
+ return err
+ }
+
+ loadOldClientConfig(cluster, oc.Client)
+
+ if oc.Listen != nil {
+ cluster.Services.Keepbalance.InternalURLs[arvados.URL{Host: *oc.Listen}] = arvados.ServiceInstance{}
+ }
+ if oc.ManagementToken != nil {
+ cluster.ManagementToken = *oc.ManagementToken
+ }
+ if oc.RunPeriod != nil {
+ cluster.Collections.BalancePeriod = *oc.RunPeriod
+ }
+ if oc.LostBlocksFile != nil {
+ cluster.Collections.BlobMissingReport = *oc.LostBlocksFile
+ }
+ if oc.CollectionBatchSize != nil {
+ cluster.Collections.BalanceCollectionBatch = *oc.CollectionBatchSize
+ }
+ if oc.CollectionBuffers != nil {
+ cluster.Collections.BalanceCollectionBuffers = *oc.CollectionBuffers
+ }
+ if oc.RequestTimeout != nil {
+ cluster.API.KeepServiceRequestTimeout = *oc.RequestTimeout
+ }
+
+ msg := "The %s configuration option is no longer supported. Please remove it from your configuration file. See the keep-balance upgrade notes at https://doc.arvados.org/admin/upgrading.html for more details."
+
+ // If the keep service type provided is "disk" silently ignore it, since
+ // this is what ends up being done anyway.
+ if oc.KeepServiceTypes != nil {
+ numTypes := len(*oc.KeepServiceTypes)
+ if numTypes != 0 && !(numTypes == 1 && (*oc.KeepServiceTypes)[0] == "disk") {
+ return fmt.Errorf(msg, "KeepServiceTypes")
+ }
+ }
+
+ if oc.KeepServiceList != nil {
+ return fmt.Errorf(msg, "KeepServiceList")
+ }
+
+ cfg.Clusters[cluster.ClusterID] = *cluster
+ return nil
+}
+
+func (ldr *Loader) loadOldEnvironmentVariables(cfg *arvados.Config) error {
+ if os.Getenv("ARVADOS_API_TOKEN") == "" && os.Getenv("ARVADOS_API_HOST") == "" {
+ return nil
+ }
+ cluster, err := cfg.GetCluster("")
+ if err != nil {
+ return err
+ }
+ if tok := os.Getenv("ARVADOS_API_TOKEN"); tok != "" && cluster.SystemRootToken == "" {
+ ldr.Logger.Warn("SystemRootToken missing from cluster config, falling back to ARVADOS_API_TOKEN environment variable")
+ cluster.SystemRootToken = tok
+ }
+ if apihost := os.Getenv("ARVADOS_API_HOST"); apihost != "" && cluster.Services.Controller.ExternalURL.Host == "" {
+ ldr.Logger.Warn("Services.Controller.ExternalURL missing from cluster config, falling back to ARVADOS_API_HOST(_INSECURE) environment variables")
+ u, err := url.Parse("https://" + apihost)
+ if err != nil {
+ return fmt.Errorf("cannot parse ARVADOS_API_HOST: %s", err)
+ }
+ cluster.Services.Controller.ExternalURL = arvados.URL(*u)
+ if i := os.Getenv("ARVADOS_API_HOST_INSECURE"); i != "" && i != "0" {
+ cluster.TLS.Insecure = true
+ }
+ }
cfg.Clusters[cluster.ClusterID] = *cluster
return nil
}
projects / arvados.git / blobdiff