1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
15 "git.arvados.org/arvados.git/sdk/go/arvados"
16 "github.com/ghodss/yaml"
19 type deprRequestLimits struct {
20 MaxItemsPerResponse *int
21 MultiClusterRequestConcurrency *int
24 type deprCluster struct {
25 RequestLimits deprRequestLimits
26 NodeProfiles map[string]nodeProfile
28 GoogleClientID *string
29 GoogleClientSecret *string
30 GoogleAlternateEmailAddresses *bool
32 ProviderAppSecret *string
36 type deprecatedConfig struct {
37 Clusters map[string]deprCluster
40 type nodeProfile struct {
41 Controller systemServiceInstance `json:"arvados-controller"`
42 Health systemServiceInstance `json:"arvados-health"`
43 Keepbalance systemServiceInstance `json:"keep-balance"`
44 Keepproxy systemServiceInstance `json:"keepproxy"`
45 Keepstore systemServiceInstance `json:"keepstore"`
46 Keepweb systemServiceInstance `json:"keep-web"`
47 DispatchCloud systemServiceInstance `json:"arvados-dispatch-cloud"`
48 RailsAPI systemServiceInstance `json:"arvados-api-server"`
49 Websocket systemServiceInstance `json:"arvados-ws"`
50 Workbench1 systemServiceInstance `json:"arvados-workbench"`
53 type systemServiceInstance struct {
59 func (ldr *Loader) applyDeprecatedConfig(cfg *arvados.Config) error {
60 var dc deprecatedConfig
61 err := yaml.Unmarshal(ldr.configdata, &dc)
65 hostname, err := os.Hostname()
69 for id, dcluster := range dc.Clusters {
70 cluster, ok := cfg.Clusters[id]
72 return fmt.Errorf("can't load legacy config %q that is not present in current config", id)
74 for name, np := range dcluster.NodeProfiles {
75 if name == "*" || name == os.Getenv("ARVADOS_NODE_PROFILE") || name == hostname {
77 } else if ldr.Logger != nil {
78 ldr.Logger.Warnf("overriding Clusters.%s.Services using Clusters.%s.NodeProfiles.%s (guessing %q is a hostname)", id, id, name, name)
80 applyDeprecatedNodeProfile(name, np.RailsAPI, &cluster.Services.RailsAPI)
81 applyDeprecatedNodeProfile(name, np.Controller, &cluster.Services.Controller)
82 applyDeprecatedNodeProfile(name, np.DispatchCloud, &cluster.Services.DispatchCloud)
84 if dst, n := &cluster.API.MaxItemsPerResponse, dcluster.RequestLimits.MaxItemsPerResponse; n != nil && *n != *dst {
87 if dst, n := &cluster.API.MaxRequestAmplification, dcluster.RequestLimits.MultiClusterRequestConcurrency; n != nil && *n != *dst {
91 // Google* moved to Google.*
92 if dst, n := &cluster.Login.Google.ClientID, dcluster.Login.GoogleClientID; n != nil && *n != *dst {
95 // In old config, non-empty ClientID meant enable
96 cluster.Login.Google.Enable = true
99 if dst, n := &cluster.Login.Google.ClientSecret, dcluster.Login.GoogleClientSecret; n != nil && *n != *dst {
102 if dst, n := &cluster.Login.Google.AlternateEmailAddresses, dcluster.Login.GoogleAlternateEmailAddresses; n != nil && *n != *dst {
106 // Provider* moved to SSO.Provider*
107 if dst, n := &cluster.Login.SSO.ProviderAppID, dcluster.Login.ProviderAppID; n != nil && *n != *dst {
110 // In old config, non-empty ID meant enable
111 cluster.Login.SSO.Enable = true
114 if dst, n := &cluster.Login.SSO.ProviderAppSecret, dcluster.Login.ProviderAppSecret; n != nil && *n != *dst {
118 cfg.Clusters[id] = cluster
123 func (ldr *Loader) applyDeprecatedVolumeDriverParameters(cfg *arvados.Config) error {
124 for clusterID, cluster := range cfg.Clusters {
125 for volID, vol := range cluster.Volumes {
126 if vol.Driver == "s3" {
128 AccessKey string `json:",omitempty"`
129 SecretKey string `json:",omitempty"`
131 SecretAccessKey string
133 err := json.Unmarshal(vol.DriverParameters, ¶ms)
135 return fmt.Errorf("error loading %s.Volumes.%s.DriverParameters: %w", clusterID, volID, err)
137 if params.AccessKey != "" || params.SecretKey != "" {
138 if params.AccessKeyID != "" || params.SecretAccessKey != "" {
139 ldr.Logger.Warnf("ignoring old config keys %s.Volumes.%s.DriverParameters.AccessKey/SecretKey because new keys AccessKeyID/SecretAccessKey are also present", clusterID, volID)
142 var allparams map[string]interface{}
143 err = json.Unmarshal(vol.DriverParameters, &allparams)
145 return fmt.Errorf("error loading %s.Volumes.%s.DriverParameters: %w", clusterID, volID, err)
147 for k := range allparams {
148 if lk := strings.ToLower(k); lk == "accesskey" || lk == "secretkey" {
152 allparams["AccessKeyID"] = params.AccessKey
153 allparams["SecretAccessKey"] = params.SecretKey
154 vol.DriverParameters, err = json.Marshal(allparams)
158 cluster.Volumes[volID] = vol
166 func applyDeprecatedNodeProfile(hostname string, ssi systemServiceInstance, svc *arvados.Service) {
171 if svc.InternalURLs == nil {
172 svc.InternalURLs = map[arvados.URL]arvados.ServiceInstance{}
178 if strings.HasPrefix(host, ":") {
179 host = hostname + host
181 svc.InternalURLs[arvados.URL{Scheme: scheme, Host: host, Path: "/"}] = arvados.ServiceInstance{}
184 func (ldr *Loader) loadOldConfigHelper(component, path string, target interface{}) error {
188 buf, err := ioutil.ReadFile(path)
193 ldr.Logger.Warnf("you should remove the legacy %v config file (%s) after migrating all config keys to the cluster configuration file (%s)", component, path, ldr.Path)
195 err = yaml.Unmarshal(buf, target)
197 return fmt.Errorf("%s: %s", path, err)
202 type oldCrunchDispatchSlurmConfig struct {
203 Client *arvados.Client
205 SbatchArguments *[]string
206 PollPeriod *arvados.Duration
207 PrioritySpread *int64
209 // crunch-run command to invoke. The container UUID will be
210 // appended. If nil, []string{"crunch-run"} will be used.
212 // Example: []string{"crunch-run", "--cgroup-parent-subsystem=memory"}
213 CrunchRunCommand *[]string
215 // Extra RAM to reserve (in Bytes) for SLURM job, in addition
216 // to the amount specified in the container's RuntimeConstraints
217 ReserveExtraRAM *int64
219 // Minimum time between two attempts to run the same container
220 MinRetryPeriod *arvados.Duration
222 // Batch size for container queries
226 const defaultCrunchDispatchSlurmConfigPath = "/etc/arvados/crunch-dispatch-slurm/crunch-dispatch-slurm.yml"
228 func loadOldClientConfig(cluster *arvados.Cluster, client *arvados.Client) {
232 if client.APIHost != "" {
233 cluster.Services.Controller.ExternalURL.Host = client.APIHost
234 cluster.Services.Controller.ExternalURL.Path = "/"
236 if client.Scheme != "" {
237 cluster.Services.Controller.ExternalURL.Scheme = client.Scheme
239 cluster.Services.Controller.ExternalURL.Scheme = "https"
241 if client.AuthToken != "" {
242 cluster.SystemRootToken = client.AuthToken
244 cluster.TLS.Insecure = client.Insecure
246 for i, u := range client.KeepServiceURIs {
252 cluster.Containers.SLURM.SbatchEnvironmentVariables = map[string]string{"ARVADOS_KEEP_SERVICES": ks}
255 // update config using values from an crunch-dispatch-slurm config file.
256 func (ldr *Loader) loadOldCrunchDispatchSlurmConfig(cfg *arvados.Config) error {
257 if ldr.CrunchDispatchSlurmPath == "" {
260 var oc oldCrunchDispatchSlurmConfig
261 err := ldr.loadOldConfigHelper("crunch-dispatch-slurm", ldr.CrunchDispatchSlurmPath, &oc)
262 if os.IsNotExist(err) && (ldr.CrunchDispatchSlurmPath == defaultCrunchDispatchSlurmConfigPath) {
264 } else if err != nil {
268 cluster, err := cfg.GetCluster("")
273 loadOldClientConfig(cluster, oc.Client)
275 if oc.SbatchArguments != nil {
276 cluster.Containers.SLURM.SbatchArgumentsList = *oc.SbatchArguments
278 if oc.PollPeriod != nil {
279 cluster.Containers.CloudVMs.PollInterval = *oc.PollPeriod
281 if oc.PrioritySpread != nil {
282 cluster.Containers.SLURM.PrioritySpread = *oc.PrioritySpread
284 if oc.CrunchRunCommand != nil {
285 if len(*oc.CrunchRunCommand) >= 1 {
286 cluster.Containers.CrunchRunCommand = (*oc.CrunchRunCommand)[0]
288 if len(*oc.CrunchRunCommand) >= 2 {
289 cluster.Containers.CrunchRunArgumentsList = (*oc.CrunchRunCommand)[1:]
292 if oc.ReserveExtraRAM != nil {
293 cluster.Containers.ReserveExtraRAM = arvados.ByteSize(*oc.ReserveExtraRAM)
295 if oc.MinRetryPeriod != nil {
296 cluster.Containers.MinRetryPeriod = *oc.MinRetryPeriod
298 if oc.BatchSize != nil {
299 cluster.API.MaxItemsPerResponse = int(*oc.BatchSize)
302 cfg.Clusters[cluster.ClusterID] = *cluster
306 type oldWsConfig struct {
307 Client *arvados.Client
308 Postgres *arvados.PostgreSQLConnection
314 PingTimeout *arvados.Duration
315 ClientEventQueue *int
316 ServerEventQueue *int
318 ManagementToken *string
321 const defaultWebsocketConfigPath = "/etc/arvados/ws/ws.yml"
323 // update config using values from an crunch-dispatch-slurm config file.
324 func (ldr *Loader) loadOldWebsocketConfig(cfg *arvados.Config) error {
325 if ldr.WebsocketPath == "" {
329 err := ldr.loadOldConfigHelper("arvados-ws", ldr.WebsocketPath, &oc)
330 if os.IsNotExist(err) && ldr.WebsocketPath == defaultWebsocketConfigPath {
332 } else if err != nil {
336 cluster, err := cfg.GetCluster("")
341 loadOldClientConfig(cluster, oc.Client)
343 if oc.Postgres != nil {
344 cluster.PostgreSQL.Connection = *oc.Postgres
346 if oc.PostgresPool != nil {
347 cluster.PostgreSQL.ConnectionPool = *oc.PostgresPool
349 if oc.Listen != nil {
350 cluster.Services.Websocket.InternalURLs[arvados.URL{Host: *oc.Listen, Path: "/"}] = arvados.ServiceInstance{}
352 if oc.LogLevel != nil {
353 cluster.SystemLogs.LogLevel = *oc.LogLevel
355 if oc.LogFormat != nil {
356 cluster.SystemLogs.Format = *oc.LogFormat
358 if oc.PingTimeout != nil {
359 cluster.API.SendTimeout = *oc.PingTimeout
361 if oc.ClientEventQueue != nil {
362 cluster.API.WebsocketClientEventQueue = *oc.ClientEventQueue
364 if oc.ServerEventQueue != nil {
365 cluster.API.WebsocketServerEventQueue = *oc.ServerEventQueue
367 if oc.ManagementToken != nil {
368 cluster.ManagementToken = *oc.ManagementToken
371 cfg.Clusters[cluster.ClusterID] = *cluster
375 type oldKeepProxyConfig struct {
376 Client *arvados.Client
381 Timeout *arvados.Duration
384 ManagementToken *string
387 const defaultKeepproxyConfigPath = "/etc/arvados/keepproxy/keepproxy.yml"
389 func (ldr *Loader) loadOldKeepproxyConfig(cfg *arvados.Config) error {
390 if ldr.KeepproxyPath == "" {
393 var oc oldKeepProxyConfig
394 err := ldr.loadOldConfigHelper("keepproxy", ldr.KeepproxyPath, &oc)
395 if os.IsNotExist(err) && ldr.KeepproxyPath == defaultKeepproxyConfigPath {
397 } else if err != nil {
401 cluster, err := cfg.GetCluster("")
406 loadOldClientConfig(cluster, oc.Client)
408 if oc.Listen != nil {
409 cluster.Services.Keepproxy.InternalURLs[arvados.URL{Host: *oc.Listen, Path: "/"}] = arvados.ServiceInstance{}
411 if oc.DefaultReplicas != nil {
412 cluster.Collections.DefaultReplication = *oc.DefaultReplicas
414 if oc.Timeout != nil {
415 cluster.API.KeepServiceRequestTimeout = *oc.Timeout
418 if *oc.Debug && cluster.SystemLogs.LogLevel != "debug" {
419 cluster.SystemLogs.LogLevel = "debug"
420 } else if !*oc.Debug && cluster.SystemLogs.LogLevel != "info" {
421 cluster.SystemLogs.LogLevel = "info"
424 if oc.ManagementToken != nil {
425 cluster.ManagementToken = *oc.ManagementToken
428 // The following legacy options are no longer supported. If they are set to
429 // true or PIDFile has a value, error out and notify the user
430 unsupportedEntry := func(cfgEntry string) error {
431 return fmt.Errorf("the keepproxy %s configuration option is no longer supported, please remove it from your configuration file", cfgEntry)
433 if oc.DisableGet != nil && *oc.DisableGet {
434 return unsupportedEntry("DisableGet")
436 if oc.DisablePut != nil && *oc.DisablePut {
437 return unsupportedEntry("DisablePut")
439 if oc.PIDFile != nil && *oc.PIDFile != "" {
440 return unsupportedEntry("PIDFile")
443 cfg.Clusters[cluster.ClusterID] = *cluster
447 const defaultKeepWebConfigPath = "/etc/arvados/keep-web/keep-web.yml"
449 type oldKeepWebConfig struct {
450 Client *arvados.Client
454 AnonymousTokens *[]string
455 AttachmentOnlyHost *string
456 TrustAllContent *bool
459 TTL *arvados.Duration
460 UUIDTTL *arvados.Duration
461 MaxCollectionEntries *int
462 MaxCollectionBytes *int64
463 MaxPermissionEntries *int
467 // Hack to support old command line flag, which is a bool
468 // meaning "get actual token from environment".
469 deprecatedAllowAnonymous *bool
471 // Authorization token to be included in all health check requests.
472 ManagementToken *string
475 func (ldr *Loader) loadOldKeepWebConfig(cfg *arvados.Config) error {
476 if ldr.KeepWebPath == "" {
479 var oc oldKeepWebConfig
480 err := ldr.loadOldConfigHelper("keep-web", ldr.KeepWebPath, &oc)
481 if os.IsNotExist(err) && ldr.KeepWebPath == defaultKeepWebConfigPath {
483 } else if err != nil {
487 cluster, err := cfg.GetCluster("")
492 loadOldClientConfig(cluster, oc.Client)
494 if oc.Listen != nil {
495 cluster.Services.WebDAV.InternalURLs[arvados.URL{Host: *oc.Listen, Path: "/"}] = arvados.ServiceInstance{}
496 cluster.Services.WebDAVDownload.InternalURLs[arvados.URL{Host: *oc.Listen, Path: "/"}] = arvados.ServiceInstance{}
498 if oc.AttachmentOnlyHost != nil {
499 cluster.Services.WebDAVDownload.ExternalURL = arvados.URL{Host: *oc.AttachmentOnlyHost, Path: "/"}
501 if oc.ManagementToken != nil {
502 cluster.ManagementToken = *oc.ManagementToken
504 if oc.TrustAllContent != nil {
505 cluster.Collections.TrustAllContent = *oc.TrustAllContent
507 if oc.Cache.TTL != nil {
508 cluster.Collections.WebDAVCache.TTL = *oc.Cache.TTL
510 if oc.Cache.UUIDTTL != nil {
511 cluster.Collections.WebDAVCache.UUIDTTL = *oc.Cache.UUIDTTL
513 if oc.Cache.MaxCollectionEntries != nil {
514 cluster.Collections.WebDAVCache.MaxCollectionEntries = *oc.Cache.MaxCollectionEntries
516 if oc.Cache.MaxCollectionBytes != nil {
517 cluster.Collections.WebDAVCache.MaxCollectionBytes = *oc.Cache.MaxCollectionBytes
519 if oc.Cache.MaxPermissionEntries != nil {
520 cluster.Collections.WebDAVCache.MaxPermissionEntries = *oc.Cache.MaxPermissionEntries
522 if oc.Cache.MaxUUIDEntries != nil {
523 cluster.Collections.WebDAVCache.MaxUUIDEntries = *oc.Cache.MaxUUIDEntries
525 if oc.AnonymousTokens != nil {
526 if len(*oc.AnonymousTokens) > 0 {
527 cluster.Users.AnonymousUserToken = (*oc.AnonymousTokens)[0]
528 if len(*oc.AnonymousTokens) > 1 {
529 ldr.Logger.Warn("More than 1 anonymous tokens configured, using only the first and discarding the rest.")
534 cfg.Clusters[cluster.ClusterID] = *cluster
538 const defaultGitHttpdConfigPath = "/etc/arvados/git-httpd/git-httpd.yml"
540 type oldGitHttpdConfig struct {
541 Client *arvados.Client
546 ManagementToken *string
549 func (ldr *Loader) loadOldGitHttpdConfig(cfg *arvados.Config) error {
550 if ldr.GitHttpdPath == "" {
553 var oc oldGitHttpdConfig
554 err := ldr.loadOldConfigHelper("arv-git-httpd", ldr.GitHttpdPath, &oc)
555 if os.IsNotExist(err) && ldr.GitHttpdPath == defaultGitHttpdConfigPath {
557 } else if err != nil {
561 cluster, err := cfg.GetCluster("")
566 loadOldClientConfig(cluster, oc.Client)
568 if oc.Listen != nil {
569 cluster.Services.GitHTTP.InternalURLs[arvados.URL{Host: *oc.Listen}] = arvados.ServiceInstance{}
571 if oc.ManagementToken != nil {
572 cluster.ManagementToken = *oc.ManagementToken
574 if oc.GitCommand != nil {
575 cluster.Git.GitCommand = *oc.GitCommand
577 if oc.GitoliteHome != nil {
578 cluster.Git.GitoliteHome = *oc.GitoliteHome
580 if oc.RepoRoot != nil {
581 cluster.Git.Repositories = *oc.RepoRoot
584 cfg.Clusters[cluster.ClusterID] = *cluster
588 const defaultKeepBalanceConfigPath = "/etc/arvados/keep-balance/keep-balance.yml"
590 type oldKeepBalanceConfig struct {
591 Client *arvados.Client
593 KeepServiceTypes *[]string
594 KeepServiceList *arvados.KeepServiceList
595 RunPeriod *arvados.Duration
596 CollectionBatchSize *int
597 CollectionBuffers *int
598 RequestTimeout *arvados.Duration
599 LostBlocksFile *string
600 ManagementToken *string
603 func (ldr *Loader) loadOldKeepBalanceConfig(cfg *arvados.Config) error {
604 if ldr.KeepBalancePath == "" {
607 var oc oldKeepBalanceConfig
608 err := ldr.loadOldConfigHelper("keep-balance", ldr.KeepBalancePath, &oc)
609 if os.IsNotExist(err) && ldr.KeepBalancePath == defaultKeepBalanceConfigPath {
611 } else if err != nil {
615 cluster, err := cfg.GetCluster("")
620 loadOldClientConfig(cluster, oc.Client)
622 if oc.Listen != nil {
623 cluster.Services.Keepbalance.InternalURLs[arvados.URL{Host: *oc.Listen}] = arvados.ServiceInstance{}
625 if oc.ManagementToken != nil {
626 cluster.ManagementToken = *oc.ManagementToken
628 if oc.RunPeriod != nil {
629 cluster.Collections.BalancePeriod = *oc.RunPeriod
631 if oc.LostBlocksFile != nil {
632 cluster.Collections.BlobMissingReport = *oc.LostBlocksFile
634 if oc.CollectionBatchSize != nil {
635 cluster.Collections.BalanceCollectionBatch = *oc.CollectionBatchSize
637 if oc.CollectionBuffers != nil {
638 cluster.Collections.BalanceCollectionBuffers = *oc.CollectionBuffers
640 if oc.RequestTimeout != nil {
641 cluster.API.KeepServiceRequestTimeout = *oc.RequestTimeout
644 msg := "The %s configuration option is no longer supported. Please remove it from your configuration file. See the keep-balance upgrade notes at https://doc.arvados.org/admin/upgrading.html for more details."
646 // If the keep service type provided is "disk" silently ignore it, since
647 // this is what ends up being done anyway.
648 if oc.KeepServiceTypes != nil {
649 numTypes := len(*oc.KeepServiceTypes)
650 if numTypes != 0 && !(numTypes == 1 && (*oc.KeepServiceTypes)[0] == "disk") {
651 return fmt.Errorf(msg, "KeepServiceTypes")
655 if oc.KeepServiceList != nil {
656 return fmt.Errorf(msg, "KeepServiceList")
659 cfg.Clusters[cluster.ClusterID] = *cluster
663 func (ldr *Loader) loadOldEnvironmentVariables(cfg *arvados.Config) error {
664 if os.Getenv("ARVADOS_API_TOKEN") == "" && os.Getenv("ARVADOS_API_HOST") == "" {
667 cluster, err := cfg.GetCluster("")
671 if tok := os.Getenv("ARVADOS_API_TOKEN"); tok != "" && cluster.SystemRootToken == "" {
672 ldr.Logger.Warn("SystemRootToken missing from cluster config, falling back to ARVADOS_API_TOKEN environment variable")
673 cluster.SystemRootToken = tok
675 if apihost := os.Getenv("ARVADOS_API_HOST"); apihost != "" && cluster.Services.Controller.ExternalURL.Host == "" {
676 ldr.Logger.Warn("Services.Controller.ExternalURL missing from cluster config, falling back to ARVADOS_API_HOST(_INSECURE) environment variables")
677 u, err := url.Parse("https://" + apihost)
679 return fmt.Errorf("cannot parse ARVADOS_API_HOST: %s", err)
681 cluster.Services.Controller.ExternalURL = arvados.URL(*u)
682 if i := os.Getenv("ARVADOS_API_HOST_INSECURE"); i != "" && i != "0" {
683 cluster.TLS.Insecure = true
686 cfg.Clusters[cluster.ClusterID] = *cluster