1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: Apache-2.0
14 "git.arvados.org/arvados.git/sdk/go/config"
17 var DefaultConfigFile = func() string {
18 if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
21 return "/etc/arvados/config.yml"
26 Clusters map[string]Cluster
30 // GetConfig returns the current system config, loading it from
31 // configFile if needed.
32 func GetConfig(configFile string) (*Config, error) {
34 err := config.LoadFile(&cfg, configFile)
38 // GetCluster returns the cluster ID and config for the given
39 // cluster, or the default/only configured cluster if clusterID is "".
40 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
42 if len(sc.Clusters) == 0 {
43 return nil, fmt.Errorf("no clusters configured")
44 } else if len(sc.Clusters) > 1 {
45 return nil, fmt.Errorf("multiple clusters configured, cannot choose")
47 for id, cc := range sc.Clusters {
53 if cc, ok := sc.Clusters[clusterID]; !ok {
54 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
56 cc.ClusterID = clusterID
61 type WebDAVCacheConfig struct {
65 MaxCollectionEntries int
66 MaxCollectionBytes int64
67 MaxPermissionEntries int
72 ClusterID string `json:"-"`
73 ManagementToken string
74 SystemRootToken string
76 InstanceTypes InstanceTypeMap
77 Containers ContainersConfig
78 RemoteClusters map[string]RemoteCluster
82 AsyncPermissionsUpdateInterval Duration
83 DisabledAPIs StringSet
84 MaxIndexDatabaseRead int
85 MaxItemsPerResponse int
86 MaxConcurrentRequests int
87 MaxKeepBlobBuffers int
88 MaxRequestAmplification int
90 RailsSessionSecretToken string
91 RequestTimeout Duration
93 WebsocketClientEventQueue int
94 WebsocketServerEventQueue int
95 KeepServiceRequestTimeout Duration
100 UnloggedAttributes StringSet
104 BlobSigningKey string
105 BlobSigningTTL Duration
107 BlobTrashLifetime Duration
108 BlobTrashCheckInterval Duration
109 BlobTrashConcurrency int
110 BlobDeleteConcurrency int
111 BlobReplicateConcurrency int
112 CollectionVersioning bool
113 DefaultTrashLifetime Duration
114 DefaultReplication int
115 ManagedProperties map[string]struct {
120 PreserveVersionIfIdle Duration
121 TrashSweepInterval Duration
123 ForwardSlashNameSubstitution string
125 BlobMissingReport string
126 BalancePeriod Duration
127 BalanceCollectionBatch int
128 BalanceCollectionBuffers int
129 BalanceTimeout Duration
131 WebDAVCache WebDAVCacheConfig
146 SearchAttribute string
147 SearchBindUser string
148 SearchBindPassword string
151 EmailAttribute string
152 UsernameAttribute string
158 AlternateEmailAddresses bool
160 OpenIDConnect struct {
166 EmailVerifiedClaim string
172 DefaultEmailDomain string
177 ProviderAppSecret string
180 RemoteTokenRefresh Duration
183 MailchimpAPIKey string
184 MailchimpListID string
185 SendUserSetupNotificationEmail bool
186 IssueReporterEmailFrom string
187 IssueReporterEmailTo string
188 SupportEmailAddress string
194 MaxRequestLogParamsSize int
202 AnonymousUserToken string
203 AdminNotifierEmailFrom string
204 AutoAdminFirstUser bool
205 AutoAdminUserWithEmail string
206 AutoSetupNewUsers bool
207 AutoSetupNewUsersWithRepository bool
208 AutoSetupNewUsersWithVmUUID string
209 AutoSetupUsernameBlacklist StringSet
210 EmailSubjectPrefix string
211 NewInactiveUserNotificationRecipients StringSet
212 NewUserNotificationRecipients StringSet
213 NewUsersAreActive bool
214 UserNotifierEmailFrom string
215 UserProfileNotificationAddress string
216 PreferDomainForUsername string
218 Volumes map[string]Volume
220 ActivationContactLink string
221 APIClientConnectTimeout Duration
222 APIClientReceiveTimeout Duration
223 APIResponseCompression bool
224 ApplicationMimetypesWithViewIcon StringSet
225 ArvadosDocsite string
226 ArvadosPublicDataDocURL string
227 DefaultOpenIdPrefix string
228 EnableGettingStartedPopup bool
229 EnablePublicProjectsPage bool
230 FileViewersConfigURL string
231 LogViewerMaxBytes ByteSize
232 MultiSiteSearch string
233 ProfilingEnabled bool
235 RepositoryCache string
236 RunningJobLogRecordsToFetch int
238 ShowRecentCollectionsOnDashboard bool
239 ShowUserAgreementInline bool
240 ShowUserNotifications bool
243 UserProfileFormFields map[string]struct {
245 FormFieldTitle string
246 FormFieldDescription string
249 Options map[string]struct{}
251 UserProfileFormMessage string
253 WelcomePageHTML string
254 InactivePageHTML string
255 SSHHelpPageHTML string
256 SSHHelpHostSuffix string
259 ForceLegacyAPI14 bool
263 AccessViaHosts map[URL]VolumeAccess
266 StorageClasses map[string]bool
268 DriverParameters json.RawMessage
271 type S3VolumeDriverParameters struct {
278 LocationConstraint bool
280 UseAWSS3v2Driver bool
282 ConnectTimeout Duration
288 type AzureVolumeDriverParameters struct {
289 StorageAccountName string
290 StorageAccountKey string
291 StorageBaseURL string
293 RequestTimeout Duration
294 ListBlobsRetryDelay Duration
295 ListBlobsMaxAttempts int
298 type DirectoryVolumeDriverParameters struct {
303 type VolumeAccess struct {
307 type Services struct {
310 DispatchCloud Service
320 WebDAVDownload Service
328 type Service struct {
329 InternalURLs map[URL]ServiceInstance
333 // URL is a url.URL that is also usable as a JSON key/value.
336 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
337 // used as a JSON key/value.
338 func (su *URL) UnmarshalText(text []byte) error {
339 u, err := url.Parse(string(text))
342 if su.Path == "" && su.Host != "" {
343 // http://example really means http://example/
350 func (su URL) MarshalText() ([]byte, error) {
351 return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
354 func (su URL) String() string {
355 return (*url.URL)(&su).String()
358 type ServiceInstance struct {
359 Rendezvous string `json:",omitempty"`
362 type PostgreSQL struct {
363 Connection PostgreSQLConnection
367 type PostgreSQLConnection map[string]string
369 type RemoteCluster struct {
377 type InstanceType struct {
383 IncludedScratch ByteSize
384 AddedScratch ByteSize
389 type ContainersConfig struct {
390 CloudVMs CloudVMsConfig
391 CrunchRunCommand string
392 CrunchRunArgumentsList []string
393 DefaultKeepCacheRAM ByteSize
394 DispatchPrivateKey string
395 LogReuseDecisions bool
397 MaxDispatchAttempts int
399 MinRetryPeriod Duration
400 ReserveExtraRAM ByteSize
401 StaleLockTimeout Duration
402 SupportedDockerImageFormats StringSet
403 UsePreemptibleInstances bool
407 GitInternalDir string
412 LogSecondsBetweenEvents Duration
413 LogThrottlePeriod Duration
416 LimitLogBytesPerJob int
417 LogPartialLineThrottlePeriod Duration
418 LogUpdatePeriod Duration
419 LogUpdateSize ByteSize
423 SbatchArgumentsList []string
424 SbatchEnvironmentVariables map[string]string
426 DNSServerConfDir string
427 DNSServerConfTemplate string
428 DNSServerReloadCommand string
429 DNSServerUpdateCommand string
430 ComputeNodeDomain string
431 ComputeNodeNameservers StringSet
432 AssignNodeHostname string
437 type CloudVMsConfig struct {
440 BootProbeCommand string
441 DeployRunnerBinary string
443 MaxCloudOpsPerSecond int
444 MaxProbesPerSecond int
445 PollInterval Duration
446 ProbeInterval Duration
448 SyncInterval Duration
449 TimeoutBooting Duration
451 TimeoutProbe Duration
452 TimeoutShutdown Duration
453 TimeoutSignal Duration
455 ResourceTags map[string]string
459 DriverParameters json.RawMessage
462 type InstanceTypeMap map[string]InstanceType
464 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
466 // UnmarshalJSON handles old config files that provide an array of
467 // instance types instead of a hash.
468 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
469 fixup := func(t InstanceType) (InstanceType, error) {
470 if t.ProviderType == "" {
471 t.ProviderType = t.Name
474 t.Scratch = t.IncludedScratch + t.AddedScratch
475 } else if t.AddedScratch == 0 {
476 t.AddedScratch = t.Scratch - t.IncludedScratch
477 } else if t.IncludedScratch == 0 {
478 t.IncludedScratch = t.Scratch - t.AddedScratch
481 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
482 return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
487 if len(data) > 0 && data[0] == '[' {
488 var arr []InstanceType
489 err := json.Unmarshal(data, &arr)
497 *it = make(map[string]InstanceType, len(arr))
498 for _, t := range arr {
499 if _, ok := (*it)[t.Name]; ok {
500 return errDuplicateInstanceTypeName
510 var hash map[string]InstanceType
511 err := json.Unmarshal(data, &hash)
515 // Fill in Name field (and ProviderType field, if not
516 // specified) using hash key.
517 *it = InstanceTypeMap(hash)
518 for name, t := range *it {
529 type StringSet map[string]struct{}
531 // UnmarshalJSON handles old config files that provide an array of
532 // instance types instead of a hash.
533 func (ss *StringSet) UnmarshalJSON(data []byte) error {
534 if len(data) > 0 && data[0] == '[' {
536 err := json.Unmarshal(data, &arr)
544 *ss = make(map[string]struct{}, len(arr))
545 for _, t := range arr {
546 (*ss)[t] = struct{}{}
550 var hash map[string]struct{}
551 err := json.Unmarshal(data, &hash)
555 *ss = make(map[string]struct{}, len(hash))
556 for t := range hash {
557 (*ss)[t] = struct{}{}
563 type ServiceName string
566 ServiceNameRailsAPI ServiceName = "arvados-api-server"
567 ServiceNameController ServiceName = "arvados-controller"
568 ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
569 ServiceNameHealth ServiceName = "arvados-health"
570 ServiceNameNodemanager ServiceName = "arvados-node-manager"
571 ServiceNameWorkbench1 ServiceName = "arvados-workbench1"
572 ServiceNameWorkbench2 ServiceName = "arvados-workbench2"
573 ServiceNameWebsocket ServiceName = "arvados-ws"
574 ServiceNameKeepbalance ServiceName = "keep-balance"
575 ServiceNameKeepweb ServiceName = "keep-web"
576 ServiceNameKeepproxy ServiceName = "keepproxy"
577 ServiceNameKeepstore ServiceName = "keepstore"
580 // Map returns all services as a map, suitable for iterating over all
581 // services or looking up a service by name.
582 func (svcs Services) Map() map[ServiceName]Service {
583 return map[ServiceName]Service{
584 ServiceNameRailsAPI: svcs.RailsAPI,
585 ServiceNameController: svcs.Controller,
586 ServiceNameDispatchCloud: svcs.DispatchCloud,
587 ServiceNameHealth: svcs.Health,
588 ServiceNameNodemanager: svcs.Nodemanager,
589 ServiceNameWorkbench1: svcs.Workbench1,
590 ServiceNameWorkbench2: svcs.Workbench2,
591 ServiceNameWebsocket: svcs.Websocket,
592 ServiceNameKeepbalance: svcs.Keepbalance,
593 ServiceNameKeepweb: svcs.WebDAV,
594 ServiceNameKeepproxy: svcs.Keepproxy,
595 ServiceNameKeepstore: svcs.Keepstore,