1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: Apache-2.0
16 "git.arvados.org/arvados.git/sdk/go/config"
19 var DefaultConfigFile = func() string {
20 if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
23 return "/etc/arvados/config.yml"
27 Clusters map[string]Cluster
29 SourceTimestamp time.Time
33 // GetConfig returns the current system config, loading it from
34 // configFile if needed.
35 func GetConfig(configFile string) (*Config, error) {
37 err := config.LoadFile(&cfg, configFile)
41 // GetCluster returns the cluster ID and config for the given
42 // cluster, or the default/only configured cluster if clusterID is "".
43 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
45 if len(sc.Clusters) == 0 {
46 return nil, fmt.Errorf("no clusters configured")
47 } else if len(sc.Clusters) > 1 {
48 return nil, fmt.Errorf("multiple clusters configured, cannot choose")
50 for id, cc := range sc.Clusters {
56 cc, ok := sc.Clusters[clusterID]
58 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
60 cc.ClusterID = clusterID
64 type WebDAVCacheConfig struct {
67 MaxCollectionBytes int64
71 type UploadDownloadPermission struct {
76 type UploadDownloadRolePermissions struct {
77 User UploadDownloadPermission
78 Admin UploadDownloadPermission
81 type ManagedProperties map[string]struct {
88 ClusterID string `json:"-"`
89 ManagementToken string
90 SystemRootToken string
92 InstanceTypes InstanceTypeMap
93 Containers ContainersConfig
94 RemoteClusters map[string]RemoteCluster
98 AsyncPermissionsUpdateInterval Duration
99 DisabledAPIs StringSet
100 MaxIndexDatabaseRead int
101 MaxItemsPerResponse int
102 MaxConcurrentRailsRequests int
103 MaxConcurrentRequests int
104 MaxQueuedRequests int
105 MaxQueueTimeForLockRequests Duration
106 LogCreateRequestFraction float64
107 MaxKeepBlobBuffers int
108 MaxRequestAmplification int
110 MaxTokenLifetime Duration
111 RequestTimeout Duration
113 WebsocketClientEventQueue int
114 WebsocketServerEventQueue int
115 KeepServiceRequestTimeout Duration
116 VocabularyPath string
117 FreezeProjectRequiresDescription bool
118 FreezeProjectRequiresProperties StringSet
119 UnfreezeProjectRequiresAdmin bool
120 LockBeforeUpdate bool
125 UnloggedAttributes StringSet
129 BlobSigningKey string
130 BlobSigningTTL Duration
132 BlobTrashLifetime Duration
133 BlobTrashCheckInterval Duration
134 BlobTrashConcurrency int
135 BlobDeleteConcurrency int
136 BlobReplicateConcurrency int
137 CollectionVersioning bool
138 DefaultTrashLifetime Duration
139 DefaultReplication int
140 ManagedProperties ManagedProperties
141 PreserveVersionIfIdle Duration
142 TrashSweepInterval Duration
144 ForwardSlashNameSubstitution string
147 BlobMissingReport string
148 BalancePeriod Duration
149 BalanceCollectionBatch int
150 BalanceCollectionBuffers int
151 BalanceTimeout Duration
152 BalanceUpdateLimit int
154 BalanceTrashLimit int
156 WebDAVCache WebDAVCacheConfig
158 KeepproxyPermission UploadDownloadRolePermissions
159 WebDAVPermission UploadDownloadRolePermissions
173 MinTLSVersion TLSVersion
176 SearchAttribute string
177 SearchBindUser string
178 SearchBindPassword string
181 EmailAttribute string
182 UsernameAttribute string
188 AlternateEmailAddresses bool
189 AuthenticationRequestParameters map[string]string
191 OpenIDConnect struct {
197 EmailVerifiedClaim string
199 AcceptAccessToken bool
200 AcceptAccessTokenScope string
201 AuthenticationRequestParameters map[string]string
206 DefaultEmailDomain string
210 Users map[string]TestUser
213 RemoteTokenRefresh Duration
214 TokenLifetime Duration
215 TrustedClients map[URL]struct{}
216 TrustPrivateNetworks bool
217 IssueTrustedTokens bool
220 MailchimpAPIKey string
221 MailchimpListID string
222 SendUserSetupNotificationEmail bool
223 IssueReporterEmailFrom string
224 IssueReporterEmailTo string
225 SupportEmailAddress string
231 MaxRequestLogParamsSize int
232 RequestQueueDumpDirectory string
243 ActivatedUsersAreVisibleToOthers bool
244 AnonymousUserToken string
245 AdminNotifierEmailFrom string
246 AutoAdminFirstUser bool
247 AutoAdminUserWithEmail string
248 AutoSetupNewUsers bool
249 AutoSetupNewUsersWithRepository bool
250 AutoSetupNewUsersWithVmUUID string
251 AutoSetupUsernameBlacklist StringSet
252 EmailSubjectPrefix string
253 NewInactiveUserNotificationRecipients StringSet
254 NewUserNotificationRecipients StringSet
255 NewUsersAreActive bool
256 UserNotifierEmailFrom string
257 UserNotifierEmailBcc StringSet
258 UserProfileNotificationAddress string
259 PreferDomainForUsername string
260 UserSetupMailText string
261 RoleGroupsVisibleToAll bool
262 CanCreateRoleGroups bool
263 ActivityLoggingPeriod Duration
264 SyncIgnoredGroups []string
265 SyncRequiredGroups []string
266 SyncUserAccounts bool
267 SyncUserAPITokens bool
271 StorageClasses map[string]StorageClassConfig
272 Volumes map[string]Volume
274 ActivationContactLink string
275 APIClientConnectTimeout Duration
276 APIClientReceiveTimeout Duration
277 APIResponseCompression bool
278 ApplicationMimetypesWithViewIcon StringSet
279 ArvadosDocsite string
280 ArvadosPublicDataDocURL string
281 DefaultOpenIdPrefix string
282 DisableSharingURLsUI bool
283 EnableGettingStartedPopup bool
284 EnablePublicProjectsPage bool
285 FileViewersConfigURL string
286 LogViewerMaxBytes ByteSize
287 MultiSiteSearch string
288 ProfilingEnabled bool
290 RepositoryCache string
291 RunningJobLogRecordsToFetch int
293 ShowRecentCollectionsOnDashboard bool
294 ShowUserAgreementInline bool
295 ShowUserNotifications bool
298 UserProfileFormFields map[string]struct {
300 FormFieldTitle string
301 FormFieldDescription string
304 Options map[string]struct{}
306 UserProfileFormMessage string
307 WelcomePageHTML string
308 InactivePageHTML string
309 SSHHelpPageHTML string
310 SSHHelpHostSuffix string
316 type StorageClassConfig struct {
322 AccessViaHosts map[URL]VolumeAccess
325 StorageClasses map[string]bool
327 DriverParameters json.RawMessage
330 type S3VolumeDriverParameters struct {
333 SecretAccessKey string
337 LocationConstraint bool
340 ConnectTimeout Duration
347 type AzureVolumeDriverParameters struct {
348 StorageAccountName string
349 StorageAccountKey string
350 StorageBaseURL string
352 RequestTimeout Duration
353 ListBlobsRetryDelay Duration
354 ListBlobsMaxAttempts int
357 type DirectoryVolumeDriverParameters struct {
362 type VolumeAccess struct {
366 type Services struct {
369 DispatchCloud Service
371 DispatchSLURM Service
379 WebDAVDownload Service
387 type Service struct {
388 InternalURLs map[URL]ServiceInstance
392 type TestUser struct {
397 // URL is a url.URL that is also usable as a JSON key/value.
400 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
401 // used as a JSON key/value.
402 func (su *URL) UnmarshalText(text []byte) error {
403 u, err := url.Parse(string(text))
406 if su.Path == "" && su.Host != "" {
407 // http://example really means http://example/
414 func (su URL) MarshalText() ([]byte, error) {
415 return []byte(su.String()), nil
418 func (su URL) String() string {
419 return (*url.URL)(&su).String()
422 type TLSVersion uint16
424 func (v TLSVersion) MarshalText() ([]byte, error) {
428 case tls.VersionTLS10:
429 return []byte("1.0"), nil
430 case tls.VersionTLS11:
431 return []byte("1.1"), nil
432 case tls.VersionTLS12:
433 return []byte("1.2"), nil
434 case tls.VersionTLS13:
435 return []byte("1.3"), nil
437 return nil, fmt.Errorf("unsupported TLSVersion %x", v)
441 func (v *TLSVersion) UnmarshalJSON(text []byte) error {
442 if len(text) > 0 && text[0] == '"' {
444 err := json.Unmarshal(text, &s)
450 switch string(text) {
454 *v = tls.VersionTLS10
456 *v = tls.VersionTLS11
458 *v = tls.VersionTLS12
460 *v = tls.VersionTLS13
462 return fmt.Errorf("unsupported TLSVersion %q", text)
467 type ServiceInstance struct {
469 Rendezvous string `json:",omitempty"`
472 type PostgreSQL struct {
473 Connection PostgreSQLConnection
477 type PostgreSQLConnection map[string]string
479 type RemoteCluster struct {
487 type CUDAFeatures struct {
489 HardwareCapability string
493 type InstanceType struct {
494 Name string `json:"-"`
498 Scratch ByteSize `json:"-"`
499 IncludedScratch ByteSize
500 AddedScratch ByteSize
506 type ContainersConfig struct {
507 CloudVMs CloudVMsConfig
508 CrunchRunCommand string
509 CrunchRunArgumentsList []string
510 DefaultKeepCacheRAM ByteSize
511 DispatchPrivateKey string
512 LogReuseDecisions bool
513 MaxDispatchAttempts int
515 MinRetryPeriod Duration
516 ReserveExtraRAM ByteSize
517 StaleLockTimeout Duration
518 SupportedDockerImageFormats StringSet
519 AlwaysUsePreemptibleInstances bool
520 PreemptiblePriceFactor float64
521 MaximumPriceFactor float64
523 LocalKeepBlobBuffersPerVCPU int
524 LocalKeepLogsToContainerLog string
528 GitInternalDir string
532 SweepInterval Duration
534 LogSecondsBetweenEvents Duration
535 LogThrottlePeriod Duration
538 LimitLogBytesPerJob int
539 LogPartialLineThrottlePeriod Duration
540 LogUpdatePeriod Duration
541 LogUpdateSize ByteSize
549 SbatchArgumentsList []string
550 SbatchEnvironmentVariables map[string]string
552 DNSServerConfDir string
553 DNSServerConfTemplate string
554 DNSServerReloadCommand string
555 DNSServerUpdateCommand string
556 ComputeNodeDomain string
557 ComputeNodeNameservers StringSet
558 AssignNodeHostname string
563 BsubArgumentsList []string
564 BsubCUDAArguments []string
568 type CloudVMsConfig struct {
571 BootProbeCommand string
572 InstanceInitCommand string
573 DeployRunnerBinary string
576 MaxCloudOpsPerSecond int
577 MaxProbesPerSecond int
578 MaxConcurrentInstanceCreateOps int
580 InitialQuotaEstimate int
581 SupervisorFraction float64
582 PollInterval Duration
583 ProbeInterval Duration
585 SyncInterval Duration
586 TimeoutBooting Duration
588 TimeoutProbe Duration
589 TimeoutShutdown Duration
590 TimeoutSignal Duration
591 TimeoutStaleRunLock Duration
593 ResourceTags map[string]string
597 DriverParameters json.RawMessage
600 type InstanceTypeMap map[string]InstanceType
602 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
604 // UnmarshalJSON does special handling of InstanceTypes:
606 // - populate computed fields (Name and Scratch)
608 // - error out if InstancesTypes are populated as an array, which was
609 // deprecated in Arvados 1.2.0
610 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
611 fixup := func(t InstanceType) (InstanceType, error) {
612 if t.ProviderType == "" {
613 t.ProviderType = t.Name
615 // If t.Scratch is set in the configuration file, it will be ignored and overwritten.
616 // It will also generate a "deprecated or unknown config entry" warning.
617 t.Scratch = t.IncludedScratch + t.AddedScratch
621 if len(data) > 0 && data[0] == '[' {
622 return fmt.Errorf("InstanceTypes must be specified as a map, not an array, see https://doc.arvados.org/admin/config.html")
624 var hash map[string]InstanceType
625 err := json.Unmarshal(data, &hash)
629 // Fill in Name field (and ProviderType field, if not
630 // specified) using hash key.
631 *it = InstanceTypeMap(hash)
632 for name, t := range *it {
643 type StringSet map[string]struct{}
645 // UnmarshalJSON handles old config files that provide an array of
646 // instance types instead of a hash.
647 func (ss *StringSet) UnmarshalJSON(data []byte) error {
648 if len(data) > 0 && data[0] == '[' {
650 err := json.Unmarshal(data, &arr)
658 *ss = make(map[string]struct{}, len(arr))
659 for _, t := range arr {
660 (*ss)[t] = struct{}{}
664 var hash map[string]struct{}
665 err := json.Unmarshal(data, &hash)
669 *ss = make(map[string]struct{}, len(hash))
670 for t := range hash {
671 (*ss)[t] = struct{}{}
677 type ServiceName string
680 ServiceNameController ServiceName = "arvados-controller"
681 ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
682 ServiceNameDispatchLSF ServiceName = "arvados-dispatch-lsf"
683 ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm"
684 ServiceNameGitHTTP ServiceName = "arvados-git-httpd"
685 ServiceNameHealth ServiceName = "arvados-health"
686 ServiceNameKeepbalance ServiceName = "keep-balance"
687 ServiceNameKeepproxy ServiceName = "keepproxy"
688 ServiceNameKeepstore ServiceName = "keepstore"
689 ServiceNameKeepweb ServiceName = "keep-web"
690 ServiceNameRailsAPI ServiceName = "arvados-api-server"
691 ServiceNameWebsocket ServiceName = "arvados-ws"
692 ServiceNameWorkbench1 ServiceName = "arvados-workbench1"
693 ServiceNameWorkbench2 ServiceName = "arvados-workbench2"
696 // Map returns all services as a map, suitable for iterating over all
697 // services or looking up a service by name.
698 func (svcs Services) Map() map[ServiceName]Service {
699 return map[ServiceName]Service{
700 ServiceNameController: svcs.Controller,
701 ServiceNameDispatchCloud: svcs.DispatchCloud,
702 ServiceNameDispatchLSF: svcs.DispatchLSF,
703 ServiceNameDispatchSLURM: svcs.DispatchSLURM,
704 ServiceNameGitHTTP: svcs.GitHTTP,
705 ServiceNameHealth: svcs.Health,
706 ServiceNameKeepbalance: svcs.Keepbalance,
707 ServiceNameKeepproxy: svcs.Keepproxy,
708 ServiceNameKeepstore: svcs.Keepstore,
709 ServiceNameKeepweb: svcs.WebDAV,
710 ServiceNameRailsAPI: svcs.RailsAPI,
711 ServiceNameWebsocket: svcs.Websocket,
712 ServiceNameWorkbench1: svcs.Workbench1,
713 ServiceNameWorkbench2: svcs.Workbench2,