1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: Apache-2.0
16 "git.arvados.org/arvados.git/sdk/go/config"
19 var DefaultConfigFile = func() string {
20 if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
23 return "/etc/arvados/config.yml"
27 Clusters map[string]Cluster
29 SourceTimestamp time.Time
33 // GetConfig returns the current system config, loading it from
34 // configFile if needed.
35 func GetConfig(configFile string) (*Config, error) {
37 err := config.LoadFile(&cfg, configFile)
41 // GetCluster returns the cluster ID and config for the given
42 // cluster, or the default/only configured cluster if clusterID is "".
43 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
45 if len(sc.Clusters) == 0 {
46 return nil, fmt.Errorf("no clusters configured")
47 } else if len(sc.Clusters) > 1 {
48 return nil, fmt.Errorf("multiple clusters configured, cannot choose")
50 for id, cc := range sc.Clusters {
56 cc, ok := sc.Clusters[clusterID]
58 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
60 cc.ClusterID = clusterID
64 type WebDAVCacheConfig struct {
67 MaxCollectionBytes int64
71 type UploadDownloadPermission struct {
76 type UploadDownloadRolePermissions struct {
77 User UploadDownloadPermission
78 Admin UploadDownloadPermission
81 type ManagedProperties map[string]struct {
88 ClusterID string `json:"-"`
89 ManagementToken string
90 SystemRootToken string
92 InstanceTypes InstanceTypeMap
93 Containers ContainersConfig
94 RemoteClusters map[string]RemoteCluster
98 AsyncPermissionsUpdateInterval Duration
99 DisabledAPIs StringSet
100 MaxIndexDatabaseRead int
101 MaxItemsPerResponse int
102 MaxConcurrentRailsRequests int
103 MaxConcurrentRequests int
104 MaxQueuedRequests int
105 MaxQueueTimeForLockRequests Duration
106 LogCreateRequestFraction float64
107 MaxKeepBlobBuffers int
108 MaxRequestAmplification int
110 MaxTokenLifetime Duration
111 RequestTimeout Duration
113 WebsocketClientEventQueue int
114 WebsocketServerEventQueue int
115 KeepServiceRequestTimeout Duration
116 VocabularyPath string
117 FreezeProjectRequiresDescription bool
118 FreezeProjectRequiresProperties StringSet
119 UnfreezeProjectRequiresAdmin bool
120 LockBeforeUpdate bool
125 UnloggedAttributes StringSet
129 BlobSigningKey string
130 BlobSigningTTL Duration
132 BlobTrashLifetime Duration
133 BlobTrashCheckInterval Duration
134 BlobTrashConcurrency int
135 BlobDeleteConcurrency int
136 BlobReplicateConcurrency int
137 CollectionVersioning bool
138 DefaultTrashLifetime Duration
139 DefaultReplication int
140 ManagedProperties ManagedProperties
141 PreserveVersionIfIdle Duration
142 TrashSweepInterval Duration
144 ForwardSlashNameSubstitution string
147 BlobMissingReport string
148 BalancePeriod Duration
149 BalanceCollectionBatch int
150 BalanceCollectionBuffers int
151 BalanceTimeout Duration
152 BalanceUpdateLimit int
154 WebDAVCache WebDAVCacheConfig
156 KeepproxyPermission UploadDownloadRolePermissions
157 WebDAVPermission UploadDownloadRolePermissions
171 MinTLSVersion TLSVersion
174 SearchAttribute string
175 SearchBindUser string
176 SearchBindPassword string
179 EmailAttribute string
180 UsernameAttribute string
186 AlternateEmailAddresses bool
187 AuthenticationRequestParameters map[string]string
189 OpenIDConnect struct {
195 EmailVerifiedClaim string
197 AcceptAccessToken bool
198 AcceptAccessTokenScope string
199 AuthenticationRequestParameters map[string]string
204 DefaultEmailDomain string
208 Users map[string]TestUser
211 RemoteTokenRefresh Duration
212 TokenLifetime Duration
213 TrustedClients map[URL]struct{}
214 TrustPrivateNetworks bool
215 IssueTrustedTokens bool
218 MailchimpAPIKey string
219 MailchimpListID string
220 SendUserSetupNotificationEmail bool
221 IssueReporterEmailFrom string
222 IssueReporterEmailTo string
223 SupportEmailAddress string
229 MaxRequestLogParamsSize int
230 RequestQueueDumpDirectory string
241 ActivatedUsersAreVisibleToOthers bool
242 AnonymousUserToken string
243 AdminNotifierEmailFrom string
244 AutoAdminFirstUser bool
245 AutoAdminUserWithEmail string
246 AutoSetupNewUsers bool
247 AutoSetupNewUsersWithRepository bool
248 AutoSetupNewUsersWithVmUUID string
249 AutoSetupUsernameBlacklist StringSet
250 EmailSubjectPrefix string
251 NewInactiveUserNotificationRecipients StringSet
252 NewUserNotificationRecipients StringSet
253 NewUsersAreActive bool
254 UserNotifierEmailFrom string
255 UserNotifierEmailBcc StringSet
256 UserProfileNotificationAddress string
257 PreferDomainForUsername string
258 UserSetupMailText string
259 RoleGroupsVisibleToAll bool
260 CanCreateRoleGroups bool
261 ActivityLoggingPeriod Duration
262 SyncIgnoredGroups []string
263 SyncRequiredGroups []string
264 SyncUserAccounts bool
265 SyncUserAPITokens bool
269 StorageClasses map[string]StorageClassConfig
270 Volumes map[string]Volume
272 ActivationContactLink string
273 APIClientConnectTimeout Duration
274 APIClientReceiveTimeout Duration
275 APIResponseCompression bool
276 ApplicationMimetypesWithViewIcon StringSet
277 ArvadosDocsite string
278 ArvadosPublicDataDocURL string
279 DefaultOpenIdPrefix string
280 DisableSharingURLsUI bool
281 EnableGettingStartedPopup bool
282 EnablePublicProjectsPage bool
283 FileViewersConfigURL string
284 LogViewerMaxBytes ByteSize
285 MultiSiteSearch string
286 ProfilingEnabled bool
288 RepositoryCache string
289 RunningJobLogRecordsToFetch int
291 ShowRecentCollectionsOnDashboard bool
292 ShowUserAgreementInline bool
293 ShowUserNotifications bool
296 UserProfileFormFields map[string]struct {
298 FormFieldTitle string
299 FormFieldDescription string
302 Options map[string]struct{}
304 UserProfileFormMessage string
305 WelcomePageHTML string
306 InactivePageHTML string
307 SSHHelpPageHTML string
308 SSHHelpHostSuffix string
314 type StorageClassConfig struct {
320 AccessViaHosts map[URL]VolumeAccess
322 AllowTrashWhenReadOnly bool
324 StorageClasses map[string]bool
326 DriverParameters json.RawMessage
329 type S3VolumeDriverParameters struct {
332 SecretAccessKey string
336 LocationConstraint bool
339 ConnectTimeout Duration
346 type AzureVolumeDriverParameters struct {
347 StorageAccountName string
348 StorageAccountKey string
349 StorageBaseURL string
351 RequestTimeout Duration
352 ListBlobsRetryDelay Duration
353 ListBlobsMaxAttempts int
356 type DirectoryVolumeDriverParameters struct {
361 type VolumeAccess struct {
365 type Services struct {
368 DispatchCloud Service
370 DispatchSLURM Service
378 WebDAVDownload Service
386 type Service struct {
387 InternalURLs map[URL]ServiceInstance
391 type TestUser struct {
396 // URL is a url.URL that is also usable as a JSON key/value.
399 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
400 // used as a JSON key/value.
401 func (su *URL) UnmarshalText(text []byte) error {
402 u, err := url.Parse(string(text))
405 if su.Path == "" && su.Host != "" {
406 // http://example really means http://example/
413 func (su URL) MarshalText() ([]byte, error) {
414 return []byte(su.String()), nil
417 func (su URL) String() string {
418 return (*url.URL)(&su).String()
421 type TLSVersion uint16
423 func (v TLSVersion) MarshalText() ([]byte, error) {
427 case tls.VersionTLS10:
428 return []byte("1.0"), nil
429 case tls.VersionTLS11:
430 return []byte("1.1"), nil
431 case tls.VersionTLS12:
432 return []byte("1.2"), nil
433 case tls.VersionTLS13:
434 return []byte("1.3"), nil
436 return nil, fmt.Errorf("unsupported TLSVersion %x", v)
440 func (v *TLSVersion) UnmarshalJSON(text []byte) error {
441 if len(text) > 0 && text[0] == '"' {
443 err := json.Unmarshal(text, &s)
449 switch string(text) {
453 *v = tls.VersionTLS10
455 *v = tls.VersionTLS11
457 *v = tls.VersionTLS12
459 *v = tls.VersionTLS13
461 return fmt.Errorf("unsupported TLSVersion %q", text)
466 type ServiceInstance struct {
468 Rendezvous string `json:",omitempty"`
471 type PostgreSQL struct {
472 Connection PostgreSQLConnection
476 type PostgreSQLConnection map[string]string
478 type RemoteCluster struct {
486 type CUDAFeatures struct {
488 HardwareCapability string
492 type InstanceType struct {
493 Name string `json:"-"`
497 Scratch ByteSize `json:"-"`
498 IncludedScratch ByteSize
499 AddedScratch ByteSize
505 type ContainersConfig struct {
506 CloudVMs CloudVMsConfig
507 CrunchRunCommand string
508 CrunchRunArgumentsList []string
509 DefaultKeepCacheRAM ByteSize
510 DispatchPrivateKey string
511 LogReuseDecisions bool
512 MaxDispatchAttempts int
514 MinRetryPeriod Duration
515 ReserveExtraRAM ByteSize
516 StaleLockTimeout Duration
517 SupportedDockerImageFormats StringSet
518 AlwaysUsePreemptibleInstances bool
519 PreemptiblePriceFactor float64
521 LocalKeepBlobBuffersPerVCPU int
522 LocalKeepLogsToContainerLog string
526 GitInternalDir string
530 SweepInterval Duration
532 LogSecondsBetweenEvents Duration
533 LogThrottlePeriod Duration
536 LimitLogBytesPerJob int
537 LogPartialLineThrottlePeriod Duration
538 LogUpdatePeriod Duration
539 LogUpdateSize ByteSize
547 SbatchArgumentsList []string
548 SbatchEnvironmentVariables map[string]string
550 DNSServerConfDir string
551 DNSServerConfTemplate string
552 DNSServerReloadCommand string
553 DNSServerUpdateCommand string
554 ComputeNodeDomain string
555 ComputeNodeNameservers StringSet
556 AssignNodeHostname string
561 BsubArgumentsList []string
562 BsubCUDAArguments []string
566 type CloudVMsConfig struct {
569 BootProbeCommand string
570 InstanceInitCommand string
571 DeployRunnerBinary string
574 MaxCloudOpsPerSecond int
575 MaxProbesPerSecond int
576 MaxConcurrentInstanceCreateOps int
578 InitialQuotaEstimate int
579 SupervisorFraction float64
580 PollInterval Duration
581 ProbeInterval Duration
583 SyncInterval Duration
584 TimeoutBooting Duration
586 TimeoutProbe Duration
587 TimeoutShutdown Duration
588 TimeoutSignal Duration
589 TimeoutStaleRunLock Duration
591 ResourceTags map[string]string
595 DriverParameters json.RawMessage
598 type InstanceTypeMap map[string]InstanceType
600 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
602 // UnmarshalJSON does special handling of InstanceTypes:
604 // - populate computed fields (Name and Scratch)
606 // - error out if InstancesTypes are populated as an array, which was
607 // deprecated in Arvados 1.2.0
608 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
609 fixup := func(t InstanceType) (InstanceType, error) {
610 if t.ProviderType == "" {
611 t.ProviderType = t.Name
613 // If t.Scratch is set in the configuration file, it will be ignored and overwritten.
614 // It will also generate a "deprecated or unknown config entry" warning.
615 t.Scratch = t.IncludedScratch + t.AddedScratch
619 if len(data) > 0 && data[0] == '[' {
620 return fmt.Errorf("InstanceTypes must be specified as a map, not an array, see https://doc.arvados.org/admin/config.html")
622 var hash map[string]InstanceType
623 err := json.Unmarshal(data, &hash)
627 // Fill in Name field (and ProviderType field, if not
628 // specified) using hash key.
629 *it = InstanceTypeMap(hash)
630 for name, t := range *it {
641 type StringSet map[string]struct{}
643 // UnmarshalJSON handles old config files that provide an array of
644 // instance types instead of a hash.
645 func (ss *StringSet) UnmarshalJSON(data []byte) error {
646 if len(data) > 0 && data[0] == '[' {
648 err := json.Unmarshal(data, &arr)
656 *ss = make(map[string]struct{}, len(arr))
657 for _, t := range arr {
658 (*ss)[t] = struct{}{}
662 var hash map[string]struct{}
663 err := json.Unmarshal(data, &hash)
667 *ss = make(map[string]struct{}, len(hash))
668 for t := range hash {
669 (*ss)[t] = struct{}{}
675 type ServiceName string
678 ServiceNameController ServiceName = "arvados-controller"
679 ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
680 ServiceNameDispatchLSF ServiceName = "arvados-dispatch-lsf"
681 ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm"
682 ServiceNameGitHTTP ServiceName = "arvados-git-httpd"
683 ServiceNameHealth ServiceName = "arvados-health"
684 ServiceNameKeepbalance ServiceName = "keep-balance"
685 ServiceNameKeepproxy ServiceName = "keepproxy"
686 ServiceNameKeepstore ServiceName = "keepstore"
687 ServiceNameKeepweb ServiceName = "keep-web"
688 ServiceNameRailsAPI ServiceName = "arvados-api-server"
689 ServiceNameWebsocket ServiceName = "arvados-ws"
690 ServiceNameWorkbench1 ServiceName = "arvados-workbench1"
691 ServiceNameWorkbench2 ServiceName = "arvados-workbench2"
694 // Map returns all services as a map, suitable for iterating over all
695 // services or looking up a service by name.
696 func (svcs Services) Map() map[ServiceName]Service {
697 return map[ServiceName]Service{
698 ServiceNameController: svcs.Controller,
699 ServiceNameDispatchCloud: svcs.DispatchCloud,
700 ServiceNameDispatchLSF: svcs.DispatchLSF,
701 ServiceNameDispatchSLURM: svcs.DispatchSLURM,
702 ServiceNameGitHTTP: svcs.GitHTTP,
703 ServiceNameHealth: svcs.Health,
704 ServiceNameKeepbalance: svcs.Keepbalance,
705 ServiceNameKeepproxy: svcs.Keepproxy,
706 ServiceNameKeepstore: svcs.Keepstore,
707 ServiceNameKeepweb: svcs.WebDAV,
708 ServiceNameRailsAPI: svcs.RailsAPI,
709 ServiceNameWebsocket: svcs.Websocket,
710 ServiceNameWorkbench1: svcs.Workbench1,
711 ServiceNameWorkbench2: svcs.Workbench2,