1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: Apache-2.0
16 "git.arvados.org/arvados.git/sdk/go/config"
19 var DefaultConfigFile = func() string {
20 if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
23 return "/etc/arvados/config.yml"
27 Clusters map[string]Cluster
29 SourceTimestamp time.Time
33 // GetConfig returns the current system config, loading it from
34 // configFile if needed.
35 func GetConfig(configFile string) (*Config, error) {
37 err := config.LoadFile(&cfg, configFile)
41 // GetCluster returns the cluster ID and config for the given
42 // cluster, or the default/only configured cluster if clusterID is "".
43 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
45 if len(sc.Clusters) == 0 {
46 return nil, fmt.Errorf("no clusters configured")
47 } else if len(sc.Clusters) > 1 {
48 return nil, fmt.Errorf("multiple clusters configured, cannot choose")
50 for id, cc := range sc.Clusters {
56 cc, ok := sc.Clusters[clusterID]
58 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
60 cc.ClusterID = clusterID
64 type WebDAVCacheConfig struct {
67 MaxCollectionBytes int64
71 type UploadDownloadPermission struct {
76 type UploadDownloadRolePermissions struct {
77 User UploadDownloadPermission
78 Admin UploadDownloadPermission
81 type ManagedProperties map[string]struct {
88 ClusterID string `json:"-"`
89 ManagementToken string
90 SystemRootToken string
92 InstanceTypes InstanceTypeMap
93 Containers ContainersConfig
94 RemoteClusters map[string]RemoteCluster
98 AsyncPermissionsUpdateInterval Duration
99 DisabledAPIs StringSet
100 MaxIndexDatabaseRead int
101 MaxItemsPerResponse int
102 MaxConcurrentRequests int
103 MaxQueuedRequests int
104 MaxQueueTimeForLockRequests Duration
105 LogCreateRequestFraction float64
106 MaxKeepBlobBuffers int
107 MaxRequestAmplification int
109 MaxTokenLifetime Duration
110 RequestTimeout Duration
112 WebsocketClientEventQueue int
113 WebsocketServerEventQueue int
114 KeepServiceRequestTimeout Duration
115 VocabularyPath string
116 FreezeProjectRequiresDescription bool
117 FreezeProjectRequiresProperties StringSet
118 UnfreezeProjectRequiresAdmin bool
119 LockBeforeUpdate bool
124 UnloggedAttributes StringSet
128 BlobSigningKey string
129 BlobSigningTTL Duration
131 BlobTrashLifetime Duration
132 BlobTrashCheckInterval Duration
133 BlobTrashConcurrency int
134 BlobDeleteConcurrency int
135 BlobReplicateConcurrency int
136 CollectionVersioning bool
137 DefaultTrashLifetime Duration
138 DefaultReplication int
139 ManagedProperties ManagedProperties
140 PreserveVersionIfIdle Duration
141 TrashSweepInterval Duration
143 ForwardSlashNameSubstitution string
146 BlobMissingReport string
147 BalancePeriod Duration
148 BalanceCollectionBatch int
149 BalanceCollectionBuffers int
150 BalanceTimeout Duration
151 BalanceUpdateLimit int
153 WebDAVCache WebDAVCacheConfig
155 KeepproxyPermission UploadDownloadRolePermissions
156 WebDAVPermission UploadDownloadRolePermissions
170 MinTLSVersion TLSVersion
173 SearchAttribute string
174 SearchBindUser string
175 SearchBindPassword string
178 EmailAttribute string
179 UsernameAttribute string
185 AlternateEmailAddresses bool
186 AuthenticationRequestParameters map[string]string
188 OpenIDConnect struct {
194 EmailVerifiedClaim string
196 AcceptAccessToken bool
197 AcceptAccessTokenScope string
198 AuthenticationRequestParameters map[string]string
203 DefaultEmailDomain string
207 Users map[string]TestUser
210 RemoteTokenRefresh Duration
211 TokenLifetime Duration
212 TrustedClients map[URL]struct{}
213 TrustPrivateNetworks bool
214 IssueTrustedTokens bool
217 MailchimpAPIKey string
218 MailchimpListID string
219 SendUserSetupNotificationEmail bool
220 IssueReporterEmailFrom string
221 IssueReporterEmailTo string
222 SupportEmailAddress string
228 MaxRequestLogParamsSize int
229 RequestQueueDumpDirectory string
240 ActivatedUsersAreVisibleToOthers bool
241 AnonymousUserToken string
242 AdminNotifierEmailFrom string
243 AutoAdminFirstUser bool
244 AutoAdminUserWithEmail string
245 AutoSetupNewUsers bool
246 AutoSetupNewUsersWithRepository bool
247 AutoSetupNewUsersWithVmUUID string
248 AutoSetupUsernameBlacklist StringSet
249 EmailSubjectPrefix string
250 NewInactiveUserNotificationRecipients StringSet
251 NewUserNotificationRecipients StringSet
252 NewUsersAreActive bool
253 UserNotifierEmailFrom string
254 UserNotifierEmailBcc StringSet
255 UserProfileNotificationAddress string
256 PreferDomainForUsername string
257 UserSetupMailText string
258 RoleGroupsVisibleToAll bool
259 CanCreateRoleGroups bool
260 ActivityLoggingPeriod Duration
261 SyncIgnoredGroups []string
262 SyncRequiredGroups []string
263 SyncUserAccounts bool
264 SyncUserAPITokens bool
268 StorageClasses map[string]StorageClassConfig
269 Volumes map[string]Volume
271 ActivationContactLink string
272 APIClientConnectTimeout Duration
273 APIClientReceiveTimeout Duration
274 APIResponseCompression bool
275 ApplicationMimetypesWithViewIcon StringSet
276 ArvadosDocsite string
277 ArvadosPublicDataDocURL string
278 DefaultOpenIdPrefix string
279 DisableSharingURLsUI bool
280 EnableGettingStartedPopup bool
281 EnablePublicProjectsPage bool
282 FileViewersConfigURL string
283 LogViewerMaxBytes ByteSize
284 MultiSiteSearch string
285 ProfilingEnabled bool
287 RepositoryCache string
288 RunningJobLogRecordsToFetch int
290 ShowRecentCollectionsOnDashboard bool
291 ShowUserAgreementInline bool
292 ShowUserNotifications bool
295 UserProfileFormFields map[string]struct {
297 FormFieldTitle string
298 FormFieldDescription string
301 Options map[string]struct{}
303 UserProfileFormMessage string
304 WelcomePageHTML string
305 InactivePageHTML string
306 SSHHelpPageHTML string
307 SSHHelpHostSuffix string
313 type StorageClassConfig struct {
319 AccessViaHosts map[URL]VolumeAccess
322 StorageClasses map[string]bool
324 DriverParameters json.RawMessage
327 type S3VolumeDriverParameters struct {
330 SecretAccessKey string
334 LocationConstraint bool
337 ConnectTimeout Duration
344 type AzureVolumeDriverParameters struct {
345 StorageAccountName string
346 StorageAccountKey string
347 StorageBaseURL string
349 RequestTimeout Duration
350 ListBlobsRetryDelay Duration
351 ListBlobsMaxAttempts int
354 type DirectoryVolumeDriverParameters struct {
359 type VolumeAccess struct {
363 type Services struct {
366 DispatchCloud Service
368 DispatchSLURM Service
376 WebDAVDownload Service
384 type Service struct {
385 InternalURLs map[URL]ServiceInstance
389 type TestUser struct {
394 // URL is a url.URL that is also usable as a JSON key/value.
397 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
398 // used as a JSON key/value.
399 func (su *URL) UnmarshalText(text []byte) error {
400 u, err := url.Parse(string(text))
403 if su.Path == "" && su.Host != "" {
404 // http://example really means http://example/
411 func (su URL) MarshalText() ([]byte, error) {
412 return []byte(su.String()), nil
415 func (su URL) String() string {
416 return (*url.URL)(&su).String()
419 type TLSVersion uint16
421 func (v TLSVersion) MarshalText() ([]byte, error) {
425 case tls.VersionTLS10:
426 return []byte("1.0"), nil
427 case tls.VersionTLS11:
428 return []byte("1.1"), nil
429 case tls.VersionTLS12:
430 return []byte("1.2"), nil
431 case tls.VersionTLS13:
432 return []byte("1.3"), nil
434 return nil, fmt.Errorf("unsupported TLSVersion %x", v)
438 func (v *TLSVersion) UnmarshalJSON(text []byte) error {
439 if len(text) > 0 && text[0] == '"' {
441 err := json.Unmarshal(text, &s)
447 switch string(text) {
451 *v = tls.VersionTLS10
453 *v = tls.VersionTLS11
455 *v = tls.VersionTLS12
457 *v = tls.VersionTLS13
459 return fmt.Errorf("unsupported TLSVersion %q", text)
464 type ServiceInstance struct {
466 Rendezvous string `json:",omitempty"`
469 type PostgreSQL struct {
470 Connection PostgreSQLConnection
474 type PostgreSQLConnection map[string]string
476 type RemoteCluster struct {
484 type CUDAFeatures struct {
486 HardwareCapability string
490 type InstanceType struct {
491 Name string `json:"-"`
495 Scratch ByteSize `json:"-"`
496 IncludedScratch ByteSize
497 AddedScratch ByteSize
503 type ContainersConfig struct {
504 CloudVMs CloudVMsConfig
505 CrunchRunCommand string
506 CrunchRunArgumentsList []string
507 DefaultKeepCacheRAM ByteSize
508 DispatchPrivateKey string
509 LogReuseDecisions bool
510 MaxDispatchAttempts int
512 MinRetryPeriod Duration
513 ReserveExtraRAM ByteSize
514 StaleLockTimeout Duration
515 SupportedDockerImageFormats StringSet
516 AlwaysUsePreemptibleInstances bool
517 PreemptiblePriceFactor float64
519 LocalKeepBlobBuffersPerVCPU int
520 LocalKeepLogsToContainerLog string
524 GitInternalDir string
528 SweepInterval Duration
530 LogSecondsBetweenEvents Duration
531 LogThrottlePeriod Duration
534 LimitLogBytesPerJob int
535 LogPartialLineThrottlePeriod Duration
536 LogUpdatePeriod Duration
537 LogUpdateSize ByteSize
545 SbatchArgumentsList []string
546 SbatchEnvironmentVariables map[string]string
548 DNSServerConfDir string
549 DNSServerConfTemplate string
550 DNSServerReloadCommand string
551 DNSServerUpdateCommand string
552 ComputeNodeDomain string
553 ComputeNodeNameservers StringSet
554 AssignNodeHostname string
559 BsubArgumentsList []string
560 BsubCUDAArguments []string
564 type CloudVMsConfig struct {
567 BootProbeCommand string
568 InstanceInitCommand string
569 DeployRunnerBinary string
572 MaxCloudOpsPerSecond int
573 MaxProbesPerSecond int
574 MaxConcurrentInstanceCreateOps int
576 InitialQuotaEstimate int
577 SupervisorFraction float64
578 PollInterval Duration
579 ProbeInterval Duration
581 SyncInterval Duration
582 TimeoutBooting Duration
584 TimeoutProbe Duration
585 TimeoutShutdown Duration
586 TimeoutSignal Duration
587 TimeoutStaleRunLock Duration
589 ResourceTags map[string]string
593 DriverParameters json.RawMessage
596 type InstanceTypeMap map[string]InstanceType
598 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
600 // UnmarshalJSON does special handling of InstanceTypes:
602 // - populate computed fields (Name and Scratch)
604 // - error out if InstancesTypes are populated as an array, which was
605 // deprecated in Arvados 1.2.0
606 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
607 fixup := func(t InstanceType) (InstanceType, error) {
608 if t.ProviderType == "" {
609 t.ProviderType = t.Name
611 // If t.Scratch is set in the configuration file, it will be ignored and overwritten.
612 // It will also generate a "deprecated or unknown config entry" warning.
613 t.Scratch = t.IncludedScratch + t.AddedScratch
617 if len(data) > 0 && data[0] == '[' {
618 return fmt.Errorf("InstanceTypes must be specified as a map, not an array, see https://doc.arvados.org/admin/config.html")
620 var hash map[string]InstanceType
621 err := json.Unmarshal(data, &hash)
625 // Fill in Name field (and ProviderType field, if not
626 // specified) using hash key.
627 *it = InstanceTypeMap(hash)
628 for name, t := range *it {
639 type StringSet map[string]struct{}
641 // UnmarshalJSON handles old config files that provide an array of
642 // instance types instead of a hash.
643 func (ss *StringSet) UnmarshalJSON(data []byte) error {
644 if len(data) > 0 && data[0] == '[' {
646 err := json.Unmarshal(data, &arr)
654 *ss = make(map[string]struct{}, len(arr))
655 for _, t := range arr {
656 (*ss)[t] = struct{}{}
660 var hash map[string]struct{}
661 err := json.Unmarshal(data, &hash)
665 *ss = make(map[string]struct{}, len(hash))
666 for t := range hash {
667 (*ss)[t] = struct{}{}
673 type ServiceName string
676 ServiceNameController ServiceName = "arvados-controller"
677 ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
678 ServiceNameDispatchLSF ServiceName = "arvados-dispatch-lsf"
679 ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm"
680 ServiceNameGitHTTP ServiceName = "arvados-git-httpd"
681 ServiceNameHealth ServiceName = "arvados-health"
682 ServiceNameKeepbalance ServiceName = "keep-balance"
683 ServiceNameKeepproxy ServiceName = "keepproxy"
684 ServiceNameKeepstore ServiceName = "keepstore"
685 ServiceNameKeepweb ServiceName = "keep-web"
686 ServiceNameRailsAPI ServiceName = "arvados-api-server"
687 ServiceNameWebsocket ServiceName = "arvados-ws"
688 ServiceNameWorkbench1 ServiceName = "arvados-workbench1"
689 ServiceNameWorkbench2 ServiceName = "arvados-workbench2"
692 // Map returns all services as a map, suitable for iterating over all
693 // services or looking up a service by name.
694 func (svcs Services) Map() map[ServiceName]Service {
695 return map[ServiceName]Service{
696 ServiceNameController: svcs.Controller,
697 ServiceNameDispatchCloud: svcs.DispatchCloud,
698 ServiceNameDispatchLSF: svcs.DispatchLSF,
699 ServiceNameDispatchSLURM: svcs.DispatchSLURM,
700 ServiceNameGitHTTP: svcs.GitHTTP,
701 ServiceNameHealth: svcs.Health,
702 ServiceNameKeepbalance: svcs.Keepbalance,
703 ServiceNameKeepproxy: svcs.Keepproxy,
704 ServiceNameKeepstore: svcs.Keepstore,
705 ServiceNameKeepweb: svcs.WebDAV,
706 ServiceNameRailsAPI: svcs.RailsAPI,
707 ServiceNameWebsocket: svcs.Websocket,
708 ServiceNameWorkbench1: svcs.Workbench1,
709 ServiceNameWorkbench2: svcs.Workbench2,