1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: Apache-2.0
16 "git.arvados.org/arvados.git/sdk/go/config"
19 var DefaultConfigFile = func() string {
20 if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
23 return "/etc/arvados/config.yml"
27 Clusters map[string]Cluster
29 SourceTimestamp time.Time
33 // GetConfig returns the current system config, loading it from
34 // configFile if needed.
35 func GetConfig(configFile string) (*Config, error) {
37 err := config.LoadFile(&cfg, configFile)
41 // GetCluster returns the cluster ID and config for the given
42 // cluster, or the default/only configured cluster if clusterID is "".
43 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
45 if len(sc.Clusters) == 0 {
46 return nil, fmt.Errorf("no clusters configured")
47 } else if len(sc.Clusters) > 1 {
48 return nil, fmt.Errorf("multiple clusters configured, cannot choose")
50 for id, cc := range sc.Clusters {
56 cc, ok := sc.Clusters[clusterID]
58 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
60 cc.ClusterID = clusterID
64 type WebDAVCacheConfig struct {
67 MaxCollectionBytes int64
71 type UploadDownloadPermission struct {
76 type UploadDownloadRolePermissions struct {
77 User UploadDownloadPermission
78 Admin UploadDownloadPermission
81 type ManagedProperties map[string]struct {
88 ClusterID string `json:"-"`
89 ManagementToken string
90 SystemRootToken string
92 InstanceTypes InstanceTypeMap
93 Containers ContainersConfig
94 RemoteClusters map[string]RemoteCluster
98 AsyncPermissionsUpdateInterval Duration
99 DisabledAPIs StringSet
100 MaxIndexDatabaseRead int
101 MaxItemsPerResponse int
102 MaxConcurrentRailsRequests int
103 MaxConcurrentRequests int
104 MaxQueuedRequests int
105 MaxQueueTimeForLockRequests Duration
106 LogCreateRequestFraction float64
107 MaxKeepBlobBuffers int
108 MaxRequestAmplification int
110 MaxTokenLifetime Duration
111 RequestTimeout Duration
113 WebsocketClientEventQueue int
114 WebsocketServerEventQueue int
115 KeepServiceRequestTimeout Duration
116 VocabularyPath string
117 FreezeProjectRequiresDescription bool
118 FreezeProjectRequiresProperties StringSet
119 UnfreezeProjectRequiresAdmin bool
120 LockBeforeUpdate bool
125 UnloggedAttributes StringSet
129 BlobSigningKey string
130 BlobSigningTTL Duration
132 BlobTrashLifetime Duration
133 BlobTrashCheckInterval Duration
134 BlobTrashConcurrency int
135 BlobDeleteConcurrency int
136 BlobReplicateConcurrency int
137 CollectionVersioning bool
138 DefaultTrashLifetime Duration
139 DefaultReplication int
140 ManagedProperties ManagedProperties
141 PreserveVersionIfIdle Duration
142 TrashSweepInterval Duration
144 ForwardSlashNameSubstitution string
147 BlobMissingReport string
148 BalancePeriod Duration
149 BalanceCollectionBatch int
150 BalanceCollectionBuffers int
151 BalanceTimeout Duration
152 BalanceUpdateLimit int
154 BalanceTrashLimit int
156 WebDAVCache WebDAVCacheConfig
158 KeepproxyPermission UploadDownloadRolePermissions
159 WebDAVPermission UploadDownloadRolePermissions
173 MinTLSVersion TLSVersion
176 SearchAttribute string
177 SearchBindUser string
178 SearchBindPassword string
181 EmailAttribute string
182 UsernameAttribute string
188 AlternateEmailAddresses bool
189 AuthenticationRequestParameters map[string]string
191 OpenIDConnect struct {
197 EmailVerifiedClaim string
199 AcceptAccessToken bool
200 AcceptAccessTokenScope string
201 AuthenticationRequestParameters map[string]string
206 DefaultEmailDomain string
210 Users map[string]TestUser
213 RemoteTokenRefresh Duration
214 TokenLifetime Duration
215 TrustedClients map[URL]struct{}
216 TrustPrivateNetworks bool
217 IssueTrustedTokens bool
220 MailchimpAPIKey string
221 MailchimpListID string
222 SendUserSetupNotificationEmail bool
223 IssueReporterEmailFrom string
224 IssueReporterEmailTo string
225 SupportEmailAddress string
231 MaxRequestLogParamsSize int
232 RequestQueueDumpDirectory string
243 ActivatedUsersAreVisibleToOthers bool
244 AnonymousUserToken string
245 AdminNotifierEmailFrom string
246 AutoAdminFirstUser bool
247 AutoAdminUserWithEmail string
248 AutoSetupNewUsers bool
249 AutoSetupNewUsersWithRepository bool
250 AutoSetupNewUsersWithVmUUID string
251 AutoSetupUsernameBlacklist StringSet
252 EmailSubjectPrefix string
253 NewInactiveUserNotificationRecipients StringSet
254 NewUserNotificationRecipients StringSet
255 NewUsersAreActive bool
256 UserNotifierEmailFrom string
257 UserNotifierEmailBcc StringSet
258 UserProfileNotificationAddress string
259 PreferDomainForUsername string
260 UserSetupMailText string
261 RoleGroupsVisibleToAll bool
262 CanCreateRoleGroups bool
263 ActivityLoggingPeriod Duration
264 SyncIgnoredGroups []string
265 SyncRequiredGroups []string
266 SyncUserAccounts bool
267 SyncUserAPITokens bool
271 StorageClasses map[string]StorageClassConfig
272 Volumes map[string]Volume
274 ActivationContactLink string
275 APIClientConnectTimeout Duration
276 APIClientReceiveTimeout Duration
277 APIResponseCompression bool
278 ApplicationMimetypesWithViewIcon StringSet
279 ArvadosDocsite string
280 ArvadosPublicDataDocURL string
281 DefaultOpenIdPrefix string
282 DisableSharingURLsUI bool
283 EnableGettingStartedPopup bool
284 EnablePublicProjectsPage bool
285 FileViewersConfigURL string
286 LogViewerMaxBytes ByteSize
287 MultiSiteSearch string
288 ProfilingEnabled bool
290 RepositoryCache string
291 RunningJobLogRecordsToFetch int
293 ShowRecentCollectionsOnDashboard bool
294 ShowUserAgreementInline bool
295 ShowUserNotifications bool
298 UserProfileFormFields map[string]struct {
300 FormFieldTitle string
301 FormFieldDescription string
304 Options map[string]struct{}
306 UserProfileFormMessage string
307 WelcomePageHTML string
308 InactivePageHTML string
309 SSHHelpPageHTML string
310 SSHHelpHostSuffix string
316 type StorageClassConfig struct {
322 AccessViaHosts map[URL]VolumeAccess
324 AllowTrashWhenReadOnly bool
326 StorageClasses map[string]bool
328 DriverParameters json.RawMessage
331 type S3VolumeDriverParameters struct {
334 SecretAccessKey string
338 LocationConstraint bool
341 ConnectTimeout Duration
348 type AzureVolumeDriverParameters struct {
349 StorageAccountName string
350 StorageAccountKey string
351 StorageBaseURL string
353 RequestTimeout Duration
354 ListBlobsRetryDelay Duration
355 ListBlobsMaxAttempts int
358 type DirectoryVolumeDriverParameters struct {
363 type VolumeAccess struct {
367 type Services struct {
370 DispatchCloud Service
372 DispatchSLURM Service
380 WebDAVDownload Service
388 type Service struct {
389 InternalURLs map[URL]ServiceInstance
393 type TestUser struct {
398 // URL is a url.URL that is also usable as a JSON key/value.
401 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
402 // used as a JSON key/value.
403 func (su *URL) UnmarshalText(text []byte) error {
404 u, err := url.Parse(string(text))
407 if su.Path == "" && su.Host != "" {
408 // http://example really means http://example/
415 func (su URL) MarshalText() ([]byte, error) {
416 return []byte(su.String()), nil
419 func (su URL) String() string {
420 return (*url.URL)(&su).String()
423 type TLSVersion uint16
425 func (v TLSVersion) MarshalText() ([]byte, error) {
429 case tls.VersionTLS10:
430 return []byte("1.0"), nil
431 case tls.VersionTLS11:
432 return []byte("1.1"), nil
433 case tls.VersionTLS12:
434 return []byte("1.2"), nil
435 case tls.VersionTLS13:
436 return []byte("1.3"), nil
438 return nil, fmt.Errorf("unsupported TLSVersion %x", v)
442 func (v *TLSVersion) UnmarshalJSON(text []byte) error {
443 if len(text) > 0 && text[0] == '"' {
445 err := json.Unmarshal(text, &s)
451 switch string(text) {
455 *v = tls.VersionTLS10
457 *v = tls.VersionTLS11
459 *v = tls.VersionTLS12
461 *v = tls.VersionTLS13
463 return fmt.Errorf("unsupported TLSVersion %q", text)
468 type ServiceInstance struct {
470 Rendezvous string `json:",omitempty"`
473 type PostgreSQL struct {
474 Connection PostgreSQLConnection
478 type PostgreSQLConnection map[string]string
480 type RemoteCluster struct {
488 type CUDAFeatures struct {
490 HardwareCapability string
494 type InstanceType struct {
495 Name string `json:"-"`
499 Scratch ByteSize `json:"-"`
500 IncludedScratch ByteSize
501 AddedScratch ByteSize
507 type ContainersConfig struct {
508 CloudVMs CloudVMsConfig
509 CrunchRunCommand string
510 CrunchRunArgumentsList []string
511 DefaultKeepCacheRAM ByteSize
512 DispatchPrivateKey string
513 LogReuseDecisions bool
514 MaxDispatchAttempts int
516 MinRetryPeriod Duration
517 ReserveExtraRAM ByteSize
518 StaleLockTimeout Duration
519 SupportedDockerImageFormats StringSet
520 AlwaysUsePreemptibleInstances bool
521 PreemptiblePriceFactor float64
522 MaximumPriceFactor float64
524 LocalKeepBlobBuffersPerVCPU int
525 LocalKeepLogsToContainerLog string
529 GitInternalDir string
533 SweepInterval Duration
535 LogSecondsBetweenEvents Duration
536 LogThrottlePeriod Duration
539 LimitLogBytesPerJob int
540 LogPartialLineThrottlePeriod Duration
541 LogUpdatePeriod Duration
542 LogUpdateSize ByteSize
550 SbatchArgumentsList []string
551 SbatchEnvironmentVariables map[string]string
553 DNSServerConfDir string
554 DNSServerConfTemplate string
555 DNSServerReloadCommand string
556 DNSServerUpdateCommand string
557 ComputeNodeDomain string
558 ComputeNodeNameservers StringSet
559 AssignNodeHostname string
564 BsubArgumentsList []string
565 BsubCUDAArguments []string
569 type CloudVMsConfig struct {
572 BootProbeCommand string
573 InstanceInitCommand string
574 DeployRunnerBinary string
577 MaxCloudOpsPerSecond int
578 MaxProbesPerSecond int
579 MaxConcurrentInstanceCreateOps int
581 InitialQuotaEstimate int
582 SupervisorFraction float64
583 PollInterval Duration
584 ProbeInterval Duration
586 SyncInterval Duration
587 TimeoutBooting Duration
589 TimeoutProbe Duration
590 TimeoutShutdown Duration
591 TimeoutSignal Duration
592 TimeoutStaleRunLock Duration
594 ResourceTags map[string]string
598 DriverParameters json.RawMessage
601 type InstanceTypeMap map[string]InstanceType
603 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
605 // UnmarshalJSON does special handling of InstanceTypes:
607 // - populate computed fields (Name and Scratch)
609 // - error out if InstancesTypes are populated as an array, which was
610 // deprecated in Arvados 1.2.0
611 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
612 fixup := func(t InstanceType) (InstanceType, error) {
613 if t.ProviderType == "" {
614 t.ProviderType = t.Name
616 // If t.Scratch is set in the configuration file, it will be ignored and overwritten.
617 // It will also generate a "deprecated or unknown config entry" warning.
618 t.Scratch = t.IncludedScratch + t.AddedScratch
622 if len(data) > 0 && data[0] == '[' {
623 return fmt.Errorf("InstanceTypes must be specified as a map, not an array, see https://doc.arvados.org/admin/config.html")
625 var hash map[string]InstanceType
626 err := json.Unmarshal(data, &hash)
630 // Fill in Name field (and ProviderType field, if not
631 // specified) using hash key.
632 *it = InstanceTypeMap(hash)
633 for name, t := range *it {
644 type StringSet map[string]struct{}
646 // UnmarshalJSON handles old config files that provide an array of
647 // instance types instead of a hash.
648 func (ss *StringSet) UnmarshalJSON(data []byte) error {
649 if len(data) > 0 && data[0] == '[' {
651 err := json.Unmarshal(data, &arr)
659 *ss = make(map[string]struct{}, len(arr))
660 for _, t := range arr {
661 (*ss)[t] = struct{}{}
665 var hash map[string]struct{}
666 err := json.Unmarshal(data, &hash)
670 *ss = make(map[string]struct{}, len(hash))
671 for t := range hash {
672 (*ss)[t] = struct{}{}
678 type ServiceName string
681 ServiceNameController ServiceName = "arvados-controller"
682 ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
683 ServiceNameDispatchLSF ServiceName = "arvados-dispatch-lsf"
684 ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm"
685 ServiceNameGitHTTP ServiceName = "arvados-git-httpd"
686 ServiceNameHealth ServiceName = "arvados-health"
687 ServiceNameKeepbalance ServiceName = "keep-balance"
688 ServiceNameKeepproxy ServiceName = "keepproxy"
689 ServiceNameKeepstore ServiceName = "keepstore"
690 ServiceNameKeepweb ServiceName = "keep-web"
691 ServiceNameRailsAPI ServiceName = "arvados-api-server"
692 ServiceNameWebsocket ServiceName = "arvados-ws"
693 ServiceNameWorkbench1 ServiceName = "arvados-workbench1"
694 ServiceNameWorkbench2 ServiceName = "arvados-workbench2"
697 // Map returns all services as a map, suitable for iterating over all
698 // services or looking up a service by name.
699 func (svcs Services) Map() map[ServiceName]Service {
700 return map[ServiceName]Service{
701 ServiceNameController: svcs.Controller,
702 ServiceNameDispatchCloud: svcs.DispatchCloud,
703 ServiceNameDispatchLSF: svcs.DispatchLSF,
704 ServiceNameDispatchSLURM: svcs.DispatchSLURM,
705 ServiceNameGitHTTP: svcs.GitHTTP,
706 ServiceNameHealth: svcs.Health,
707 ServiceNameKeepbalance: svcs.Keepbalance,
708 ServiceNameKeepproxy: svcs.Keepproxy,
709 ServiceNameKeepstore: svcs.Keepstore,
710 ServiceNameKeepweb: svcs.WebDAV,
711 ServiceNameRailsAPI: svcs.RailsAPI,
712 ServiceNameWebsocket: svcs.Websocket,
713 ServiceNameWorkbench1: svcs.Workbench1,
714 ServiceNameWorkbench2: svcs.Workbench2,