1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: Apache-2.0
16 "git.arvados.org/arvados.git/sdk/go/config"
19 var DefaultConfigFile = func() string {
20 if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
23 return "/etc/arvados/config.yml"
27 Clusters map[string]Cluster
29 SourceTimestamp time.Time
33 // GetConfig returns the current system config, loading it from
34 // configFile if needed.
35 func GetConfig(configFile string) (*Config, error) {
37 err := config.LoadFile(&cfg, configFile)
41 // GetCluster returns the cluster ID and config for the given
42 // cluster, or the default/only configured cluster if clusterID is "".
43 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
45 if len(sc.Clusters) == 0 {
46 return nil, fmt.Errorf("no clusters configured")
47 } else if len(sc.Clusters) > 1 {
48 return nil, fmt.Errorf("multiple clusters configured, cannot choose")
50 for id, cc := range sc.Clusters {
56 cc, ok := sc.Clusters[clusterID]
58 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
60 cc.ClusterID = clusterID
64 type WebDAVCacheConfig struct {
66 DiskCacheSize ByteSizeOrPercent
67 MaxCollectionBytes ByteSize
71 type UploadDownloadPermission struct {
76 type UploadDownloadRolePermissions struct {
77 User UploadDownloadPermission
78 Admin UploadDownloadPermission
81 type ManagedProperties map[string]struct {
88 ClusterID string `json:"-"`
89 ManagementToken string
90 SystemRootToken string
92 InstanceTypes InstanceTypeMap
93 Containers ContainersConfig
94 RemoteClusters map[string]RemoteCluster
98 AsyncPermissionsUpdateInterval Duration
99 DisabledAPIs StringSet
100 MaxIndexDatabaseRead int
101 MaxItemsPerResponse int
102 MaxConcurrentRailsRequests int
103 MaxConcurrentRequests int
104 MaxQueuedRequests int
105 MaxGatewayTunnels int
106 MaxQueueTimeForLockRequests Duration
107 LogCreateRequestFraction float64
108 MaxKeepBlobBuffers int
109 MaxRequestAmplification int
111 MaxTokenLifetime Duration
112 RequestTimeout Duration
114 WebsocketClientEventQueue int
115 WebsocketServerEventQueue int
116 KeepServiceRequestTimeout Duration
117 VocabularyPath string
118 FreezeProjectRequiresDescription bool
119 FreezeProjectRequiresProperties StringSet
120 UnfreezeProjectRequiresAdmin bool
121 LockBeforeUpdate bool
126 UnloggedAttributes StringSet
130 BlobSigningKey string
131 BlobSigningTTL Duration
133 BlobTrashLifetime Duration
134 BlobTrashCheckInterval Duration
135 BlobTrashConcurrency int
136 BlobDeleteConcurrency int
137 BlobReplicateConcurrency int
138 CollectionVersioning bool
139 DefaultTrashLifetime Duration
140 DefaultReplication int
141 ManagedProperties ManagedProperties
142 PreserveVersionIfIdle Duration
143 TrashSweepInterval Duration
145 ForwardSlashNameSubstitution string
148 BlobMissingReport string
149 BalancePeriod Duration
150 BalanceCollectionBatch int
151 BalanceCollectionBuffers int
152 BalanceTimeout Duration
153 BalanceUpdateLimit int
155 BalanceTrashLimit int
157 WebDAVCache WebDAVCacheConfig
159 KeepproxyPermission UploadDownloadRolePermissions
160 WebDAVPermission UploadDownloadRolePermissions
174 MinTLSVersion TLSVersion
177 SearchAttribute string
178 SearchBindUser string
179 SearchBindPassword string
182 EmailAttribute string
183 UsernameAttribute string
189 AlternateEmailAddresses bool
190 AuthenticationRequestParameters map[string]string
192 OpenIDConnect struct {
198 EmailVerifiedClaim string
200 AcceptAccessToken bool
201 AcceptAccessTokenScope string
202 AuthenticationRequestParameters map[string]string
207 DefaultEmailDomain string
211 Users map[string]TestUser
214 RemoteTokenRefresh Duration
215 TokenLifetime Duration
216 TrustedClients map[URL]struct{}
217 TrustPrivateNetworks bool
218 IssueTrustedTokens bool
221 MailchimpAPIKey string
222 MailchimpListID string
223 SendUserSetupNotificationEmail bool
224 IssueReporterEmailFrom string
225 IssueReporterEmailTo string
226 SupportEmailAddress string
232 MaxRequestLogParamsSize int
233 RequestQueueDumpDirectory string
244 ActivatedUsersAreVisibleToOthers bool
245 AnonymousUserToken string
246 AdminNotifierEmailFrom string
247 AutoAdminFirstUser bool
248 AutoAdminUserWithEmail string
249 AutoSetupNewUsers bool
250 AutoSetupNewUsersWithRepository bool
251 AutoSetupNewUsersWithVmUUID string
252 AutoSetupUsernameBlacklist StringSet
253 EmailSubjectPrefix string
254 NewInactiveUserNotificationRecipients StringSet
255 NewUserNotificationRecipients StringSet
256 NewUsersAreActive bool
257 UserNotifierEmailFrom string
258 UserNotifierEmailBcc StringSet
259 UserProfileNotificationAddress string
260 PreferDomainForUsername string
261 UserSetupMailText string
262 RoleGroupsVisibleToAll bool
263 CanCreateRoleGroups bool
264 ActivityLoggingPeriod Duration
265 SyncIgnoredGroups []string
266 SyncRequiredGroups []string
267 SyncUserAccounts bool
268 SyncUserAPITokens bool
272 StorageClasses map[string]StorageClassConfig
273 Volumes map[string]Volume
275 ActivationContactLink string
276 ArvadosDocsite string
277 ArvadosPublicDataDocURL string
278 DisableSharingURLsUI bool
279 FileViewersConfigURL string
280 ShowUserAgreementInline bool
283 UserProfileFormFields map[string]struct {
285 FormFieldTitle string
286 FormFieldDescription string
289 Options map[string]struct{}
291 UserProfileFormMessage string
292 WelcomePageHTML string
293 InactivePageHTML string
294 SSHHelpPageHTML string
295 SSHHelpHostSuffix string
301 type StorageClassConfig struct {
307 AccessViaHosts map[URL]VolumeAccess
309 AllowTrashWhenReadOnly bool
311 StorageClasses map[string]bool
313 DriverParameters json.RawMessage
316 type S3VolumeDriverParameters struct {
319 SecretAccessKey string
323 LocationConstraint bool
326 ConnectTimeout Duration
333 type AzureVolumeDriverParameters struct {
334 StorageAccountName string
335 StorageAccountKey string
336 StorageBaseURL string
338 RequestTimeout Duration
339 ListBlobsRetryDelay Duration
340 ListBlobsMaxAttempts int
343 type DirectoryVolumeDriverParameters struct {
348 type VolumeAccess struct {
352 type Services struct {
355 DispatchCloud Service
357 DispatchSLURM Service
365 WebDAVDownload Service
373 type Service struct {
374 InternalURLs map[URL]ServiceInstance
378 type TestUser struct {
383 // URL is a url.URL that is also usable as a JSON key/value.
386 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
387 // used as a JSON key/value.
388 func (su *URL) UnmarshalText(text []byte) error {
389 u, err := url.Parse(string(text))
392 if su.Path == "" && su.Host != "" {
393 // http://example really means http://example/
400 func (su URL) MarshalText() ([]byte, error) {
401 return []byte(su.String()), nil
404 func (su URL) String() string {
405 return (*url.URL)(&su).String()
408 type TLSVersion uint16
410 func (v TLSVersion) MarshalText() ([]byte, error) {
414 case tls.VersionTLS10:
415 return []byte("1.0"), nil
416 case tls.VersionTLS11:
417 return []byte("1.1"), nil
418 case tls.VersionTLS12:
419 return []byte("1.2"), nil
420 case tls.VersionTLS13:
421 return []byte("1.3"), nil
423 return nil, fmt.Errorf("unsupported TLSVersion %x", v)
427 func (v *TLSVersion) UnmarshalJSON(text []byte) error {
428 if len(text) > 0 && text[0] == '"' {
430 err := json.Unmarshal(text, &s)
436 switch string(text) {
440 *v = tls.VersionTLS10
442 *v = tls.VersionTLS11
444 *v = tls.VersionTLS12
446 *v = tls.VersionTLS13
448 return fmt.Errorf("unsupported TLSVersion %q", text)
453 type ServiceInstance struct {
455 Rendezvous string `json:",omitempty"`
458 type PostgreSQL struct {
459 Connection PostgreSQLConnection
463 type PostgreSQLConnection map[string]string
465 type RemoteCluster struct {
473 type CUDAFeatures struct {
475 HardwareCapability string
479 type InstanceType struct {
480 Name string `json:"-"`
484 Scratch ByteSize `json:"-"`
485 IncludedScratch ByteSize
486 AddedScratch ByteSize
492 type ContainersConfig struct {
493 CloudVMs CloudVMsConfig
494 CrunchRunCommand string
495 CrunchRunArgumentsList []string
496 DefaultKeepCacheRAM ByteSize
497 DispatchPrivateKey string
498 LogReuseDecisions bool
499 MaxDispatchAttempts int
501 MinRetryPeriod Duration
502 ReserveExtraRAM ByteSize
503 StaleLockTimeout Duration
504 SupportedDockerImageFormats StringSet
505 AlwaysUsePreemptibleInstances bool
506 PreemptiblePriceFactor float64
507 MaximumPriceFactor float64
509 LocalKeepBlobBuffersPerVCPU int
510 LocalKeepLogsToContainerLog string
514 GitInternalDir string
518 SweepInterval Duration
520 LogSecondsBetweenEvents Duration
521 LogThrottlePeriod Duration
524 LimitLogBytesPerJob int
525 LogPartialLineThrottlePeriod Duration
526 LogUpdatePeriod Duration
527 LogUpdateSize ByteSize
535 SbatchArgumentsList []string
536 SbatchEnvironmentVariables map[string]string
538 DNSServerConfDir string
539 DNSServerConfTemplate string
540 DNSServerReloadCommand string
541 DNSServerUpdateCommand string
542 ComputeNodeDomain string
543 ComputeNodeNameservers StringSet
544 AssignNodeHostname string
549 BsubArgumentsList []string
550 BsubCUDAArguments []string
551 MaxRunTimeOverhead Duration
552 MaxRunTimeDefault Duration
556 type CloudVMsConfig struct {
559 BootProbeCommand string
560 InstanceInitCommand string
561 DeployRunnerBinary string
564 MaxCloudOpsPerSecond int
565 MaxProbesPerSecond int
566 MaxConcurrentInstanceCreateOps int
568 InitialQuotaEstimate int
569 SupervisorFraction float64
570 PollInterval Duration
571 ProbeInterval Duration
573 SyncInterval Duration
574 TimeoutBooting Duration
576 TimeoutProbe Duration
577 TimeoutShutdown Duration
578 TimeoutSignal Duration
579 TimeoutStaleRunLock Duration
581 ResourceTags map[string]string
585 DriverParameters json.RawMessage
588 type InstanceTypeMap map[string]InstanceType
590 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
592 // UnmarshalJSON does special handling of InstanceTypes:
594 // - populate computed fields (Name and Scratch)
596 // - error out if InstancesTypes are populated as an array, which was
597 // deprecated in Arvados 1.2.0
598 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
599 fixup := func(t InstanceType) (InstanceType, error) {
600 if t.ProviderType == "" {
601 t.ProviderType = t.Name
603 // If t.Scratch is set in the configuration file, it will be ignored and overwritten.
604 // It will also generate a "deprecated or unknown config entry" warning.
605 t.Scratch = t.IncludedScratch + t.AddedScratch
609 if len(data) > 0 && data[0] == '[' {
610 return fmt.Errorf("InstanceTypes must be specified as a map, not an array, see https://doc.arvados.org/admin/config.html")
612 var hash map[string]InstanceType
613 err := json.Unmarshal(data, &hash)
617 // Fill in Name field (and ProviderType field, if not
618 // specified) using hash key.
619 *it = InstanceTypeMap(hash)
620 for name, t := range *it {
631 type StringSet map[string]struct{}
633 // UnmarshalJSON handles old config files that provide an array of
634 // instance types instead of a hash.
635 func (ss *StringSet) UnmarshalJSON(data []byte) error {
636 if len(data) > 0 && data[0] == '[' {
638 err := json.Unmarshal(data, &arr)
646 *ss = make(map[string]struct{}, len(arr))
647 for _, t := range arr {
648 (*ss)[t] = struct{}{}
652 var hash map[string]struct{}
653 err := json.Unmarshal(data, &hash)
657 *ss = make(map[string]struct{}, len(hash))
658 for t := range hash {
659 (*ss)[t] = struct{}{}
665 type ServiceName string
668 ServiceNameController ServiceName = "arvados-controller"
669 ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
670 ServiceNameDispatchLSF ServiceName = "arvados-dispatch-lsf"
671 ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm"
672 ServiceNameGitHTTP ServiceName = "arvados-git-httpd"
673 ServiceNameHealth ServiceName = "arvados-health"
674 ServiceNameKeepbalance ServiceName = "keep-balance"
675 ServiceNameKeepproxy ServiceName = "keepproxy"
676 ServiceNameKeepstore ServiceName = "keepstore"
677 ServiceNameKeepweb ServiceName = "keep-web"
678 ServiceNameRailsAPI ServiceName = "arvados-api-server"
679 ServiceNameWebsocket ServiceName = "arvados-ws"
680 ServiceNameWorkbench1 ServiceName = "arvados-workbench1"
681 ServiceNameWorkbench2 ServiceName = "arvados-workbench2"
684 // Map returns all services as a map, suitable for iterating over all
685 // services or looking up a service by name.
686 func (svcs Services) Map() map[ServiceName]Service {
687 return map[ServiceName]Service{
688 ServiceNameController: svcs.Controller,
689 ServiceNameDispatchCloud: svcs.DispatchCloud,
690 ServiceNameDispatchLSF: svcs.DispatchLSF,
691 ServiceNameDispatchSLURM: svcs.DispatchSLURM,
692 ServiceNameGitHTTP: svcs.GitHTTP,
693 ServiceNameHealth: svcs.Health,
694 ServiceNameKeepbalance: svcs.Keepbalance,
695 ServiceNameKeepproxy: svcs.Keepproxy,
696 ServiceNameKeepstore: svcs.Keepstore,
697 ServiceNameKeepweb: svcs.WebDAV,
698 ServiceNameRailsAPI: svcs.RailsAPI,
699 ServiceNameWebsocket: svcs.Websocket,
700 ServiceNameWorkbench1: svcs.Workbench1,
701 ServiceNameWorkbench2: svcs.Workbench2,