1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: Apache-2.0
16 "git.arvados.org/arvados.git/sdk/go/config"
19 var DefaultConfigFile = func() string {
20 if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
23 return "/etc/arvados/config.yml"
27 Clusters map[string]Cluster
29 SourceTimestamp time.Time
33 // GetConfig returns the current system config, loading it from
34 // configFile if needed.
35 func GetConfig(configFile string) (*Config, error) {
37 err := config.LoadFile(&cfg, configFile)
41 // GetCluster returns the cluster ID and config for the given
42 // cluster, or the default/only configured cluster if clusterID is "".
43 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
45 if len(sc.Clusters) == 0 {
46 return nil, fmt.Errorf("no clusters configured")
47 } else if len(sc.Clusters) > 1 {
48 return nil, fmt.Errorf("multiple clusters configured, cannot choose")
50 for id, cc := range sc.Clusters {
56 cc, ok := sc.Clusters[clusterID]
58 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
60 cc.ClusterID = clusterID
64 type WebDAVCacheConfig struct {
67 MaxCollectionBytes int64
71 type UploadDownloadPermission struct {
76 type UploadDownloadRolePermissions struct {
77 User UploadDownloadPermission
78 Admin UploadDownloadPermission
81 type ManagedProperties map[string]struct {
88 ClusterID string `json:"-"`
89 ManagementToken string
90 SystemRootToken string
92 InstanceTypes InstanceTypeMap
93 Containers ContainersConfig
94 RemoteClusters map[string]RemoteCluster
98 AsyncPermissionsUpdateInterval Duration
99 DisabledAPIs StringSet
100 MaxIndexDatabaseRead int
101 MaxItemsPerResponse int
102 MaxConcurrentRequests int
103 LogCreateRequestFraction float64
104 MaxKeepBlobBuffers int
105 MaxRequestAmplification int
107 MaxTokenLifetime Duration
108 RequestTimeout Duration
110 WebsocketClientEventQueue int
111 WebsocketServerEventQueue int
112 KeepServiceRequestTimeout Duration
113 VocabularyPath string
114 FreezeProjectRequiresDescription bool
115 FreezeProjectRequiresProperties StringSet
116 UnfreezeProjectRequiresAdmin bool
117 LockBeforeUpdate bool
122 UnloggedAttributes StringSet
126 BlobSigningKey string
127 BlobSigningTTL Duration
129 BlobTrashLifetime Duration
130 BlobTrashCheckInterval Duration
131 BlobTrashConcurrency int
132 BlobDeleteConcurrency int
133 BlobReplicateConcurrency int
134 CollectionVersioning bool
135 DefaultTrashLifetime Duration
136 DefaultReplication int
137 ManagedProperties ManagedProperties
138 PreserveVersionIfIdle Duration
139 TrashSweepInterval Duration
141 ForwardSlashNameSubstitution string
144 BlobMissingReport string
145 BalancePeriod Duration
146 BalanceCollectionBatch int
147 BalanceCollectionBuffers int
148 BalanceTimeout Duration
149 BalanceUpdateLimit int
151 WebDAVCache WebDAVCacheConfig
153 KeepproxyPermission UploadDownloadRolePermissions
154 WebDAVPermission UploadDownloadRolePermissions
168 MinTLSVersion TLSVersion
171 SearchAttribute string
172 SearchBindUser string
173 SearchBindPassword string
176 EmailAttribute string
177 UsernameAttribute string
183 AlternateEmailAddresses bool
184 AuthenticationRequestParameters map[string]string
186 OpenIDConnect struct {
192 EmailVerifiedClaim string
194 AcceptAccessToken bool
195 AcceptAccessTokenScope string
196 AuthenticationRequestParameters map[string]string
201 DefaultEmailDomain string
205 Users map[string]TestUser
208 RemoteTokenRefresh Duration
209 TokenLifetime Duration
210 TrustedClients map[URL]struct{}
211 TrustPrivateNetworks bool
212 IssueTrustedTokens bool
215 MailchimpAPIKey string
216 MailchimpListID string
217 SendUserSetupNotificationEmail bool
218 IssueReporterEmailFrom string
219 IssueReporterEmailTo string
220 SupportEmailAddress string
226 MaxRequestLogParamsSize int
237 ActivatedUsersAreVisibleToOthers bool
238 AnonymousUserToken string
239 AdminNotifierEmailFrom string
240 AutoAdminFirstUser bool
241 AutoAdminUserWithEmail string
242 AutoSetupNewUsers bool
243 AutoSetupNewUsersWithRepository bool
244 AutoSetupNewUsersWithVmUUID string
245 AutoSetupUsernameBlacklist StringSet
246 EmailSubjectPrefix string
247 NewInactiveUserNotificationRecipients StringSet
248 NewUserNotificationRecipients StringSet
249 NewUsersAreActive bool
250 UserNotifierEmailFrom string
251 UserNotifierEmailBcc StringSet
252 UserProfileNotificationAddress string
253 PreferDomainForUsername string
254 UserSetupMailText string
255 RoleGroupsVisibleToAll bool
256 CanCreateRoleGroups bool
257 ActivityLoggingPeriod Duration
259 StorageClasses map[string]StorageClassConfig
260 Volumes map[string]Volume
262 ActivationContactLink string
263 APIClientConnectTimeout Duration
264 APIClientReceiveTimeout Duration
265 APIResponseCompression bool
266 ApplicationMimetypesWithViewIcon StringSet
267 ArvadosDocsite string
268 ArvadosPublicDataDocURL string
269 DefaultOpenIdPrefix string
270 DisableSharingURLsUI bool
271 EnableGettingStartedPopup bool
272 EnablePublicProjectsPage bool
273 FileViewersConfigURL string
274 LogViewerMaxBytes ByteSize
275 MultiSiteSearch string
276 ProfilingEnabled bool
278 RepositoryCache string
279 RunningJobLogRecordsToFetch int
281 ShowRecentCollectionsOnDashboard bool
282 ShowUserAgreementInline bool
283 ShowUserNotifications bool
286 UserProfileFormFields map[string]struct {
288 FormFieldTitle string
289 FormFieldDescription string
292 Options map[string]struct{}
294 UserProfileFormMessage string
295 WelcomePageHTML string
296 InactivePageHTML string
297 SSHHelpPageHTML string
298 SSHHelpHostSuffix string
304 type StorageClassConfig struct {
310 AccessViaHosts map[URL]VolumeAccess
313 StorageClasses map[string]bool
315 DriverParameters json.RawMessage
318 type S3VolumeDriverParameters struct {
321 SecretAccessKey string
325 LocationConstraint bool
328 ConnectTimeout Duration
335 type AzureVolumeDriverParameters struct {
336 StorageAccountName string
337 StorageAccountKey string
338 StorageBaseURL string
340 RequestTimeout Duration
341 ListBlobsRetryDelay Duration
342 ListBlobsMaxAttempts int
345 type DirectoryVolumeDriverParameters struct {
350 type VolumeAccess struct {
354 type Services struct {
357 DispatchCloud Service
359 DispatchSLURM Service
367 WebDAVDownload Service
375 type Service struct {
376 InternalURLs map[URL]ServiceInstance
380 type TestUser struct {
385 // URL is a url.URL that is also usable as a JSON key/value.
388 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
389 // used as a JSON key/value.
390 func (su *URL) UnmarshalText(text []byte) error {
391 u, err := url.Parse(string(text))
394 if su.Path == "" && su.Host != "" {
395 // http://example really means http://example/
402 func (su URL) MarshalText() ([]byte, error) {
403 return []byte(su.String()), nil
406 func (su URL) String() string {
407 return (*url.URL)(&su).String()
410 type TLSVersion uint16
412 func (v TLSVersion) MarshalText() ([]byte, error) {
416 case tls.VersionTLS10:
417 return []byte("1.0"), nil
418 case tls.VersionTLS11:
419 return []byte("1.1"), nil
420 case tls.VersionTLS12:
421 return []byte("1.2"), nil
422 case tls.VersionTLS13:
423 return []byte("1.3"), nil
425 return nil, fmt.Errorf("unsupported TLSVersion %x", v)
429 func (v *TLSVersion) UnmarshalJSON(text []byte) error {
430 if len(text) > 0 && text[0] == '"' {
432 err := json.Unmarshal(text, &s)
438 switch string(text) {
442 *v = tls.VersionTLS10
444 *v = tls.VersionTLS11
446 *v = tls.VersionTLS12
448 *v = tls.VersionTLS13
450 return fmt.Errorf("unsupported TLSVersion %q", text)
455 type ServiceInstance struct {
457 Rendezvous string `json:",omitempty"`
460 type PostgreSQL struct {
461 Connection PostgreSQLConnection
465 type PostgreSQLConnection map[string]string
467 type RemoteCluster struct {
475 type CUDAFeatures struct {
477 HardwareCapability string
481 type InstanceType struct {
482 Name string `json:"-"`
486 Scratch ByteSize `json:"-"`
487 IncludedScratch ByteSize
488 AddedScratch ByteSize
494 type ContainersConfig struct {
495 CloudVMs CloudVMsConfig
496 CrunchRunCommand string
497 CrunchRunArgumentsList []string
498 DefaultKeepCacheRAM ByteSize
499 DispatchPrivateKey string
500 LogReuseDecisions bool
501 MaxDispatchAttempts int
503 MinRetryPeriod Duration
504 ReserveExtraRAM ByteSize
505 StaleLockTimeout Duration
506 SupportedDockerImageFormats StringSet
507 AlwaysUsePreemptibleInstances bool
508 PreemptiblePriceFactor float64
510 LocalKeepBlobBuffersPerVCPU int
511 LocalKeepLogsToContainerLog string
515 GitInternalDir string
519 SweepInterval Duration
521 LogSecondsBetweenEvents Duration
522 LogThrottlePeriod Duration
525 LimitLogBytesPerJob int
526 LogPartialLineThrottlePeriod Duration
527 LogUpdatePeriod Duration
528 LogUpdateSize ByteSize
536 SbatchArgumentsList []string
537 SbatchEnvironmentVariables map[string]string
539 DNSServerConfDir string
540 DNSServerConfTemplate string
541 DNSServerReloadCommand string
542 DNSServerUpdateCommand string
543 ComputeNodeDomain string
544 ComputeNodeNameservers StringSet
545 AssignNodeHostname string
550 BsubArgumentsList []string
551 BsubCUDAArguments []string
555 type CloudVMsConfig struct {
558 BootProbeCommand string
559 DeployRunnerBinary string
561 MaxCloudOpsPerSecond int
562 MaxProbesPerSecond int
563 MaxConcurrentInstanceCreateOps int
565 SupervisorFraction float64
566 PollInterval Duration
567 ProbeInterval Duration
569 SyncInterval Duration
570 TimeoutBooting Duration
572 TimeoutProbe Duration
573 TimeoutShutdown Duration
574 TimeoutSignal Duration
575 TimeoutStaleRunLock Duration
577 ResourceTags map[string]string
581 DriverParameters json.RawMessage
584 type InstanceTypeMap map[string]InstanceType
586 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
588 // UnmarshalJSON does special handling of InstanceTypes:
590 // - populate computed fields (Name and Scratch)
592 // - error out if InstancesTypes are populated as an array, which was
593 // deprecated in Arvados 1.2.0
594 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
595 fixup := func(t InstanceType) (InstanceType, error) {
596 if t.ProviderType == "" {
597 t.ProviderType = t.Name
599 // If t.Scratch is set in the configuration file, it will be ignored and overwritten.
600 // It will also generate a "deprecated or unknown config entry" warning.
601 t.Scratch = t.IncludedScratch + t.AddedScratch
605 if len(data) > 0 && data[0] == '[' {
606 return fmt.Errorf("InstanceTypes must be specified as a map, not an array, see https://doc.arvados.org/admin/config.html")
608 var hash map[string]InstanceType
609 err := json.Unmarshal(data, &hash)
613 // Fill in Name field (and ProviderType field, if not
614 // specified) using hash key.
615 *it = InstanceTypeMap(hash)
616 for name, t := range *it {
627 type StringSet map[string]struct{}
629 // UnmarshalJSON handles old config files that provide an array of
630 // instance types instead of a hash.
631 func (ss *StringSet) UnmarshalJSON(data []byte) error {
632 if len(data) > 0 && data[0] == '[' {
634 err := json.Unmarshal(data, &arr)
642 *ss = make(map[string]struct{}, len(arr))
643 for _, t := range arr {
644 (*ss)[t] = struct{}{}
648 var hash map[string]struct{}
649 err := json.Unmarshal(data, &hash)
653 *ss = make(map[string]struct{}, len(hash))
654 for t := range hash {
655 (*ss)[t] = struct{}{}
661 type ServiceName string
664 ServiceNameController ServiceName = "arvados-controller"
665 ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
666 ServiceNameDispatchLSF ServiceName = "arvados-dispatch-lsf"
667 ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm"
668 ServiceNameGitHTTP ServiceName = "arvados-git-httpd"
669 ServiceNameHealth ServiceName = "arvados-health"
670 ServiceNameKeepbalance ServiceName = "keep-balance"
671 ServiceNameKeepproxy ServiceName = "keepproxy"
672 ServiceNameKeepstore ServiceName = "keepstore"
673 ServiceNameKeepweb ServiceName = "keep-web"
674 ServiceNameRailsAPI ServiceName = "arvados-api-server"
675 ServiceNameWebsocket ServiceName = "arvados-ws"
676 ServiceNameWorkbench1 ServiceName = "arvados-workbench1"
677 ServiceNameWorkbench2 ServiceName = "arvados-workbench2"
680 // Map returns all services as a map, suitable for iterating over all
681 // services or looking up a service by name.
682 func (svcs Services) Map() map[ServiceName]Service {
683 return map[ServiceName]Service{
684 ServiceNameController: svcs.Controller,
685 ServiceNameDispatchCloud: svcs.DispatchCloud,
686 ServiceNameDispatchLSF: svcs.DispatchLSF,
687 ServiceNameDispatchSLURM: svcs.DispatchSLURM,
688 ServiceNameGitHTTP: svcs.GitHTTP,
689 ServiceNameHealth: svcs.Health,
690 ServiceNameKeepbalance: svcs.Keepbalance,
691 ServiceNameKeepproxy: svcs.Keepproxy,
692 ServiceNameKeepstore: svcs.Keepstore,
693 ServiceNameKeepweb: svcs.WebDAV,
694 ServiceNameRailsAPI: svcs.RailsAPI,
695 ServiceNameWebsocket: svcs.Websocket,
696 ServiceNameWorkbench1: svcs.Workbench1,
697 ServiceNameWorkbench2: svcs.Workbench2,