1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: Apache-2.0
16 "git.arvados.org/arvados.git/sdk/go/config"
19 var DefaultConfigFile = func() string {
20 if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
23 return "/etc/arvados/config.yml"
27 Clusters map[string]Cluster
29 SourceTimestamp time.Time
33 // GetConfig returns the current system config, loading it from
34 // configFile if needed.
35 func GetConfig(configFile string) (*Config, error) {
37 err := config.LoadFile(&cfg, configFile)
41 // GetCluster returns the cluster ID and config for the given
42 // cluster, or the default/only configured cluster if clusterID is "".
43 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
45 if len(sc.Clusters) == 0 {
46 return nil, fmt.Errorf("no clusters configured")
47 } else if len(sc.Clusters) > 1 {
48 return nil, fmt.Errorf("multiple clusters configured, cannot choose")
50 for id, cc := range sc.Clusters {
56 cc, ok := sc.Clusters[clusterID]
58 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
60 cc.ClusterID = clusterID
64 type WebDAVCacheConfig struct {
67 MaxCollectionBytes int64
71 type UploadDownloadPermission struct {
76 type UploadDownloadRolePermissions struct {
77 User UploadDownloadPermission
78 Admin UploadDownloadPermission
81 type ManagedProperties map[string]struct {
88 ClusterID string `json:"-"`
89 ManagementToken string
90 SystemRootToken string
92 InstanceTypes InstanceTypeMap
93 Containers ContainersConfig
94 RemoteClusters map[string]RemoteCluster
98 AsyncPermissionsUpdateInterval Duration
99 DisabledAPIs StringSet
100 MaxIndexDatabaseRead int
101 MaxItemsPerResponse int
102 MaxConcurrentRequests int
103 MaxKeepBlobBuffers int
104 MaxRequestAmplification int
106 MaxTokenLifetime Duration
107 RequestTimeout Duration
109 WebsocketClientEventQueue int
110 WebsocketServerEventQueue int
111 KeepServiceRequestTimeout Duration
112 VocabularyPath string
113 FreezeProjectRequiresDescription bool
114 FreezeProjectRequiresProperties StringSet
115 UnfreezeProjectRequiresAdmin bool
120 UnloggedAttributes StringSet
124 BlobSigningKey string
125 BlobSigningTTL Duration
127 BlobTrashLifetime Duration
128 BlobTrashCheckInterval Duration
129 BlobTrashConcurrency int
130 BlobDeleteConcurrency int
131 BlobReplicateConcurrency int
132 CollectionVersioning bool
133 DefaultTrashLifetime Duration
134 DefaultReplication int
135 ManagedProperties ManagedProperties
136 PreserveVersionIfIdle Duration
137 TrashSweepInterval Duration
139 ForwardSlashNameSubstitution string
142 BlobMissingReport string
143 BalancePeriod Duration
144 BalanceCollectionBatch int
145 BalanceCollectionBuffers int
146 BalanceTimeout Duration
147 BalanceUpdateLimit int
149 WebDAVCache WebDAVCacheConfig
151 KeepproxyPermission UploadDownloadRolePermissions
152 WebDAVPermission UploadDownloadRolePermissions
166 MinTLSVersion TLSVersion
169 SearchAttribute string
170 SearchBindUser string
171 SearchBindPassword string
174 EmailAttribute string
175 UsernameAttribute string
181 AlternateEmailAddresses bool
182 AuthenticationRequestParameters map[string]string
184 OpenIDConnect struct {
190 EmailVerifiedClaim string
192 AcceptAccessToken bool
193 AcceptAccessTokenScope string
194 AuthenticationRequestParameters map[string]string
199 DefaultEmailDomain string
203 Users map[string]TestUser
206 RemoteTokenRefresh Duration
207 TokenLifetime Duration
208 TrustedClients map[URL]struct{}
209 TrustPrivateNetworks bool
210 IssueTrustedTokens bool
213 MailchimpAPIKey string
214 MailchimpListID string
215 SendUserSetupNotificationEmail bool
216 IssueReporterEmailFrom string
217 IssueReporterEmailTo string
218 SupportEmailAddress string
224 MaxRequestLogParamsSize int
235 ActivatedUsersAreVisibleToOthers bool
236 AnonymousUserToken string
237 AdminNotifierEmailFrom string
238 AutoAdminFirstUser bool
239 AutoAdminUserWithEmail string
240 AutoSetupNewUsers bool
241 AutoSetupNewUsersWithRepository bool
242 AutoSetupNewUsersWithVmUUID string
243 AutoSetupUsernameBlacklist StringSet
244 EmailSubjectPrefix string
245 NewInactiveUserNotificationRecipients StringSet
246 NewUserNotificationRecipients StringSet
247 NewUsersAreActive bool
248 UserNotifierEmailFrom string
249 UserNotifierEmailBcc StringSet
250 UserProfileNotificationAddress string
251 PreferDomainForUsername string
252 UserSetupMailText string
253 RoleGroupsVisibleToAll bool
254 CanCreateRoleGroups bool
255 ActivityLoggingPeriod Duration
257 StorageClasses map[string]StorageClassConfig
258 Volumes map[string]Volume
260 ActivationContactLink string
261 APIClientConnectTimeout Duration
262 APIClientReceiveTimeout Duration
263 APIResponseCompression bool
264 ApplicationMimetypesWithViewIcon StringSet
265 ArvadosDocsite string
266 ArvadosPublicDataDocURL string
267 DefaultOpenIdPrefix string
268 DisableSharingURLsUI bool
269 EnableGettingStartedPopup bool
270 EnablePublicProjectsPage bool
271 FileViewersConfigURL string
272 LogViewerMaxBytes ByteSize
273 MultiSiteSearch string
274 ProfilingEnabled bool
276 RepositoryCache string
277 RunningJobLogRecordsToFetch int
279 ShowRecentCollectionsOnDashboard bool
280 ShowUserAgreementInline bool
281 ShowUserNotifications bool
284 UserProfileFormFields map[string]struct {
286 FormFieldTitle string
287 FormFieldDescription string
290 Options map[string]struct{}
292 UserProfileFormMessage string
293 WelcomePageHTML string
294 InactivePageHTML string
295 SSHHelpPageHTML string
296 SSHHelpHostSuffix string
302 type StorageClassConfig struct {
308 AccessViaHosts map[URL]VolumeAccess
311 StorageClasses map[string]bool
313 DriverParameters json.RawMessage
316 type S3VolumeDriverParameters struct {
319 SecretAccessKey string
323 LocationConstraint bool
325 UseAWSS3v2Driver bool
327 ConnectTimeout Duration
334 type AzureVolumeDriverParameters struct {
335 StorageAccountName string
336 StorageAccountKey string
337 StorageBaseURL string
339 RequestTimeout Duration
340 ListBlobsRetryDelay Duration
341 ListBlobsMaxAttempts int
344 type DirectoryVolumeDriverParameters struct {
349 type VolumeAccess struct {
353 type Services struct {
356 DispatchCloud Service
358 DispatchSLURM Service
366 WebDAVDownload Service
374 type Service struct {
375 InternalURLs map[URL]ServiceInstance
379 type TestUser struct {
384 // URL is a url.URL that is also usable as a JSON key/value.
387 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
388 // used as a JSON key/value.
389 func (su *URL) UnmarshalText(text []byte) error {
390 u, err := url.Parse(string(text))
393 if su.Path == "" && su.Host != "" {
394 // http://example really means http://example/
401 func (su URL) MarshalText() ([]byte, error) {
402 return []byte(su.String()), nil
405 func (su URL) String() string {
406 return (*url.URL)(&su).String()
409 type TLSVersion uint16
411 func (v TLSVersion) MarshalText() ([]byte, error) {
415 case tls.VersionTLS10:
416 return []byte("1.0"), nil
417 case tls.VersionTLS11:
418 return []byte("1.1"), nil
419 case tls.VersionTLS12:
420 return []byte("1.2"), nil
421 case tls.VersionTLS13:
422 return []byte("1.3"), nil
424 return nil, fmt.Errorf("unsupported TLSVersion %x", v)
428 func (v *TLSVersion) UnmarshalJSON(text []byte) error {
429 if len(text) > 0 && text[0] == '"' {
431 err := json.Unmarshal(text, &s)
437 switch string(text) {
441 *v = tls.VersionTLS10
443 *v = tls.VersionTLS11
445 *v = tls.VersionTLS12
447 *v = tls.VersionTLS13
449 return fmt.Errorf("unsupported TLSVersion %q", text)
454 type ServiceInstance struct {
456 Rendezvous string `json:",omitempty"`
459 type PostgreSQL struct {
460 Connection PostgreSQLConnection
464 type PostgreSQLConnection map[string]string
466 type RemoteCluster struct {
474 type CUDAFeatures struct {
476 HardwareCapability string
480 type InstanceType struct {
481 Name string `json:"-"`
485 Scratch ByteSize `json:"-"`
486 IncludedScratch ByteSize
487 AddedScratch ByteSize
493 type ContainersConfig struct {
494 CloudVMs CloudVMsConfig
495 CrunchRunCommand string
496 CrunchRunArgumentsList []string
497 DefaultKeepCacheRAM ByteSize
498 DispatchPrivateKey string
499 LogReuseDecisions bool
501 MaxDispatchAttempts int
503 MinRetryPeriod Duration
504 ReserveExtraRAM ByteSize
505 StaleLockTimeout Duration
506 SupportedDockerImageFormats StringSet
507 AlwaysUsePreemptibleInstances bool
508 PreemptiblePriceFactor float64
510 LocalKeepBlobBuffersPerVCPU int
511 LocalKeepLogsToContainerLog string
515 GitInternalDir string
519 SweepInterval Duration
521 LogSecondsBetweenEvents Duration
522 LogThrottlePeriod Duration
525 LimitLogBytesPerJob int
526 LogPartialLineThrottlePeriod Duration
527 LogUpdatePeriod Duration
528 LogUpdateSize ByteSize
536 SbatchArgumentsList []string
537 SbatchEnvironmentVariables map[string]string
539 DNSServerConfDir string
540 DNSServerConfTemplate string
541 DNSServerReloadCommand string
542 DNSServerUpdateCommand string
543 ComputeNodeDomain string
544 ComputeNodeNameservers StringSet
545 AssignNodeHostname string
550 BsubArgumentsList []string
551 BsubCUDAArguments []string
555 type CloudVMsConfig struct {
558 BootProbeCommand string
559 DeployRunnerBinary string
561 MaxCloudOpsPerSecond int
562 MaxProbesPerSecond int
563 MaxConcurrentInstanceCreateOps int
564 PollInterval Duration
565 ProbeInterval Duration
567 SyncInterval Duration
568 TimeoutBooting Duration
570 TimeoutProbe Duration
571 TimeoutShutdown Duration
572 TimeoutSignal Duration
573 TimeoutStaleRunLock Duration
575 ResourceTags map[string]string
579 DriverParameters json.RawMessage
582 type InstanceTypeMap map[string]InstanceType
584 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
586 // UnmarshalJSON does special handling of InstanceTypes:
588 // - populate computed fields (Name and Scratch)
590 // - error out if InstancesTypes are populated as an array, which was
591 // deprecated in Arvados 1.2.0
592 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
593 fixup := func(t InstanceType) (InstanceType, error) {
594 if t.ProviderType == "" {
595 t.ProviderType = t.Name
597 // If t.Scratch is set in the configuration file, it will be ignored and overwritten.
598 // It will also generate a "deprecated or unknown config entry" warning.
599 t.Scratch = t.IncludedScratch + t.AddedScratch
603 if len(data) > 0 && data[0] == '[' {
604 return fmt.Errorf("InstanceTypes must be specified as a map, not an array, see https://doc.arvados.org/admin/config.html")
606 var hash map[string]InstanceType
607 err := json.Unmarshal(data, &hash)
611 // Fill in Name field (and ProviderType field, if not
612 // specified) using hash key.
613 *it = InstanceTypeMap(hash)
614 for name, t := range *it {
625 type StringSet map[string]struct{}
627 // UnmarshalJSON handles old config files that provide an array of
628 // instance types instead of a hash.
629 func (ss *StringSet) UnmarshalJSON(data []byte) error {
630 if len(data) > 0 && data[0] == '[' {
632 err := json.Unmarshal(data, &arr)
640 *ss = make(map[string]struct{}, len(arr))
641 for _, t := range arr {
642 (*ss)[t] = struct{}{}
646 var hash map[string]struct{}
647 err := json.Unmarshal(data, &hash)
651 *ss = make(map[string]struct{}, len(hash))
652 for t := range hash {
653 (*ss)[t] = struct{}{}
659 type ServiceName string
662 ServiceNameController ServiceName = "arvados-controller"
663 ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
664 ServiceNameDispatchLSF ServiceName = "arvados-dispatch-lsf"
665 ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm"
666 ServiceNameGitHTTP ServiceName = "arvados-git-httpd"
667 ServiceNameHealth ServiceName = "arvados-health"
668 ServiceNameKeepbalance ServiceName = "keep-balance"
669 ServiceNameKeepproxy ServiceName = "keepproxy"
670 ServiceNameKeepstore ServiceName = "keepstore"
671 ServiceNameKeepweb ServiceName = "keep-web"
672 ServiceNameRailsAPI ServiceName = "arvados-api-server"
673 ServiceNameWebsocket ServiceName = "arvados-ws"
674 ServiceNameWorkbench1 ServiceName = "arvados-workbench1"
675 ServiceNameWorkbench2 ServiceName = "arvados-workbench2"
678 // Map returns all services as a map, suitable for iterating over all
679 // services or looking up a service by name.
680 func (svcs Services) Map() map[ServiceName]Service {
681 return map[ServiceName]Service{
682 ServiceNameController: svcs.Controller,
683 ServiceNameDispatchCloud: svcs.DispatchCloud,
684 ServiceNameDispatchLSF: svcs.DispatchLSF,
685 ServiceNameDispatchSLURM: svcs.DispatchSLURM,
686 ServiceNameGitHTTP: svcs.GitHTTP,
687 ServiceNameHealth: svcs.Health,
688 ServiceNameKeepbalance: svcs.Keepbalance,
689 ServiceNameKeepproxy: svcs.Keepproxy,
690 ServiceNameKeepstore: svcs.Keepstore,
691 ServiceNameKeepweb: svcs.WebDAV,
692 ServiceNameRailsAPI: svcs.RailsAPI,
693 ServiceNameWebsocket: svcs.Websocket,
694 ServiceNameWorkbench1: svcs.Workbench1,
695 ServiceNameWorkbench2: svcs.Workbench2,