1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: Apache-2.0
15 "git.arvados.org/arvados.git/sdk/go/config"
18 var DefaultConfigFile = func() string {
19 if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
22 return "/etc/arvados/config.yml"
26 Clusters map[string]Cluster
28 SourceTimestamp time.Time
32 // GetConfig returns the current system config, loading it from
33 // configFile if needed.
34 func GetConfig(configFile string) (*Config, error) {
36 err := config.LoadFile(&cfg, configFile)
40 // GetCluster returns the cluster ID and config for the given
41 // cluster, or the default/only configured cluster if clusterID is "".
42 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
44 if len(sc.Clusters) == 0 {
45 return nil, fmt.Errorf("no clusters configured")
46 } else if len(sc.Clusters) > 1 {
47 return nil, fmt.Errorf("multiple clusters configured, cannot choose")
49 for id, cc := range sc.Clusters {
55 cc, ok := sc.Clusters[clusterID]
57 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
59 cc.ClusterID = clusterID
63 type WebDAVCacheConfig struct {
67 MaxCollectionEntries int
68 MaxCollectionBytes int64
73 type UploadDownloadPermission struct {
78 type UploadDownloadRolePermissions struct {
79 User UploadDownloadPermission
80 Admin UploadDownloadPermission
83 type ManagedProperties map[string]struct {
90 ClusterID string `json:"-"`
91 ManagementToken string
92 SystemRootToken string
94 InstanceTypes InstanceTypeMap
95 Containers ContainersConfig
96 RemoteClusters map[string]RemoteCluster
100 AsyncPermissionsUpdateInterval Duration
101 DisabledAPIs StringSet
102 MaxIndexDatabaseRead int
103 MaxItemsPerResponse int
104 MaxConcurrentRequests int
105 MaxKeepBlobBuffers int
106 MaxRequestAmplification int
108 MaxTokenLifetime Duration
109 RequestTimeout Duration
111 WebsocketClientEventQueue int
112 WebsocketServerEventQueue int
113 KeepServiceRequestTimeout Duration
114 VocabularyPath string
115 FreezeProjectRequiresDescription bool
116 FreezeProjectRequiresProperties StringSet
117 UnfreezeProjectRequiresAdmin bool
122 UnloggedAttributes StringSet
126 BlobSigningKey string
127 BlobSigningTTL Duration
129 BlobTrashLifetime Duration
130 BlobTrashCheckInterval Duration
131 BlobTrashConcurrency int
132 BlobDeleteConcurrency int
133 BlobReplicateConcurrency int
134 CollectionVersioning bool
135 DefaultTrashLifetime Duration
136 DefaultReplication int
137 ManagedProperties ManagedProperties
138 PreserveVersionIfIdle Duration
139 TrashSweepInterval Duration
141 ForwardSlashNameSubstitution string
144 BlobMissingReport string
145 BalancePeriod Duration
146 BalanceCollectionBatch int
147 BalanceCollectionBuffers int
148 BalanceTimeout Duration
149 BalanceUpdateLimit int
151 WebDAVCache WebDAVCacheConfig
153 KeepproxyPermission UploadDownloadRolePermissions
154 WebDAVPermission UploadDownloadRolePermissions
170 SearchAttribute string
171 SearchBindUser string
172 SearchBindPassword string
175 EmailAttribute string
176 UsernameAttribute string
182 AlternateEmailAddresses bool
183 AuthenticationRequestParameters map[string]string
185 OpenIDConnect struct {
191 EmailVerifiedClaim string
193 AcceptAccessToken bool
194 AcceptAccessTokenScope string
195 AuthenticationRequestParameters map[string]string
200 DefaultEmailDomain string
204 Users map[string]TestUser
207 RemoteTokenRefresh Duration
208 TokenLifetime Duration
209 TrustedClients map[string]struct{}
210 IssueTrustedTokens bool
213 MailchimpAPIKey string
214 MailchimpListID string
215 SendUserSetupNotificationEmail bool
216 IssueReporterEmailFrom string
217 IssueReporterEmailTo string
218 SupportEmailAddress string
224 MaxRequestLogParamsSize int
234 ActivatedUsersAreVisibleToOthers bool
235 AnonymousUserToken string
236 AdminNotifierEmailFrom string
237 AutoAdminFirstUser bool
238 AutoAdminUserWithEmail string
239 AutoSetupNewUsers bool
240 AutoSetupNewUsersWithRepository bool
241 AutoSetupNewUsersWithVmUUID string
242 AutoSetupUsernameBlacklist StringSet
243 EmailSubjectPrefix string
244 NewInactiveUserNotificationRecipients StringSet
245 NewUserNotificationRecipients StringSet
246 NewUsersAreActive bool
247 UserNotifierEmailFrom string
248 UserNotifierEmailBcc StringSet
249 UserProfileNotificationAddress string
250 PreferDomainForUsername string
251 UserSetupMailText string
252 RoleGroupsVisibleToAll bool
254 StorageClasses map[string]StorageClassConfig
255 Volumes map[string]Volume
257 ActivationContactLink string
258 APIClientConnectTimeout Duration
259 APIClientReceiveTimeout Duration
260 APIResponseCompression bool
261 ApplicationMimetypesWithViewIcon StringSet
262 ArvadosDocsite string
263 ArvadosPublicDataDocURL string
264 DefaultOpenIdPrefix string
265 DisableSharingURLsUI bool
266 EnableGettingStartedPopup bool
267 EnablePublicProjectsPage bool
268 FileViewersConfigURL string
269 LogViewerMaxBytes ByteSize
270 MultiSiteSearch string
271 ProfilingEnabled bool
273 RepositoryCache string
274 RunningJobLogRecordsToFetch int
276 ShowRecentCollectionsOnDashboard bool
277 ShowUserAgreementInline bool
278 ShowUserNotifications bool
281 UserProfileFormFields map[string]struct {
283 FormFieldTitle string
284 FormFieldDescription string
287 Options map[string]struct{}
289 UserProfileFormMessage string
290 WelcomePageHTML string
291 InactivePageHTML string
292 SSHHelpPageHTML string
293 SSHHelpHostSuffix string
298 type StorageClassConfig struct {
304 AccessViaHosts map[URL]VolumeAccess
307 StorageClasses map[string]bool
309 DriverParameters json.RawMessage
312 type S3VolumeDriverParameters struct {
315 SecretAccessKey string
319 LocationConstraint bool
321 UseAWSS3v2Driver bool
323 ConnectTimeout Duration
330 type AzureVolumeDriverParameters struct {
331 StorageAccountName string
332 StorageAccountKey string
333 StorageBaseURL string
335 RequestTimeout Duration
336 ListBlobsRetryDelay Duration
337 ListBlobsMaxAttempts int
340 type DirectoryVolumeDriverParameters struct {
345 type VolumeAccess struct {
349 type Services struct {
352 DispatchCloud Service
354 DispatchSLURM Service
362 WebDAVDownload Service
370 type Service struct {
371 InternalURLs map[URL]ServiceInstance
375 type TestUser struct {
380 // URL is a url.URL that is also usable as a JSON key/value.
383 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
384 // used as a JSON key/value.
385 func (su *URL) UnmarshalText(text []byte) error {
386 u, err := url.Parse(string(text))
389 if su.Path == "" && su.Host != "" {
390 // http://example really means http://example/
397 func (su URL) MarshalText() ([]byte, error) {
398 return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
401 func (su URL) String() string {
402 return (*url.URL)(&su).String()
405 type ServiceInstance struct {
407 Rendezvous string `json:",omitempty"`
410 type PostgreSQL struct {
411 Connection PostgreSQLConnection
415 type PostgreSQLConnection map[string]string
417 type RemoteCluster struct {
425 type CUDAFeatures struct {
427 HardwareCapability string
431 type InstanceType struct {
432 Name string `json:"-"`
436 Scratch ByteSize `json:"-"`
437 IncludedScratch ByteSize
438 AddedScratch ByteSize
444 type ContainersConfig struct {
445 CloudVMs CloudVMsConfig
446 CrunchRunCommand string
447 CrunchRunArgumentsList []string
448 DefaultKeepCacheRAM ByteSize
449 DispatchPrivateKey string
450 LogReuseDecisions bool
452 MaxDispatchAttempts int
454 MinRetryPeriod Duration
455 ReserveExtraRAM ByteSize
456 StaleLockTimeout Duration
457 SupportedDockerImageFormats StringSet
458 AlwaysUsePreemptibleInstances bool
459 PreemptiblePriceFactor float64
461 LocalKeepBlobBuffersPerVCPU int
462 LocalKeepLogsToContainerLog string
466 GitInternalDir string
471 LogSecondsBetweenEvents Duration
472 LogThrottlePeriod Duration
475 LimitLogBytesPerJob int
476 LogPartialLineThrottlePeriod Duration
477 LogUpdatePeriod Duration
478 LogUpdateSize ByteSize
486 SbatchArgumentsList []string
487 SbatchEnvironmentVariables map[string]string
489 DNSServerConfDir string
490 DNSServerConfTemplate string
491 DNSServerReloadCommand string
492 DNSServerUpdateCommand string
493 ComputeNodeDomain string
494 ComputeNodeNameservers StringSet
495 AssignNodeHostname string
500 BsubArgumentsList []string
501 BsubCUDAArguments []string
505 type CloudVMsConfig struct {
508 BootProbeCommand string
509 DeployRunnerBinary string
511 MaxCloudOpsPerSecond int
512 MaxProbesPerSecond int
513 MaxConcurrentInstanceCreateOps int
514 PollInterval Duration
515 ProbeInterval Duration
517 SyncInterval Duration
518 TimeoutBooting Duration
520 TimeoutProbe Duration
521 TimeoutShutdown Duration
522 TimeoutSignal Duration
523 TimeoutStaleRunLock Duration
525 ResourceTags map[string]string
529 DriverParameters json.RawMessage
532 type InstanceTypeMap map[string]InstanceType
534 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
536 // UnmarshalJSON does special handling of InstanceTypes:
537 // * populate computed fields (Name and Scratch)
538 // * error out if InstancesTypes are populated as an array, which was
539 // deprecated in Arvados 1.2.0
540 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
541 fixup := func(t InstanceType) (InstanceType, error) {
542 if t.ProviderType == "" {
543 t.ProviderType = t.Name
545 // If t.Scratch is set in the configuration file, it will be ignored and overwritten.
546 // It will also generate a "deprecated or unknown config entry" warning.
547 t.Scratch = t.IncludedScratch + t.AddedScratch
551 if len(data) > 0 && data[0] == '[' {
552 return fmt.Errorf("InstanceTypes must be specified as a map, not an array, see https://doc.arvados.org/admin/config.html")
554 var hash map[string]InstanceType
555 err := json.Unmarshal(data, &hash)
559 // Fill in Name field (and ProviderType field, if not
560 // specified) using hash key.
561 *it = InstanceTypeMap(hash)
562 for name, t := range *it {
573 type StringSet map[string]struct{}
575 // UnmarshalJSON handles old config files that provide an array of
576 // instance types instead of a hash.
577 func (ss *StringSet) UnmarshalJSON(data []byte) error {
578 if len(data) > 0 && data[0] == '[' {
580 err := json.Unmarshal(data, &arr)
588 *ss = make(map[string]struct{}, len(arr))
589 for _, t := range arr {
590 (*ss)[t] = struct{}{}
594 var hash map[string]struct{}
595 err := json.Unmarshal(data, &hash)
599 *ss = make(map[string]struct{}, len(hash))
600 for t := range hash {
601 (*ss)[t] = struct{}{}
607 type ServiceName string
610 ServiceNameController ServiceName = "arvados-controller"
611 ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
612 ServiceNameDispatchLSF ServiceName = "arvados-dispatch-lsf"
613 ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm"
614 ServiceNameGitHTTP ServiceName = "arvados-git-httpd"
615 ServiceNameHealth ServiceName = "arvados-health"
616 ServiceNameKeepbalance ServiceName = "keep-balance"
617 ServiceNameKeepproxy ServiceName = "keepproxy"
618 ServiceNameKeepstore ServiceName = "keepstore"
619 ServiceNameKeepweb ServiceName = "keep-web"
620 ServiceNameRailsAPI ServiceName = "arvados-api-server"
621 ServiceNameWebsocket ServiceName = "arvados-ws"
622 ServiceNameWorkbench1 ServiceName = "arvados-workbench1"
623 ServiceNameWorkbench2 ServiceName = "arvados-workbench2"
626 // Map returns all services as a map, suitable for iterating over all
627 // services or looking up a service by name.
628 func (svcs Services) Map() map[ServiceName]Service {
629 return map[ServiceName]Service{
630 ServiceNameController: svcs.Controller,
631 ServiceNameDispatchCloud: svcs.DispatchCloud,
632 ServiceNameDispatchLSF: svcs.DispatchLSF,
633 ServiceNameDispatchSLURM: svcs.DispatchSLURM,
634 ServiceNameGitHTTP: svcs.GitHTTP,
635 ServiceNameHealth: svcs.Health,
636 ServiceNameKeepbalance: svcs.Keepbalance,
637 ServiceNameKeepproxy: svcs.Keepproxy,
638 ServiceNameKeepstore: svcs.Keepstore,
639 ServiceNameKeepweb: svcs.WebDAV,
640 ServiceNameRailsAPI: svcs.RailsAPI,
641 ServiceNameWebsocket: svcs.Websocket,
642 ServiceNameWorkbench1: svcs.Workbench1,
643 ServiceNameWorkbench2: svcs.Workbench2,