1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: Apache-2.0
15 "git.arvados.org/arvados.git/sdk/go/config"
18 var DefaultConfigFile = func() string {
19 if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
22 return "/etc/arvados/config.yml"
26 Clusters map[string]Cluster
28 SourceTimestamp time.Time
32 // GetConfig returns the current system config, loading it from
33 // configFile if needed.
34 func GetConfig(configFile string) (*Config, error) {
36 err := config.LoadFile(&cfg, configFile)
40 // GetCluster returns the cluster ID and config for the given
41 // cluster, or the default/only configured cluster if clusterID is "".
42 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
44 if len(sc.Clusters) == 0 {
45 return nil, fmt.Errorf("no clusters configured")
46 } else if len(sc.Clusters) > 1 {
47 return nil, fmt.Errorf("multiple clusters configured, cannot choose")
49 for id, cc := range sc.Clusters {
55 cc, ok := sc.Clusters[clusterID]
57 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
59 cc.ClusterID = clusterID
63 type WebDAVCacheConfig struct {
67 MaxCollectionEntries int
68 MaxCollectionBytes int64
73 type UploadDownloadPermission struct {
78 type UploadDownloadRolePermissions struct {
79 User UploadDownloadPermission
80 Admin UploadDownloadPermission
83 type ManagedProperties map[string]struct {
90 ClusterID string `json:"-"`
91 ManagementToken string
92 SystemRootToken string
94 InstanceTypes InstanceTypeMap
95 Containers ContainersConfig
96 RemoteClusters map[string]RemoteCluster
100 AsyncPermissionsUpdateInterval Duration
101 DisabledAPIs StringSet
102 MaxIndexDatabaseRead int
103 MaxItemsPerResponse int
104 MaxConcurrentRequests int
105 MaxKeepBlobBuffers int
106 MaxRequestAmplification int
108 MaxTokenLifetime Duration
109 RequestTimeout Duration
111 WebsocketClientEventQueue int
112 WebsocketServerEventQueue int
113 KeepServiceRequestTimeout Duration
114 VocabularyPath string
115 FreezeProjectRequiresDescription bool
116 FreezeProjectRequiresProperties StringSet
117 UnfreezeProjectRequiresAdmin bool
122 UnloggedAttributes StringSet
126 BlobSigningKey string
127 BlobSigningTTL Duration
129 BlobTrashLifetime Duration
130 BlobTrashCheckInterval Duration
131 BlobTrashConcurrency int
132 BlobDeleteConcurrency int
133 BlobReplicateConcurrency int
134 CollectionVersioning bool
135 DefaultTrashLifetime Duration
136 DefaultReplication int
137 ManagedProperties ManagedProperties
138 PreserveVersionIfIdle Duration
139 TrashSweepInterval Duration
141 ForwardSlashNameSubstitution string
144 BlobMissingReport string
145 BalancePeriod Duration
146 BalanceCollectionBatch int
147 BalanceCollectionBuffers int
148 BalanceTimeout Duration
149 BalanceUpdateLimit int
151 WebDAVCache WebDAVCacheConfig
153 KeepproxyPermission UploadDownloadRolePermissions
154 WebDAVPermission UploadDownloadRolePermissions
170 SearchAttribute string
171 SearchBindUser string
172 SearchBindPassword string
175 EmailAttribute string
176 UsernameAttribute string
182 AlternateEmailAddresses bool
183 AuthenticationRequestParameters map[string]string
185 OpenIDConnect struct {
191 EmailVerifiedClaim string
193 AcceptAccessToken bool
194 AcceptAccessTokenScope string
195 AuthenticationRequestParameters map[string]string
200 DefaultEmailDomain string
204 Users map[string]TestUser
207 RemoteTokenRefresh Duration
208 TokenLifetime Duration
209 TrustedClients map[string]struct{}
210 IssueTrustedTokens bool
213 MailchimpAPIKey string
214 MailchimpListID string
215 SendUserSetupNotificationEmail bool
216 IssueReporterEmailFrom string
217 IssueReporterEmailTo string
218 SupportEmailAddress string
224 MaxRequestLogParamsSize int
232 ActivatedUsersAreVisibleToOthers bool
233 AnonymousUserToken string
234 AdminNotifierEmailFrom string
235 AutoAdminFirstUser bool
236 AutoAdminUserWithEmail string
237 AutoSetupNewUsers bool
238 AutoSetupNewUsersWithRepository bool
239 AutoSetupNewUsersWithVmUUID string
240 AutoSetupUsernameBlacklist StringSet
241 EmailSubjectPrefix string
242 NewInactiveUserNotificationRecipients StringSet
243 NewUserNotificationRecipients StringSet
244 NewUsersAreActive bool
245 UserNotifierEmailFrom string
246 UserNotifierEmailBcc StringSet
247 UserProfileNotificationAddress string
248 PreferDomainForUsername string
249 UserSetupMailText string
250 RoleGroupsVisibleToAll bool
252 StorageClasses map[string]StorageClassConfig
253 Volumes map[string]Volume
255 ActivationContactLink string
256 APIClientConnectTimeout Duration
257 APIClientReceiveTimeout Duration
258 APIResponseCompression bool
259 ApplicationMimetypesWithViewIcon StringSet
260 ArvadosDocsite string
261 ArvadosPublicDataDocURL string
262 DefaultOpenIdPrefix string
263 EnableGettingStartedPopup bool
264 EnablePublicProjectsPage bool
265 FileViewersConfigURL string
266 LogViewerMaxBytes ByteSize
267 MultiSiteSearch string
268 ProfilingEnabled bool
270 RepositoryCache string
271 RunningJobLogRecordsToFetch int
273 ShowRecentCollectionsOnDashboard bool
274 ShowUserAgreementInline bool
275 ShowUserNotifications bool
278 UserProfileFormFields map[string]struct {
280 FormFieldTitle string
281 FormFieldDescription string
284 Options map[string]struct{}
286 UserProfileFormMessage string
287 WelcomePageHTML string
288 InactivePageHTML string
289 SSHHelpPageHTML string
290 SSHHelpHostSuffix string
295 type StorageClassConfig struct {
301 AccessViaHosts map[URL]VolumeAccess
304 StorageClasses map[string]bool
306 DriverParameters json.RawMessage
309 type S3VolumeDriverParameters struct {
312 SecretAccessKey string
316 LocationConstraint bool
318 UseAWSS3v2Driver bool
320 ConnectTimeout Duration
327 type AzureVolumeDriverParameters struct {
328 StorageAccountName string
329 StorageAccountKey string
330 StorageBaseURL string
332 RequestTimeout Duration
333 ListBlobsRetryDelay Duration
334 ListBlobsMaxAttempts int
337 type DirectoryVolumeDriverParameters struct {
342 type VolumeAccess struct {
346 type Services struct {
349 DispatchCloud Service
351 DispatchSLURM Service
359 WebDAVDownload Service
367 type Service struct {
368 InternalURLs map[URL]ServiceInstance
372 type TestUser struct {
377 // URL is a url.URL that is also usable as a JSON key/value.
380 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
381 // used as a JSON key/value.
382 func (su *URL) UnmarshalText(text []byte) error {
383 u, err := url.Parse(string(text))
386 if su.Path == "" && su.Host != "" {
387 // http://example really means http://example/
394 func (su URL) MarshalText() ([]byte, error) {
395 return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
398 func (su URL) String() string {
399 return (*url.URL)(&su).String()
402 type ServiceInstance struct {
403 Rendezvous string `json:",omitempty"`
406 type PostgreSQL struct {
407 Connection PostgreSQLConnection
411 type PostgreSQLConnection map[string]string
413 type RemoteCluster struct {
421 type CUDAFeatures struct {
423 HardwareCapability string
427 type InstanceType struct {
433 IncludedScratch ByteSize
434 AddedScratch ByteSize
440 type ContainersConfig struct {
441 CloudVMs CloudVMsConfig
442 CrunchRunCommand string
443 CrunchRunArgumentsList []string
444 DefaultKeepCacheRAM ByteSize
445 DispatchPrivateKey string
446 LogReuseDecisions bool
448 MaxDispatchAttempts int
450 MinRetryPeriod Duration
451 ReserveExtraRAM ByteSize
452 StaleLockTimeout Duration
453 SupportedDockerImageFormats StringSet
454 AlwaysUsePreemptibleInstances bool
455 PreemptiblePriceFactor float64
457 LocalKeepBlobBuffersPerVCPU int
458 LocalKeepLogsToContainerLog string
462 GitInternalDir string
467 LogSecondsBetweenEvents Duration
468 LogThrottlePeriod Duration
471 LimitLogBytesPerJob int
472 LogPartialLineThrottlePeriod Duration
473 LogUpdatePeriod Duration
474 LogUpdateSize ByteSize
482 SbatchArgumentsList []string
483 SbatchEnvironmentVariables map[string]string
485 DNSServerConfDir string
486 DNSServerConfTemplate string
487 DNSServerReloadCommand string
488 DNSServerUpdateCommand string
489 ComputeNodeDomain string
490 ComputeNodeNameservers StringSet
491 AssignNodeHostname string
496 BsubArgumentsList []string
497 BsubCUDAArguments []string
501 type CloudVMsConfig struct {
504 BootProbeCommand string
505 DeployRunnerBinary string
507 MaxCloudOpsPerSecond int
508 MaxProbesPerSecond int
509 MaxConcurrentInstanceCreateOps int
510 PollInterval Duration
511 ProbeInterval Duration
513 SyncInterval Duration
514 TimeoutBooting Duration
516 TimeoutProbe Duration
517 TimeoutShutdown Duration
518 TimeoutSignal Duration
519 TimeoutStaleRunLock Duration
521 ResourceTags map[string]string
525 DriverParameters json.RawMessage
528 type InstanceTypeMap map[string]InstanceType
530 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
532 // UnmarshalJSON handles old config files that provide an array of
533 // instance types instead of a hash.
534 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
535 fixup := func(t InstanceType) (InstanceType, error) {
536 if t.ProviderType == "" {
537 t.ProviderType = t.Name
540 t.Scratch = t.IncludedScratch + t.AddedScratch
541 } else if t.AddedScratch == 0 {
542 t.AddedScratch = t.Scratch - t.IncludedScratch
543 } else if t.IncludedScratch == 0 {
544 t.IncludedScratch = t.Scratch - t.AddedScratch
547 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
548 return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
553 if len(data) > 0 && data[0] == '[' {
554 var arr []InstanceType
555 err := json.Unmarshal(data, &arr)
563 *it = make(map[string]InstanceType, len(arr))
564 for _, t := range arr {
565 if _, ok := (*it)[t.Name]; ok {
566 return errDuplicateInstanceTypeName
576 var hash map[string]InstanceType
577 err := json.Unmarshal(data, &hash)
581 // Fill in Name field (and ProviderType field, if not
582 // specified) using hash key.
583 *it = InstanceTypeMap(hash)
584 for name, t := range *it {
595 type StringSet map[string]struct{}
597 // UnmarshalJSON handles old config files that provide an array of
598 // instance types instead of a hash.
599 func (ss *StringSet) UnmarshalJSON(data []byte) error {
600 if len(data) > 0 && data[0] == '[' {
602 err := json.Unmarshal(data, &arr)
610 *ss = make(map[string]struct{}, len(arr))
611 for _, t := range arr {
612 (*ss)[t] = struct{}{}
616 var hash map[string]struct{}
617 err := json.Unmarshal(data, &hash)
621 *ss = make(map[string]struct{}, len(hash))
622 for t := range hash {
623 (*ss)[t] = struct{}{}
629 type ServiceName string
632 ServiceNameController ServiceName = "arvados-controller"
633 ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
634 ServiceNameDispatchLSF ServiceName = "arvados-dispatch-lsf"
635 ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm"
636 ServiceNameGitHTTP ServiceName = "arvados-git-httpd"
637 ServiceNameHealth ServiceName = "arvados-health"
638 ServiceNameKeepbalance ServiceName = "keep-balance"
639 ServiceNameKeepproxy ServiceName = "keepproxy"
640 ServiceNameKeepstore ServiceName = "keepstore"
641 ServiceNameKeepweb ServiceName = "keep-web"
642 ServiceNameRailsAPI ServiceName = "arvados-api-server"
643 ServiceNameWebsocket ServiceName = "arvados-ws"
644 ServiceNameWorkbench1 ServiceName = "arvados-workbench1"
645 ServiceNameWorkbench2 ServiceName = "arvados-workbench2"
648 // Map returns all services as a map, suitable for iterating over all
649 // services or looking up a service by name.
650 func (svcs Services) Map() map[ServiceName]Service {
651 return map[ServiceName]Service{
652 ServiceNameController: svcs.Controller,
653 ServiceNameDispatchCloud: svcs.DispatchCloud,
654 ServiceNameDispatchLSF: svcs.DispatchLSF,
655 ServiceNameDispatchSLURM: svcs.DispatchSLURM,
656 ServiceNameGitHTTP: svcs.GitHTTP,
657 ServiceNameHealth: svcs.Health,
658 ServiceNameKeepbalance: svcs.Keepbalance,
659 ServiceNameKeepproxy: svcs.Keepproxy,
660 ServiceNameKeepstore: svcs.Keepstore,
661 ServiceNameKeepweb: svcs.WebDAV,
662 ServiceNameRailsAPI: svcs.RailsAPI,
663 ServiceNameWebsocket: svcs.Websocket,
664 ServiceNameWorkbench1: svcs.Workbench1,
665 ServiceNameWorkbench2: svcs.Workbench2,