1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: Apache-2.0
15 "git.arvados.org/arvados.git/sdk/go/config"
18 var DefaultConfigFile = func() string {
19 if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
22 return "/etc/arvados/config.yml"
26 Clusters map[string]Cluster
28 SourceTimestamp time.Time
32 // GetConfig returns the current system config, loading it from
33 // configFile if needed.
34 func GetConfig(configFile string) (*Config, error) {
36 err := config.LoadFile(&cfg, configFile)
40 // GetCluster returns the cluster ID and config for the given
41 // cluster, or the default/only configured cluster if clusterID is "".
42 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
44 if len(sc.Clusters) == 0 {
45 return nil, fmt.Errorf("no clusters configured")
46 } else if len(sc.Clusters) > 1 {
47 return nil, fmt.Errorf("multiple clusters configured, cannot choose")
49 for id, cc := range sc.Clusters {
55 cc, ok := sc.Clusters[clusterID]
57 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
59 cc.ClusterID = clusterID
63 type WebDAVCacheConfig struct {
67 MaxCollectionEntries int
68 MaxCollectionBytes int64
73 type UploadDownloadPermission struct {
78 type UploadDownloadRolePermissions struct {
79 User UploadDownloadPermission
80 Admin UploadDownloadPermission
83 type ManagedProperties map[string]struct {
90 ClusterID string `json:"-"`
91 ManagementToken string
92 SystemRootToken string
94 InstanceTypes InstanceTypeMap
95 Containers ContainersConfig
96 RemoteClusters map[string]RemoteCluster
100 AsyncPermissionsUpdateInterval Duration
101 DisabledAPIs StringSet
102 MaxIndexDatabaseRead int
103 MaxItemsPerResponse int
104 MaxConcurrentRequests int
105 MaxKeepBlobBuffers int
106 MaxRequestAmplification int
108 MaxTokenLifetime Duration
109 RequestTimeout Duration
111 WebsocketClientEventQueue int
112 WebsocketServerEventQueue int
113 KeepServiceRequestTimeout Duration
114 VocabularyPath string
115 FreezeProjectRequiresDescription bool
116 FreezeProjectRequiresProperties StringSet
117 UnfreezeProjectRequiresAdmin bool
122 UnloggedAttributes StringSet
126 BlobSigningKey string
127 BlobSigningTTL Duration
129 BlobTrashLifetime Duration
130 BlobTrashCheckInterval Duration
131 BlobTrashConcurrency int
132 BlobDeleteConcurrency int
133 BlobReplicateConcurrency int
134 CollectionVersioning bool
135 DefaultTrashLifetime Duration
136 DefaultReplication int
137 ManagedProperties ManagedProperties
138 PreserveVersionIfIdle Duration
139 TrashSweepInterval Duration
141 ForwardSlashNameSubstitution string
144 BlobMissingReport string
145 BalancePeriod Duration
146 BalanceCollectionBatch int
147 BalanceCollectionBuffers int
148 BalanceTimeout Duration
149 BalanceUpdateLimit int
151 WebDAVCache WebDAVCacheConfig
153 KeepproxyPermission UploadDownloadRolePermissions
154 WebDAVPermission UploadDownloadRolePermissions
170 SearchAttribute string
171 SearchBindUser string
172 SearchBindPassword string
175 EmailAttribute string
176 UsernameAttribute string
182 AlternateEmailAddresses bool
183 AuthenticationRequestParameters map[string]string
185 OpenIDConnect struct {
191 EmailVerifiedClaim string
193 AcceptAccessToken bool
194 AcceptAccessTokenScope string
195 AuthenticationRequestParameters map[string]string
200 DefaultEmailDomain string
204 Users map[string]TestUser
207 RemoteTokenRefresh Duration
208 TokenLifetime Duration
209 TrustedClients map[string]struct{}
210 IssueTrustedTokens bool
213 MailchimpAPIKey string
214 MailchimpListID string
215 SendUserSetupNotificationEmail bool
216 IssueReporterEmailFrom string
217 IssueReporterEmailTo string
218 SupportEmailAddress string
224 MaxRequestLogParamsSize int
232 ActivatedUsersAreVisibleToOthers bool
233 AnonymousUserToken string
234 AdminNotifierEmailFrom string
235 AutoAdminFirstUser bool
236 AutoAdminUserWithEmail string
237 AutoSetupNewUsers bool
238 AutoSetupNewUsersWithRepository bool
239 AutoSetupNewUsersWithVmUUID string
240 AutoSetupUsernameBlacklist StringSet
241 EmailSubjectPrefix string
242 NewInactiveUserNotificationRecipients StringSet
243 NewUserNotificationRecipients StringSet
244 NewUsersAreActive bool
245 UserNotifierEmailFrom string
246 UserNotifierEmailBcc StringSet
247 UserProfileNotificationAddress string
248 PreferDomainForUsername string
249 UserSetupMailText string
250 RoleGroupsVisibleToAll bool
252 StorageClasses map[string]StorageClassConfig
253 Volumes map[string]Volume
255 ActivationContactLink string
256 APIClientConnectTimeout Duration
257 APIClientReceiveTimeout Duration
258 APIResponseCompression bool
259 ApplicationMimetypesWithViewIcon StringSet
260 ArvadosDocsite string
261 ArvadosPublicDataDocURL string
262 DefaultOpenIdPrefix string
263 EnableGettingStartedPopup bool
264 EnablePublicProjectsPage bool
265 FileViewersConfigURL string
266 LogViewerMaxBytes ByteSize
267 MultiSiteSearch string
268 ProfilingEnabled bool
270 RepositoryCache string
271 RunningJobLogRecordsToFetch int
273 ShowRecentCollectionsOnDashboard bool
274 ShowUserAgreementInline bool
275 ShowUserNotifications bool
278 UserProfileFormFields map[string]struct {
280 FormFieldTitle string
281 FormFieldDescription string
284 Options map[string]struct{}
286 UserProfileFormMessage string
287 WelcomePageHTML string
288 InactivePageHTML string
289 SSHHelpPageHTML string
290 SSHHelpHostSuffix string
295 type StorageClassConfig struct {
301 AccessViaHosts map[URL]VolumeAccess
304 StorageClasses map[string]bool
306 DriverParameters json.RawMessage
309 type S3VolumeDriverParameters struct {
312 SecretAccessKey string
316 LocationConstraint bool
318 UseAWSS3v2Driver bool
320 ConnectTimeout Duration
327 type AzureVolumeDriverParameters struct {
328 StorageAccountName string
329 StorageAccountKey string
330 StorageBaseURL string
332 RequestTimeout Duration
333 ListBlobsRetryDelay Duration
334 ListBlobsMaxAttempts int
337 type DirectoryVolumeDriverParameters struct {
342 type VolumeAccess struct {
346 type Services struct {
349 DispatchCloud Service
358 WebDAVDownload Service
366 type Service struct {
367 InternalURLs map[URL]ServiceInstance
371 type TestUser struct {
376 // URL is a url.URL that is also usable as a JSON key/value.
379 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
380 // used as a JSON key/value.
381 func (su *URL) UnmarshalText(text []byte) error {
382 u, err := url.Parse(string(text))
385 if su.Path == "" && su.Host != "" {
386 // http://example really means http://example/
393 func (su URL) MarshalText() ([]byte, error) {
394 return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
397 func (su URL) String() string {
398 return (*url.URL)(&su).String()
401 type ServiceInstance struct {
402 Rendezvous string `json:",omitempty"`
405 type PostgreSQL struct {
406 Connection PostgreSQLConnection
410 type PostgreSQLConnection map[string]string
412 type RemoteCluster struct {
420 type CUDAFeatures struct {
422 HardwareCapability string
426 type InstanceType struct {
427 Name string `json:"-"`
431 Scratch ByteSize `json:"-"`
432 IncludedScratch ByteSize
433 AddedScratch ByteSize
439 type ContainersConfig struct {
440 CloudVMs CloudVMsConfig
441 CrunchRunCommand string
442 CrunchRunArgumentsList []string
443 DefaultKeepCacheRAM ByteSize
444 DispatchPrivateKey string
445 LogReuseDecisions bool
447 MaxDispatchAttempts int
449 MinRetryPeriod Duration
450 ReserveExtraRAM ByteSize
451 StaleLockTimeout Duration
452 SupportedDockerImageFormats StringSet
453 AlwaysUsePreemptibleInstances bool
454 PreemptiblePriceFactor float64
456 LocalKeepBlobBuffersPerVCPU int
457 LocalKeepLogsToContainerLog string
461 GitInternalDir string
466 LogSecondsBetweenEvents Duration
467 LogThrottlePeriod Duration
470 LimitLogBytesPerJob int
471 LogPartialLineThrottlePeriod Duration
472 LogUpdatePeriod Duration
473 LogUpdateSize ByteSize
481 SbatchArgumentsList []string
482 SbatchEnvironmentVariables map[string]string
484 DNSServerConfDir string
485 DNSServerConfTemplate string
486 DNSServerReloadCommand string
487 DNSServerUpdateCommand string
488 ComputeNodeDomain string
489 ComputeNodeNameservers StringSet
490 AssignNodeHostname string
495 BsubArgumentsList []string
496 BsubCUDAArguments []string
500 type CloudVMsConfig struct {
503 BootProbeCommand string
504 DeployRunnerBinary string
506 MaxCloudOpsPerSecond int
507 MaxProbesPerSecond int
508 MaxConcurrentInstanceCreateOps int
509 PollInterval Duration
510 ProbeInterval Duration
512 SyncInterval Duration
513 TimeoutBooting Duration
515 TimeoutProbe Duration
516 TimeoutShutdown Duration
517 TimeoutSignal Duration
518 TimeoutStaleRunLock Duration
520 ResourceTags map[string]string
524 DriverParameters json.RawMessage
527 type InstanceTypeMap map[string]InstanceType
529 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
531 // UnmarshalJSON does special handling of InstanceTypes:
532 // * populate computed fields (Name and Scratch)
533 // * error out if InstancesTypes are populated as an array, which was
534 // deprecated in Arvados 1.2.0
535 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
536 fixup := func(t InstanceType) (InstanceType, error) {
537 if t.ProviderType == "" {
538 t.ProviderType = t.Name
540 // If t.Scratch is set in the configuration file, it will be ignored and overwritten.
541 // It will also generate a "deprecated or unknown config entry" warning.
542 t.Scratch = t.IncludedScratch + t.AddedScratch
546 if len(data) > 0 && data[0] == '[' {
547 return fmt.Errorf("InstanceTypes must be specified as a map, not an array, see https://doc.arvados.org/admin/config.html")
549 var hash map[string]InstanceType
550 err := json.Unmarshal(data, &hash)
554 // Fill in Name field (and ProviderType field, if not
555 // specified) using hash key.
556 *it = InstanceTypeMap(hash)
557 for name, t := range *it {
568 type StringSet map[string]struct{}
570 // UnmarshalJSON handles old config files that provide an array of
571 // instance types instead of a hash.
572 func (ss *StringSet) UnmarshalJSON(data []byte) error {
573 if len(data) > 0 && data[0] == '[' {
575 err := json.Unmarshal(data, &arr)
583 *ss = make(map[string]struct{}, len(arr))
584 for _, t := range arr {
585 (*ss)[t] = struct{}{}
589 var hash map[string]struct{}
590 err := json.Unmarshal(data, &hash)
594 *ss = make(map[string]struct{}, len(hash))
595 for t := range hash {
596 (*ss)[t] = struct{}{}
602 type ServiceName string
605 ServiceNameController ServiceName = "arvados-controller"
606 ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
607 ServiceNameDispatchLSF ServiceName = "arvados-dispatch-lsf"
608 ServiceNameGitHTTP ServiceName = "arvados-git-httpd"
609 ServiceNameHealth ServiceName = "arvados-health"
610 ServiceNameKeepbalance ServiceName = "keep-balance"
611 ServiceNameKeepproxy ServiceName = "keepproxy"
612 ServiceNameKeepstore ServiceName = "keepstore"
613 ServiceNameKeepweb ServiceName = "keep-web"
614 ServiceNameRailsAPI ServiceName = "arvados-api-server"
615 ServiceNameWebsocket ServiceName = "arvados-ws"
616 ServiceNameWorkbench1 ServiceName = "arvados-workbench1"
617 ServiceNameWorkbench2 ServiceName = "arvados-workbench2"
620 // Map returns all services as a map, suitable for iterating over all
621 // services or looking up a service by name.
622 func (svcs Services) Map() map[ServiceName]Service {
623 return map[ServiceName]Service{
624 ServiceNameController: svcs.Controller,
625 ServiceNameDispatchCloud: svcs.DispatchCloud,
626 ServiceNameDispatchLSF: svcs.DispatchLSF,
627 ServiceNameGitHTTP: svcs.GitHTTP,
628 ServiceNameHealth: svcs.Health,
629 ServiceNameKeepbalance: svcs.Keepbalance,
630 ServiceNameKeepproxy: svcs.Keepproxy,
631 ServiceNameKeepstore: svcs.Keepstore,
632 ServiceNameKeepweb: svcs.WebDAV,
633 ServiceNameRailsAPI: svcs.RailsAPI,
634 ServiceNameWebsocket: svcs.Websocket,
635 ServiceNameWorkbench1: svcs.Workbench1,
636 ServiceNameWorkbench2: svcs.Workbench2,