1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: Apache-2.0
15 "git.arvados.org/arvados.git/sdk/go/config"
18 var DefaultConfigFile = func() string {
19 if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
22 return "/etc/arvados/config.yml"
26 Clusters map[string]Cluster
28 SourceTimestamp time.Time
32 // GetConfig returns the current system config, loading it from
33 // configFile if needed.
34 func GetConfig(configFile string) (*Config, error) {
36 err := config.LoadFile(&cfg, configFile)
40 // GetCluster returns the cluster ID and config for the given
41 // cluster, or the default/only configured cluster if clusterID is "".
42 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
44 if len(sc.Clusters) == 0 {
45 return nil, fmt.Errorf("no clusters configured")
46 } else if len(sc.Clusters) > 1 {
47 return nil, fmt.Errorf("multiple clusters configured, cannot choose")
49 for id, cc := range sc.Clusters {
55 cc, ok := sc.Clusters[clusterID]
57 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
59 cc.ClusterID = clusterID
63 type WebDAVCacheConfig struct {
66 MaxCollectionBytes int64
70 type UploadDownloadPermission struct {
75 type UploadDownloadRolePermissions struct {
76 User UploadDownloadPermission
77 Admin UploadDownloadPermission
80 type ManagedProperties map[string]struct {
87 ClusterID string `json:"-"`
88 ManagementToken string
89 SystemRootToken string
91 InstanceTypes InstanceTypeMap
92 Containers ContainersConfig
93 RemoteClusters map[string]RemoteCluster
97 AsyncPermissionsUpdateInterval Duration
98 DisabledAPIs StringSet
99 MaxIndexDatabaseRead int
100 MaxItemsPerResponse int
101 MaxConcurrentRequests int
102 MaxKeepBlobBuffers int
103 MaxRequestAmplification int
105 MaxTokenLifetime Duration
106 RequestTimeout Duration
108 WebsocketClientEventQueue int
109 WebsocketServerEventQueue int
110 KeepServiceRequestTimeout Duration
111 VocabularyPath string
112 FreezeProjectRequiresDescription bool
113 FreezeProjectRequiresProperties StringSet
114 UnfreezeProjectRequiresAdmin bool
119 UnloggedAttributes StringSet
123 BlobSigningKey string
124 BlobSigningTTL Duration
126 BlobTrashLifetime Duration
127 BlobTrashCheckInterval Duration
128 BlobTrashConcurrency int
129 BlobDeleteConcurrency int
130 BlobReplicateConcurrency int
131 CollectionVersioning bool
132 DefaultTrashLifetime Duration
133 DefaultReplication int
134 ManagedProperties ManagedProperties
135 PreserveVersionIfIdle Duration
136 TrashSweepInterval Duration
138 ForwardSlashNameSubstitution string
141 BlobMissingReport string
142 BalancePeriod Duration
143 BalanceCollectionBatch int
144 BalanceCollectionBuffers int
145 BalanceTimeout Duration
146 BalanceUpdateLimit int
148 WebDAVCache WebDAVCacheConfig
150 KeepproxyPermission UploadDownloadRolePermissions
151 WebDAVPermission UploadDownloadRolePermissions
167 SearchAttribute string
168 SearchBindUser string
169 SearchBindPassword string
172 EmailAttribute string
173 UsernameAttribute string
179 AlternateEmailAddresses bool
180 AuthenticationRequestParameters map[string]string
182 OpenIDConnect struct {
188 EmailVerifiedClaim string
190 AcceptAccessToken bool
191 AcceptAccessTokenScope string
192 AuthenticationRequestParameters map[string]string
197 DefaultEmailDomain string
201 Users map[string]TestUser
204 RemoteTokenRefresh Duration
205 TokenLifetime Duration
206 TrustedClients map[string]struct{}
207 IssueTrustedTokens bool
210 MailchimpAPIKey string
211 MailchimpListID string
212 SendUserSetupNotificationEmail bool
213 IssueReporterEmailFrom string
214 IssueReporterEmailTo string
215 SupportEmailAddress string
221 MaxRequestLogParamsSize int
232 ActivatedUsersAreVisibleToOthers bool
233 AnonymousUserToken string
234 AdminNotifierEmailFrom string
235 AutoAdminFirstUser bool
236 AutoAdminUserWithEmail string
237 AutoSetupNewUsers bool
238 AutoSetupNewUsersWithRepository bool
239 AutoSetupNewUsersWithVmUUID string
240 AutoSetupUsernameBlacklist StringSet
241 EmailSubjectPrefix string
242 NewInactiveUserNotificationRecipients StringSet
243 NewUserNotificationRecipients StringSet
244 NewUsersAreActive bool
245 UserNotifierEmailFrom string
246 UserNotifierEmailBcc StringSet
247 UserProfileNotificationAddress string
248 PreferDomainForUsername string
249 UserSetupMailText string
250 RoleGroupsVisibleToAll bool
251 ActivityLoggingPeriod Duration
253 StorageClasses map[string]StorageClassConfig
254 Volumes map[string]Volume
256 ActivationContactLink string
257 APIClientConnectTimeout Duration
258 APIClientReceiveTimeout Duration
259 APIResponseCompression bool
260 ApplicationMimetypesWithViewIcon StringSet
261 ArvadosDocsite string
262 ArvadosPublicDataDocURL string
263 DefaultOpenIdPrefix string
264 DisableSharingURLsUI bool
265 EnableGettingStartedPopup bool
266 EnablePublicProjectsPage bool
267 FileViewersConfigURL string
268 LogViewerMaxBytes ByteSize
269 MultiSiteSearch string
270 ProfilingEnabled bool
272 RepositoryCache string
273 RunningJobLogRecordsToFetch int
275 ShowRecentCollectionsOnDashboard bool
276 ShowUserAgreementInline bool
277 ShowUserNotifications bool
280 UserProfileFormFields map[string]struct {
282 FormFieldTitle string
283 FormFieldDescription string
286 Options map[string]struct{}
288 UserProfileFormMessage string
289 WelcomePageHTML string
290 InactivePageHTML string
291 SSHHelpPageHTML string
292 SSHHelpHostSuffix string
298 type StorageClassConfig struct {
304 AccessViaHosts map[URL]VolumeAccess
307 StorageClasses map[string]bool
309 DriverParameters json.RawMessage
312 type S3VolumeDriverParameters struct {
315 SecretAccessKey string
319 LocationConstraint bool
321 UseAWSS3v2Driver bool
323 ConnectTimeout Duration
330 type AzureVolumeDriverParameters struct {
331 StorageAccountName string
332 StorageAccountKey string
333 StorageBaseURL string
335 RequestTimeout Duration
336 ListBlobsRetryDelay Duration
337 ListBlobsMaxAttempts int
340 type DirectoryVolumeDriverParameters struct {
345 type VolumeAccess struct {
349 type Services struct {
352 DispatchCloud Service
354 DispatchSLURM Service
362 WebDAVDownload Service
370 type Service struct {
371 InternalURLs map[URL]ServiceInstance
375 type TestUser struct {
380 // URL is a url.URL that is also usable as a JSON key/value.
383 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
384 // used as a JSON key/value.
385 func (su *URL) UnmarshalText(text []byte) error {
386 u, err := url.Parse(string(text))
389 if su.Path == "" && su.Host != "" {
390 // http://example really means http://example/
397 func (su URL) MarshalText() ([]byte, error) {
398 return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
401 func (su URL) String() string {
402 return (*url.URL)(&su).String()
405 type ServiceInstance struct {
407 Rendezvous string `json:",omitempty"`
410 type PostgreSQL struct {
411 Connection PostgreSQLConnection
415 type PostgreSQLConnection map[string]string
417 type RemoteCluster struct {
425 type CUDAFeatures struct {
427 HardwareCapability string
431 type InstanceType struct {
432 Name string `json:"-"`
436 Scratch ByteSize `json:"-"`
437 IncludedScratch ByteSize
438 AddedScratch ByteSize
444 type ContainersConfig struct {
445 CloudVMs CloudVMsConfig
446 CrunchRunCommand string
447 CrunchRunArgumentsList []string
448 DefaultKeepCacheRAM ByteSize
449 DefaultKeepCacheDisk ByteSize
450 DispatchPrivateKey string
451 LogReuseDecisions bool
453 MaxDispatchAttempts int
455 MinRetryPeriod Duration
456 ReserveExtraRAM ByteSize
457 StaleLockTimeout Duration
458 SupportedDockerImageFormats StringSet
459 AlwaysUsePreemptibleInstances bool
460 PreemptiblePriceFactor float64
462 LocalKeepBlobBuffersPerVCPU int
463 LocalKeepLogsToContainerLog string
467 GitInternalDir string
472 LogSecondsBetweenEvents Duration
473 LogThrottlePeriod Duration
476 LimitLogBytesPerJob int
477 LogPartialLineThrottlePeriod Duration
478 LogUpdatePeriod Duration
479 LogUpdateSize ByteSize
487 SbatchArgumentsList []string
488 SbatchEnvironmentVariables map[string]string
490 DNSServerConfDir string
491 DNSServerConfTemplate string
492 DNSServerReloadCommand string
493 DNSServerUpdateCommand string
494 ComputeNodeDomain string
495 ComputeNodeNameservers StringSet
496 AssignNodeHostname string
501 BsubArgumentsList []string
502 BsubCUDAArguments []string
506 type CloudVMsConfig struct {
509 BootProbeCommand string
510 DeployRunnerBinary string
512 MaxCloudOpsPerSecond int
513 MaxProbesPerSecond int
514 MaxConcurrentInstanceCreateOps int
515 PollInterval Duration
516 ProbeInterval Duration
518 SyncInterval Duration
519 TimeoutBooting Duration
521 TimeoutProbe Duration
522 TimeoutShutdown Duration
523 TimeoutSignal Duration
524 TimeoutStaleRunLock Duration
526 ResourceTags map[string]string
530 DriverParameters json.RawMessage
533 type InstanceTypeMap map[string]InstanceType
535 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
537 // UnmarshalJSON does special handling of InstanceTypes:
538 // * populate computed fields (Name and Scratch)
539 // * error out if InstancesTypes are populated as an array, which was
540 // deprecated in Arvados 1.2.0
541 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
542 fixup := func(t InstanceType) (InstanceType, error) {
543 if t.ProviderType == "" {
544 t.ProviderType = t.Name
546 // If t.Scratch is set in the configuration file, it will be ignored and overwritten.
547 // It will also generate a "deprecated or unknown config entry" warning.
548 t.Scratch = t.IncludedScratch + t.AddedScratch
552 if len(data) > 0 && data[0] == '[' {
553 return fmt.Errorf("InstanceTypes must be specified as a map, not an array, see https://doc.arvados.org/admin/config.html")
555 var hash map[string]InstanceType
556 err := json.Unmarshal(data, &hash)
560 // Fill in Name field (and ProviderType field, if not
561 // specified) using hash key.
562 *it = InstanceTypeMap(hash)
563 for name, t := range *it {
574 type StringSet map[string]struct{}
576 // UnmarshalJSON handles old config files that provide an array of
577 // instance types instead of a hash.
578 func (ss *StringSet) UnmarshalJSON(data []byte) error {
579 if len(data) > 0 && data[0] == '[' {
581 err := json.Unmarshal(data, &arr)
589 *ss = make(map[string]struct{}, len(arr))
590 for _, t := range arr {
591 (*ss)[t] = struct{}{}
595 var hash map[string]struct{}
596 err := json.Unmarshal(data, &hash)
600 *ss = make(map[string]struct{}, len(hash))
601 for t := range hash {
602 (*ss)[t] = struct{}{}
608 type ServiceName string
611 ServiceNameController ServiceName = "arvados-controller"
612 ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
613 ServiceNameDispatchLSF ServiceName = "arvados-dispatch-lsf"
614 ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm"
615 ServiceNameGitHTTP ServiceName = "arvados-git-httpd"
616 ServiceNameHealth ServiceName = "arvados-health"
617 ServiceNameKeepbalance ServiceName = "keep-balance"
618 ServiceNameKeepproxy ServiceName = "keepproxy"
619 ServiceNameKeepstore ServiceName = "keepstore"
620 ServiceNameKeepweb ServiceName = "keep-web"
621 ServiceNameRailsAPI ServiceName = "arvados-api-server"
622 ServiceNameWebsocket ServiceName = "arvados-ws"
623 ServiceNameWorkbench1 ServiceName = "arvados-workbench1"
624 ServiceNameWorkbench2 ServiceName = "arvados-workbench2"
627 // Map returns all services as a map, suitable for iterating over all
628 // services or looking up a service by name.
629 func (svcs Services) Map() map[ServiceName]Service {
630 return map[ServiceName]Service{
631 ServiceNameController: svcs.Controller,
632 ServiceNameDispatchCloud: svcs.DispatchCloud,
633 ServiceNameDispatchLSF: svcs.DispatchLSF,
634 ServiceNameDispatchSLURM: svcs.DispatchSLURM,
635 ServiceNameGitHTTP: svcs.GitHTTP,
636 ServiceNameHealth: svcs.Health,
637 ServiceNameKeepbalance: svcs.Keepbalance,
638 ServiceNameKeepproxy: svcs.Keepproxy,
639 ServiceNameKeepstore: svcs.Keepstore,
640 ServiceNameKeepweb: svcs.WebDAV,
641 ServiceNameRailsAPI: svcs.RailsAPI,
642 ServiceNameWebsocket: svcs.Websocket,
643 ServiceNameWorkbench1: svcs.Workbench1,
644 ServiceNameWorkbench2: svcs.Workbench2,