1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: Apache-2.0
15 "git.arvados.org/arvados.git/sdk/go/config"
18 var DefaultConfigFile = func() string {
19 if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
22 return "/etc/arvados/config.yml"
26 Clusters map[string]Cluster
28 SourceTimestamp time.Time
32 // GetConfig returns the current system config, loading it from
33 // configFile if needed.
34 func GetConfig(configFile string) (*Config, error) {
36 err := config.LoadFile(&cfg, configFile)
40 // GetCluster returns the cluster ID and config for the given
41 // cluster, or the default/only configured cluster if clusterID is "".
42 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
44 if len(sc.Clusters) == 0 {
45 return nil, fmt.Errorf("no clusters configured")
46 } else if len(sc.Clusters) > 1 {
47 return nil, fmt.Errorf("multiple clusters configured, cannot choose")
49 for id, cc := range sc.Clusters {
55 cc, ok := sc.Clusters[clusterID]
57 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
59 cc.ClusterID = clusterID
63 type WebDAVCacheConfig struct {
66 MaxCollectionBytes int64
70 type UploadDownloadPermission struct {
75 type UploadDownloadRolePermissions struct {
76 User UploadDownloadPermission
77 Admin UploadDownloadPermission
80 type ManagedProperties map[string]struct {
87 ClusterID string `json:"-"`
88 ManagementToken string
89 SystemRootToken string
91 InstanceTypes InstanceTypeMap
92 Containers ContainersConfig
93 RemoteClusters map[string]RemoteCluster
97 AsyncPermissionsUpdateInterval Duration
98 DisabledAPIs StringSet
99 MaxIndexDatabaseRead int
100 MaxItemsPerResponse int
101 MaxConcurrentRequests int
102 MaxKeepBlobBuffers int
103 MaxRequestAmplification int
105 MaxTokenLifetime Duration
106 RequestTimeout Duration
108 WebsocketClientEventQueue int
109 WebsocketServerEventQueue int
110 KeepServiceRequestTimeout Duration
111 VocabularyPath string
112 FreezeProjectRequiresDescription bool
113 FreezeProjectRequiresProperties StringSet
114 UnfreezeProjectRequiresAdmin bool
119 UnloggedAttributes StringSet
123 BlobSigningKey string
124 BlobSigningTTL Duration
126 BlobTrashLifetime Duration
127 BlobTrashCheckInterval Duration
128 BlobTrashConcurrency int
129 BlobDeleteConcurrency int
130 BlobReplicateConcurrency int
131 CollectionVersioning bool
132 DefaultTrashLifetime Duration
133 DefaultReplication int
134 ManagedProperties ManagedProperties
135 PreserveVersionIfIdle Duration
136 TrashSweepInterval Duration
138 ForwardSlashNameSubstitution string
141 BlobMissingReport string
142 BalancePeriod Duration
143 BalanceCollectionBatch int
144 BalanceCollectionBuffers int
145 BalanceTimeout Duration
146 BalanceUpdateLimit int
148 WebDAVCache WebDAVCacheConfig
150 KeepproxyPermission UploadDownloadRolePermissions
151 WebDAVPermission UploadDownloadRolePermissions
167 SearchAttribute string
168 SearchBindUser string
169 SearchBindPassword string
172 EmailAttribute string
173 UsernameAttribute string
179 AlternateEmailAddresses bool
180 AuthenticationRequestParameters map[string]string
182 OpenIDConnect struct {
188 EmailVerifiedClaim string
190 AcceptAccessToken bool
191 AcceptAccessTokenScope string
192 AuthenticationRequestParameters map[string]string
197 DefaultEmailDomain string
201 Users map[string]TestUser
204 RemoteTokenRefresh Duration
205 TokenLifetime Duration
206 TrustedClients map[string]struct{}
207 IssueTrustedTokens bool
210 MailchimpAPIKey string
211 MailchimpListID string
212 SendUserSetupNotificationEmail bool
213 IssueReporterEmailFrom string
214 IssueReporterEmailTo string
215 SupportEmailAddress string
221 MaxRequestLogParamsSize int
232 ActivatedUsersAreVisibleToOthers bool
233 AnonymousUserToken string
234 AdminNotifierEmailFrom string
235 AutoAdminFirstUser bool
236 AutoAdminUserWithEmail string
237 AutoSetupNewUsers bool
238 AutoSetupNewUsersWithRepository bool
239 AutoSetupNewUsersWithVmUUID string
240 AutoSetupUsernameBlacklist StringSet
241 EmailSubjectPrefix string
242 NewInactiveUserNotificationRecipients StringSet
243 NewUserNotificationRecipients StringSet
244 NewUsersAreActive bool
245 UserNotifierEmailFrom string
246 UserNotifierEmailBcc StringSet
247 UserProfileNotificationAddress string
248 PreferDomainForUsername string
249 UserSetupMailText string
250 RoleGroupsVisibleToAll bool
251 ActivityLoggingPeriod Duration
253 StorageClasses map[string]StorageClassConfig
254 Volumes map[string]Volume
256 ActivationContactLink string
257 APIClientConnectTimeout Duration
258 APIClientReceiveTimeout Duration
259 APIResponseCompression bool
260 ApplicationMimetypesWithViewIcon StringSet
261 ArvadosDocsite string
262 ArvadosPublicDataDocURL string
263 DefaultOpenIdPrefix string
264 DisableSharingURLsUI bool
265 EnableGettingStartedPopup bool
266 EnablePublicProjectsPage bool
267 FileViewersConfigURL string
268 LogViewerMaxBytes ByteSize
269 MultiSiteSearch string
270 ProfilingEnabled bool
272 RepositoryCache string
273 RunningJobLogRecordsToFetch int
275 ShowRecentCollectionsOnDashboard bool
276 ShowUserAgreementInline bool
277 ShowUserNotifications bool
280 UserProfileFormFields map[string]struct {
282 FormFieldTitle string
283 FormFieldDescription string
286 Options map[string]struct{}
288 UserProfileFormMessage string
289 WelcomePageHTML string
290 InactivePageHTML string
291 SSHHelpPageHTML string
292 SSHHelpHostSuffix string
298 type StorageClassConfig struct {
304 AccessViaHosts map[URL]VolumeAccess
307 StorageClasses map[string]bool
309 DriverParameters json.RawMessage
312 type S3VolumeDriverParameters struct {
315 SecretAccessKey string
319 LocationConstraint bool
321 UseAWSS3v2Driver bool
323 ConnectTimeout Duration
330 type AzureVolumeDriverParameters struct {
331 StorageAccountName string
332 StorageAccountKey string
333 StorageBaseURL string
335 RequestTimeout Duration
336 ListBlobsRetryDelay Duration
337 ListBlobsMaxAttempts int
340 type DirectoryVolumeDriverParameters struct {
345 type VolumeAccess struct {
349 type Services struct {
352 DispatchCloud Service
354 DispatchSLURM Service
362 WebDAVDownload Service
370 type Service struct {
371 InternalURLs map[URL]ServiceInstance
375 type TestUser struct {
380 // URL is a url.URL that is also usable as a JSON key/value.
383 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
384 // used as a JSON key/value.
385 func (su *URL) UnmarshalText(text []byte) error {
386 u, err := url.Parse(string(text))
389 if su.Path == "" && su.Host != "" {
390 // http://example really means http://example/
397 func (su URL) MarshalText() ([]byte, error) {
398 return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
401 func (su URL) String() string {
402 return (*url.URL)(&su).String()
405 type ServiceInstance struct {
407 Rendezvous string `json:",omitempty"`
410 type PostgreSQL struct {
411 Connection PostgreSQLConnection
415 type PostgreSQLConnection map[string]string
417 type RemoteCluster struct {
425 type CUDAFeatures struct {
427 HardwareCapability string
431 type InstanceType struct {
432 Name string `json:"-"`
436 Scratch ByteSize `json:"-"`
437 IncludedScratch ByteSize
438 AddedScratch ByteSize
444 type ContainersConfig struct {
445 CloudVMs CloudVMsConfig
446 CrunchRunCommand string
447 CrunchRunArgumentsList []string
448 DefaultKeepCacheRAM ByteSize
449 DispatchPrivateKey string
450 LogReuseDecisions bool
452 MaxDispatchAttempts int
454 MinRetryPeriod Duration
455 ReserveExtraRAM ByteSize
456 StaleLockTimeout Duration
457 SupportedDockerImageFormats StringSet
458 AlwaysUsePreemptibleInstances bool
459 PreemptiblePriceFactor float64
461 LocalKeepBlobBuffersPerVCPU int
462 LocalKeepLogsToContainerLog string
466 GitInternalDir string
470 SweepInterval Duration
472 LogSecondsBetweenEvents Duration
473 LogThrottlePeriod Duration
476 LimitLogBytesPerJob int
477 LogPartialLineThrottlePeriod Duration
478 LogUpdatePeriod Duration
479 LogUpdateSize ByteSize
487 SbatchArgumentsList []string
488 SbatchEnvironmentVariables map[string]string
490 DNSServerConfDir string
491 DNSServerConfTemplate string
492 DNSServerReloadCommand string
493 DNSServerUpdateCommand string
494 ComputeNodeDomain string
495 ComputeNodeNameservers StringSet
496 AssignNodeHostname string
501 BsubArgumentsList []string
502 BsubCUDAArguments []string
506 type CloudVMsConfig struct {
509 BootProbeCommand string
510 DeployRunnerBinary string
512 MaxCloudOpsPerSecond int
513 MaxProbesPerSecond int
514 MaxConcurrentInstanceCreateOps int
515 PollInterval Duration
516 ProbeInterval Duration
518 SyncInterval Duration
519 TimeoutBooting Duration
521 TimeoutProbe Duration
522 TimeoutShutdown Duration
523 TimeoutSignal Duration
524 TimeoutStaleRunLock Duration
526 ResourceTags map[string]string
530 DriverParameters json.RawMessage
533 type InstanceTypeMap map[string]InstanceType
535 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
537 // UnmarshalJSON does special handling of InstanceTypes:
539 // - populate computed fields (Name and Scratch)
541 // - error out if InstancesTypes are populated as an array, which was
542 // deprecated in Arvados 1.2.0
543 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
544 fixup := func(t InstanceType) (InstanceType, error) {
545 if t.ProviderType == "" {
546 t.ProviderType = t.Name
548 // If t.Scratch is set in the configuration file, it will be ignored and overwritten.
549 // It will also generate a "deprecated or unknown config entry" warning.
550 t.Scratch = t.IncludedScratch + t.AddedScratch
554 if len(data) > 0 && data[0] == '[' {
555 return fmt.Errorf("InstanceTypes must be specified as a map, not an array, see https://doc.arvados.org/admin/config.html")
557 var hash map[string]InstanceType
558 err := json.Unmarshal(data, &hash)
562 // Fill in Name field (and ProviderType field, if not
563 // specified) using hash key.
564 *it = InstanceTypeMap(hash)
565 for name, t := range *it {
576 type StringSet map[string]struct{}
578 // UnmarshalJSON handles old config files that provide an array of
579 // instance types instead of a hash.
580 func (ss *StringSet) UnmarshalJSON(data []byte) error {
581 if len(data) > 0 && data[0] == '[' {
583 err := json.Unmarshal(data, &arr)
591 *ss = make(map[string]struct{}, len(arr))
592 for _, t := range arr {
593 (*ss)[t] = struct{}{}
597 var hash map[string]struct{}
598 err := json.Unmarshal(data, &hash)
602 *ss = make(map[string]struct{}, len(hash))
603 for t := range hash {
604 (*ss)[t] = struct{}{}
610 type ServiceName string
613 ServiceNameController ServiceName = "arvados-controller"
614 ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
615 ServiceNameDispatchLSF ServiceName = "arvados-dispatch-lsf"
616 ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm"
617 ServiceNameGitHTTP ServiceName = "arvados-git-httpd"
618 ServiceNameHealth ServiceName = "arvados-health"
619 ServiceNameKeepbalance ServiceName = "keep-balance"
620 ServiceNameKeepproxy ServiceName = "keepproxy"
621 ServiceNameKeepstore ServiceName = "keepstore"
622 ServiceNameKeepweb ServiceName = "keep-web"
623 ServiceNameRailsAPI ServiceName = "arvados-api-server"
624 ServiceNameWebsocket ServiceName = "arvados-ws"
625 ServiceNameWorkbench1 ServiceName = "arvados-workbench1"
626 ServiceNameWorkbench2 ServiceName = "arvados-workbench2"
629 // Map returns all services as a map, suitable for iterating over all
630 // services or looking up a service by name.
631 func (svcs Services) Map() map[ServiceName]Service {
632 return map[ServiceName]Service{
633 ServiceNameController: svcs.Controller,
634 ServiceNameDispatchCloud: svcs.DispatchCloud,
635 ServiceNameDispatchLSF: svcs.DispatchLSF,
636 ServiceNameDispatchSLURM: svcs.DispatchSLURM,
637 ServiceNameGitHTTP: svcs.GitHTTP,
638 ServiceNameHealth: svcs.Health,
639 ServiceNameKeepbalance: svcs.Keepbalance,
640 ServiceNameKeepproxy: svcs.Keepproxy,
641 ServiceNameKeepstore: svcs.Keepstore,
642 ServiceNameKeepweb: svcs.WebDAV,
643 ServiceNameRailsAPI: svcs.RailsAPI,
644 ServiceNameWebsocket: svcs.Websocket,
645 ServiceNameWorkbench1: svcs.Workbench1,
646 ServiceNameWorkbench2: svcs.Workbench2,