1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: Apache-2.0
15 "git.arvados.org/arvados.git/sdk/go/config"
18 var DefaultConfigFile = func() string {
19 if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
22 return "/etc/arvados/config.yml"
26 Clusters map[string]Cluster
28 SourceTimestamp time.Time
32 // GetConfig returns the current system config, loading it from
33 // configFile if needed.
34 func GetConfig(configFile string) (*Config, error) {
36 err := config.LoadFile(&cfg, configFile)
40 // GetCluster returns the cluster ID and config for the given
41 // cluster, or the default/only configured cluster if clusterID is "".
42 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
44 if len(sc.Clusters) == 0 {
45 return nil, fmt.Errorf("no clusters configured")
46 } else if len(sc.Clusters) > 1 {
47 return nil, fmt.Errorf("multiple clusters configured, cannot choose")
49 for id, cc := range sc.Clusters {
55 cc, ok := sc.Clusters[clusterID]
57 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
59 cc.ClusterID = clusterID
63 type WebDAVCacheConfig struct {
67 MaxCollectionEntries int
68 MaxCollectionBytes int64
73 type UploadDownloadPermission struct {
78 type UploadDownloadRolePermissions struct {
79 User UploadDownloadPermission
80 Admin UploadDownloadPermission
83 type ManagedProperties map[string]struct {
90 ClusterID string `json:"-"`
91 ManagementToken string
92 SystemRootToken string
94 InstanceTypes InstanceTypeMap
95 Containers ContainersConfig
96 RemoteClusters map[string]RemoteCluster
100 AsyncPermissionsUpdateInterval Duration
101 DisabledAPIs StringSet
102 MaxIndexDatabaseRead int
103 MaxItemsPerResponse int
104 MaxConcurrentRequests int
105 MaxKeepBlobBuffers int
106 MaxRequestAmplification int
108 MaxTokenLifetime Duration
109 RequestTimeout Duration
111 WebsocketClientEventQueue int
112 WebsocketServerEventQueue int
113 KeepServiceRequestTimeout Duration
114 VocabularyPath string
115 FreezeProjectRequiresDescription bool
116 FreezeProjectRequiresProperties StringSet
117 UnfreezeProjectRequiresAdmin bool
122 UnloggedAttributes StringSet
126 BlobSigningKey string
127 BlobSigningTTL Duration
129 BlobTrashLifetime Duration
130 BlobTrashCheckInterval Duration
131 BlobTrashConcurrency int
132 BlobDeleteConcurrency int
133 BlobReplicateConcurrency int
134 CollectionVersioning bool
135 DefaultTrashLifetime Duration
136 DefaultReplication int
137 ManagedProperties ManagedProperties
138 PreserveVersionIfIdle Duration
139 TrashSweepInterval Duration
141 ForwardSlashNameSubstitution string
144 BlobMissingReport string
145 BalancePeriod Duration
146 BalanceCollectionBatch int
147 BalanceCollectionBuffers int
148 BalanceTimeout Duration
149 BalanceUpdateLimit int
151 WebDAVCache WebDAVCacheConfig
153 KeepproxyPermission UploadDownloadRolePermissions
154 WebDAVPermission UploadDownloadRolePermissions
170 SearchAttribute string
171 SearchBindUser string
172 SearchBindPassword string
175 EmailAttribute string
176 UsernameAttribute string
182 AlternateEmailAddresses bool
183 AuthenticationRequestParameters map[string]string
185 OpenIDConnect struct {
191 EmailVerifiedClaim string
193 AcceptAccessToken bool
194 AcceptAccessTokenScope string
195 AuthenticationRequestParameters map[string]string
200 DefaultEmailDomain string
204 Users map[string]TestUser
207 RemoteTokenRefresh Duration
208 TokenLifetime Duration
209 TrustedClients map[string]struct{}
210 IssueTrustedTokens bool
213 MailchimpAPIKey string
214 MailchimpListID string
215 SendUserSetupNotificationEmail bool
216 IssueReporterEmailFrom string
217 IssueReporterEmailTo string
218 SupportEmailAddress string
224 MaxRequestLogParamsSize int
232 ActivatedUsersAreVisibleToOthers bool
233 AnonymousUserToken string
234 AdminNotifierEmailFrom string
235 AutoAdminFirstUser bool
236 AutoAdminUserWithEmail string
237 AutoSetupNewUsers bool
238 AutoSetupNewUsersWithRepository bool
239 AutoSetupNewUsersWithVmUUID string
240 AutoSetupUsernameBlacklist StringSet
241 EmailSubjectPrefix string
242 NewInactiveUserNotificationRecipients StringSet
243 NewUserNotificationRecipients StringSet
244 NewUsersAreActive bool
245 UserNotifierEmailFrom string
246 UserNotifierEmailBcc StringSet
247 UserProfileNotificationAddress string
248 PreferDomainForUsername string
249 UserSetupMailText string
250 RoleGroupsVisibleToAll bool
252 StorageClasses map[string]StorageClassConfig
253 Volumes map[string]Volume
255 ActivationContactLink string
256 APIClientConnectTimeout Duration
257 APIClientReceiveTimeout Duration
258 APIResponseCompression bool
259 ApplicationMimetypesWithViewIcon StringSet
260 ArvadosDocsite string
261 ArvadosPublicDataDocURL string
262 DefaultOpenIdPrefix string
263 DisableSharingURLsUI bool
264 EnableGettingStartedPopup bool
265 EnablePublicProjectsPage bool
266 FileViewersConfigURL string
267 LogViewerMaxBytes ByteSize
268 MultiSiteSearch string
269 ProfilingEnabled bool
271 RepositoryCache string
272 RunningJobLogRecordsToFetch int
274 ShowRecentCollectionsOnDashboard bool
275 ShowUserAgreementInline bool
276 ShowUserNotifications bool
279 UserProfileFormFields map[string]struct {
281 FormFieldTitle string
282 FormFieldDescription string
285 Options map[string]struct{}
287 UserProfileFormMessage string
288 WelcomePageHTML string
289 InactivePageHTML string
290 SSHHelpPageHTML string
291 SSHHelpHostSuffix string
296 type StorageClassConfig struct {
302 AccessViaHosts map[URL]VolumeAccess
305 StorageClasses map[string]bool
307 DriverParameters json.RawMessage
310 type S3VolumeDriverParameters struct {
313 SecretAccessKey string
317 LocationConstraint bool
319 UseAWSS3v2Driver bool
321 ConnectTimeout Duration
328 type AzureVolumeDriverParameters struct {
329 StorageAccountName string
330 StorageAccountKey string
331 StorageBaseURL string
333 RequestTimeout Duration
334 ListBlobsRetryDelay Duration
335 ListBlobsMaxAttempts int
338 type DirectoryVolumeDriverParameters struct {
343 type VolumeAccess struct {
347 type Services struct {
350 DispatchCloud Service
352 DispatchSLURM Service
360 WebDAVDownload Service
368 type Service struct {
370 InternalURLs map[URL]ServiceInstance
374 type TestUser struct {
379 // URL is a url.URL that is also usable as a JSON key/value.
382 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
383 // used as a JSON key/value.
384 func (su *URL) UnmarshalText(text []byte) error {
385 u, err := url.Parse(string(text))
388 if su.Path == "" && su.Host != "" {
389 // http://example really means http://example/
396 func (su URL) MarshalText() ([]byte, error) {
397 return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
400 func (su URL) String() string {
401 return (*url.URL)(&su).String()
404 type ServiceInstance struct {
405 Rendezvous string `json:",omitempty"`
408 type PostgreSQL struct {
409 Connection PostgreSQLConnection
413 type PostgreSQLConnection map[string]string
415 type RemoteCluster struct {
423 type CUDAFeatures struct {
425 HardwareCapability string
429 type InstanceType struct {
430 Name string `json:"-"`
434 Scratch ByteSize `json:"-"`
435 IncludedScratch ByteSize
436 AddedScratch ByteSize
442 type ContainersConfig struct {
443 CloudVMs CloudVMsConfig
444 CrunchRunCommand string
445 CrunchRunArgumentsList []string
446 DefaultKeepCacheRAM ByteSize
447 DispatchPrivateKey string
448 LogReuseDecisions bool
450 MaxDispatchAttempts int
452 MinRetryPeriod Duration
453 ReserveExtraRAM ByteSize
454 StaleLockTimeout Duration
455 SupportedDockerImageFormats StringSet
456 AlwaysUsePreemptibleInstances bool
457 PreemptiblePriceFactor float64
459 LocalKeepBlobBuffersPerVCPU int
460 LocalKeepLogsToContainerLog string
464 GitInternalDir string
469 LogSecondsBetweenEvents Duration
470 LogThrottlePeriod Duration
473 LimitLogBytesPerJob int
474 LogPartialLineThrottlePeriod Duration
475 LogUpdatePeriod Duration
476 LogUpdateSize ByteSize
484 SbatchArgumentsList []string
485 SbatchEnvironmentVariables map[string]string
487 DNSServerConfDir string
488 DNSServerConfTemplate string
489 DNSServerReloadCommand string
490 DNSServerUpdateCommand string
491 ComputeNodeDomain string
492 ComputeNodeNameservers StringSet
493 AssignNodeHostname string
498 BsubArgumentsList []string
499 BsubCUDAArguments []string
503 type CloudVMsConfig struct {
506 BootProbeCommand string
507 DeployRunnerBinary string
509 MaxCloudOpsPerSecond int
510 MaxProbesPerSecond int
511 MaxConcurrentInstanceCreateOps int
512 PollInterval Duration
513 ProbeInterval Duration
515 SyncInterval Duration
516 TimeoutBooting Duration
518 TimeoutProbe Duration
519 TimeoutShutdown Duration
520 TimeoutSignal Duration
521 TimeoutStaleRunLock Duration
523 ResourceTags map[string]string
527 DriverParameters json.RawMessage
530 type InstanceTypeMap map[string]InstanceType
532 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
534 // UnmarshalJSON does special handling of InstanceTypes:
535 // * populate computed fields (Name and Scratch)
536 // * error out if InstancesTypes are populated as an array, which was
537 // deprecated in Arvados 1.2.0
538 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
539 fixup := func(t InstanceType) (InstanceType, error) {
540 if t.ProviderType == "" {
541 t.ProviderType = t.Name
543 // If t.Scratch is set in the configuration file, it will be ignored and overwritten.
544 // It will also generate a "deprecated or unknown config entry" warning.
545 t.Scratch = t.IncludedScratch + t.AddedScratch
549 if len(data) > 0 && data[0] == '[' {
550 return fmt.Errorf("InstanceTypes must be specified as a map, not an array, see https://doc.arvados.org/admin/config.html")
552 var hash map[string]InstanceType
553 err := json.Unmarshal(data, &hash)
557 // Fill in Name field (and ProviderType field, if not
558 // specified) using hash key.
559 *it = InstanceTypeMap(hash)
560 for name, t := range *it {
571 type StringSet map[string]struct{}
573 // UnmarshalJSON handles old config files that provide an array of
574 // instance types instead of a hash.
575 func (ss *StringSet) UnmarshalJSON(data []byte) error {
576 if len(data) > 0 && data[0] == '[' {
578 err := json.Unmarshal(data, &arr)
586 *ss = make(map[string]struct{}, len(arr))
587 for _, t := range arr {
588 (*ss)[t] = struct{}{}
592 var hash map[string]struct{}
593 err := json.Unmarshal(data, &hash)
597 *ss = make(map[string]struct{}, len(hash))
598 for t := range hash {
599 (*ss)[t] = struct{}{}
605 type ServiceName string
608 ServiceNameController ServiceName = "arvados-controller"
609 ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
610 ServiceNameDispatchLSF ServiceName = "arvados-dispatch-lsf"
611 ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm"
612 ServiceNameGitHTTP ServiceName = "arvados-git-httpd"
613 ServiceNameHealth ServiceName = "arvados-health"
614 ServiceNameKeepbalance ServiceName = "keep-balance"
615 ServiceNameKeepproxy ServiceName = "keepproxy"
616 ServiceNameKeepstore ServiceName = "keepstore"
617 ServiceNameKeepweb ServiceName = "keep-web"
618 ServiceNameRailsAPI ServiceName = "arvados-api-server"
619 ServiceNameWebsocket ServiceName = "arvados-ws"
620 ServiceNameWorkbench1 ServiceName = "arvados-workbench1"
621 ServiceNameWorkbench2 ServiceName = "arvados-workbench2"
624 // Map returns all services as a map, suitable for iterating over all
625 // services or looking up a service by name.
626 func (svcs Services) Map() map[ServiceName]Service {
627 return map[ServiceName]Service{
628 ServiceNameController: svcs.Controller,
629 ServiceNameDispatchCloud: svcs.DispatchCloud,
630 ServiceNameDispatchLSF: svcs.DispatchLSF,
631 ServiceNameDispatchSLURM: svcs.DispatchSLURM,
632 ServiceNameGitHTTP: svcs.GitHTTP,
633 ServiceNameHealth: svcs.Health,
634 ServiceNameKeepbalance: svcs.Keepbalance,
635 ServiceNameKeepproxy: svcs.Keepproxy,
636 ServiceNameKeepstore: svcs.Keepstore,
637 ServiceNameKeepweb: svcs.WebDAV,
638 ServiceNameRailsAPI: svcs.RailsAPI,
639 ServiceNameWebsocket: svcs.Websocket,
640 ServiceNameWorkbench1: svcs.Workbench1,
641 ServiceNameWorkbench2: svcs.Workbench2,