1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: Apache-2.0
14 "git.arvados.org/arvados.git/sdk/go/config"
17 var DefaultConfigFile = func() string {
18 if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
21 return "/etc/arvados/config.yml"
25 Clusters map[string]Cluster
29 // GetConfig returns the current system config, loading it from
30 // configFile if needed.
31 func GetConfig(configFile string) (*Config, error) {
33 err := config.LoadFile(&cfg, configFile)
37 // GetCluster returns the cluster ID and config for the given
38 // cluster, or the default/only configured cluster if clusterID is "".
39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
41 if len(sc.Clusters) == 0 {
42 return nil, fmt.Errorf("no clusters configured")
43 } else if len(sc.Clusters) > 1 {
44 return nil, fmt.Errorf("multiple clusters configured, cannot choose")
46 for id, cc := range sc.Clusters {
52 cc, ok := sc.Clusters[clusterID]
54 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
56 cc.ClusterID = clusterID
60 type WebDAVCacheConfig struct {
64 MaxCollectionEntries int
65 MaxCollectionBytes int64
70 type UploadDownloadPermission struct {
75 type UploadDownloadRolePermissions struct {
76 User UploadDownloadPermission
77 Admin UploadDownloadPermission
80 type ManagedProperties map[string]struct {
87 ClusterID string `json:"-"`
88 ManagementToken string
89 SystemRootToken string
91 InstanceTypes InstanceTypeMap
92 Containers ContainersConfig
93 RemoteClusters map[string]RemoteCluster
97 AsyncPermissionsUpdateInterval Duration
98 DisabledAPIs StringSet
99 MaxIndexDatabaseRead int
100 MaxItemsPerResponse int
101 MaxConcurrentRequests int
102 MaxKeepBlobBuffers int
103 MaxRequestAmplification int
105 MaxTokenLifetime Duration
106 RequestTimeout Duration
108 WebsocketClientEventQueue int
109 WebsocketServerEventQueue int
110 KeepServiceRequestTimeout Duration
111 VocabularyPath string
112 FreezeProjectRequiresDescription bool
113 FreezeProjectRequiresProperties StringSet
114 UnfreezeProjectRequiresAdmin bool
119 UnloggedAttributes StringSet
123 BlobSigningKey string
124 BlobSigningTTL Duration
126 BlobTrashLifetime Duration
127 BlobTrashCheckInterval Duration
128 BlobTrashConcurrency int
129 BlobDeleteConcurrency int
130 BlobReplicateConcurrency int
131 CollectionVersioning bool
132 DefaultTrashLifetime Duration
133 DefaultReplication int
134 ManagedProperties ManagedProperties
135 PreserveVersionIfIdle Duration
136 TrashSweepInterval Duration
138 ForwardSlashNameSubstitution string
141 BlobMissingReport string
142 BalancePeriod Duration
143 BalanceCollectionBatch int
144 BalanceCollectionBuffers int
145 BalanceTimeout Duration
146 BalanceUpdateLimit int
148 WebDAVCache WebDAVCacheConfig
150 KeepproxyPermission UploadDownloadRolePermissions
151 WebDAVPermission UploadDownloadRolePermissions
167 SearchAttribute string
168 SearchBindUser string
169 SearchBindPassword string
172 EmailAttribute string
173 UsernameAttribute string
179 AlternateEmailAddresses bool
180 AuthenticationRequestParameters map[string]string
182 OpenIDConnect struct {
188 EmailVerifiedClaim string
190 AcceptAccessToken bool
191 AcceptAccessTokenScope string
192 AuthenticationRequestParameters map[string]string
197 DefaultEmailDomain string
201 Users map[string]TestUser
204 RemoteTokenRefresh Duration
205 TokenLifetime Duration
206 TrustedClients map[string]struct{}
207 IssueTrustedTokens bool
210 MailchimpAPIKey string
211 MailchimpListID string
212 SendUserSetupNotificationEmail bool
213 IssueReporterEmailFrom string
214 IssueReporterEmailTo string
215 SupportEmailAddress string
221 MaxRequestLogParamsSize int
229 ActivatedUsersAreVisibleToOthers bool
230 AnonymousUserToken string
231 AdminNotifierEmailFrom string
232 AutoAdminFirstUser bool
233 AutoAdminUserWithEmail string
234 AutoSetupNewUsers bool
235 AutoSetupNewUsersWithRepository bool
236 AutoSetupNewUsersWithVmUUID string
237 AutoSetupUsernameBlacklist StringSet
238 EmailSubjectPrefix string
239 NewInactiveUserNotificationRecipients StringSet
240 NewUserNotificationRecipients StringSet
241 NewUsersAreActive bool
242 UserNotifierEmailFrom string
243 UserNotifierEmailBcc StringSet
244 UserProfileNotificationAddress string
245 PreferDomainForUsername string
246 UserSetupMailText string
247 RoleGroupsVisibleToAll bool
249 StorageClasses map[string]StorageClassConfig
250 Volumes map[string]Volume
252 ActivationContactLink string
253 APIClientConnectTimeout Duration
254 APIClientReceiveTimeout Duration
255 APIResponseCompression bool
256 ApplicationMimetypesWithViewIcon StringSet
257 ArvadosDocsite string
258 ArvadosPublicDataDocURL string
259 DefaultOpenIdPrefix string
260 EnableGettingStartedPopup bool
261 EnablePublicProjectsPage bool
262 FileViewersConfigURL string
263 LogViewerMaxBytes ByteSize
264 MultiSiteSearch string
265 ProfilingEnabled bool
267 RepositoryCache string
268 RunningJobLogRecordsToFetch int
270 ShowRecentCollectionsOnDashboard bool
271 ShowUserAgreementInline bool
272 ShowUserNotifications bool
275 UserProfileFormFields map[string]struct {
277 FormFieldTitle string
278 FormFieldDescription string
281 Options map[string]struct{}
283 UserProfileFormMessage string
284 WelcomePageHTML string
285 InactivePageHTML string
286 SSHHelpPageHTML string
287 SSHHelpHostSuffix string
292 type StorageClassConfig struct {
298 AccessViaHosts map[URL]VolumeAccess
301 StorageClasses map[string]bool
303 DriverParameters json.RawMessage
306 type S3VolumeDriverParameters struct {
309 SecretAccessKey string
313 LocationConstraint bool
315 UseAWSS3v2Driver bool
317 ConnectTimeout Duration
324 type AzureVolumeDriverParameters struct {
325 StorageAccountName string
326 StorageAccountKey string
327 StorageBaseURL string
329 RequestTimeout Duration
330 ListBlobsRetryDelay Duration
331 ListBlobsMaxAttempts int
334 type DirectoryVolumeDriverParameters struct {
339 type VolumeAccess struct {
343 type Services struct {
346 DispatchCloud Service
355 WebDAVDownload Service
363 type Service struct {
364 InternalURLs map[URL]ServiceInstance
368 type TestUser struct {
373 // URL is a url.URL that is also usable as a JSON key/value.
376 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
377 // used as a JSON key/value.
378 func (su *URL) UnmarshalText(text []byte) error {
379 u, err := url.Parse(string(text))
382 if su.Path == "" && su.Host != "" {
383 // http://example really means http://example/
390 func (su URL) MarshalText() ([]byte, error) {
391 return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
394 func (su URL) String() string {
395 return (*url.URL)(&su).String()
398 type ServiceInstance struct {
399 Rendezvous string `json:",omitempty"`
402 type PostgreSQL struct {
403 Connection PostgreSQLConnection
407 type PostgreSQLConnection map[string]string
409 type RemoteCluster struct {
417 type CUDAFeatures struct {
419 HardwareCapability string
423 type InstanceType struct {
424 Name string `json:"-"`
428 Scratch ByteSize `json:"-"`
429 IncludedScratch ByteSize
430 AddedScratch ByteSize
436 type ContainersConfig struct {
437 CloudVMs CloudVMsConfig
438 CrunchRunCommand string
439 CrunchRunArgumentsList []string
440 DefaultKeepCacheRAM ByteSize
441 DispatchPrivateKey string
442 LogReuseDecisions bool
444 MaxDispatchAttempts int
446 MinRetryPeriod Duration
447 ReserveExtraRAM ByteSize
448 StaleLockTimeout Duration
449 SupportedDockerImageFormats StringSet
450 AlwaysUsePreemptibleInstances bool
451 PreemptiblePriceFactor float64
453 LocalKeepBlobBuffersPerVCPU int
454 LocalKeepLogsToContainerLog string
458 GitInternalDir string
463 LogSecondsBetweenEvents Duration
464 LogThrottlePeriod Duration
467 LimitLogBytesPerJob int
468 LogPartialLineThrottlePeriod Duration
469 LogUpdatePeriod Duration
470 LogUpdateSize ByteSize
478 SbatchArgumentsList []string
479 SbatchEnvironmentVariables map[string]string
481 DNSServerConfDir string
482 DNSServerConfTemplate string
483 DNSServerReloadCommand string
484 DNSServerUpdateCommand string
485 ComputeNodeDomain string
486 ComputeNodeNameservers StringSet
487 AssignNodeHostname string
492 BsubArgumentsList []string
493 BsubCUDAArguments []string
497 type CloudVMsConfig struct {
500 BootProbeCommand string
501 DeployRunnerBinary string
503 MaxCloudOpsPerSecond int
504 MaxProbesPerSecond int
505 MaxConcurrentInstanceCreateOps int
506 PollInterval Duration
507 ProbeInterval Duration
509 SyncInterval Duration
510 TimeoutBooting Duration
512 TimeoutProbe Duration
513 TimeoutShutdown Duration
514 TimeoutSignal Duration
515 TimeoutStaleRunLock Duration
517 ResourceTags map[string]string
521 DriverParameters json.RawMessage
524 type InstanceTypeMap map[string]InstanceType
526 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
528 // UnmarshalJSON does special handling of InstanceTypes:
529 // * populate computed fields (Name and Scratch)
530 // * error out if InstancesTypes are populated as an array, which was
531 // deprecated in Arvados 1.2.0
532 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
533 fixup := func(t InstanceType) (InstanceType, error) {
534 if t.ProviderType == "" {
535 t.ProviderType = t.Name
537 // If t.Scratch is set in the configuration file, it will be ignored and overwritten.
538 // It will also generate a "deprecated or unknown config entry" warning.
539 t.Scratch = t.IncludedScratch + t.AddedScratch
543 if len(data) > 0 && data[0] == '[' {
544 return fmt.Errorf("InstanceTypes must be specified as a map, not an array, see https://doc.arvados.org/admin/config.html")
546 var hash map[string]InstanceType
547 err := json.Unmarshal(data, &hash)
551 // Fill in Name field (and ProviderType field, if not
552 // specified) using hash key.
553 *it = InstanceTypeMap(hash)
554 for name, t := range *it {
565 type StringSet map[string]struct{}
567 // UnmarshalJSON handles old config files that provide an array of
568 // instance types instead of a hash.
569 func (ss *StringSet) UnmarshalJSON(data []byte) error {
570 if len(data) > 0 && data[0] == '[' {
572 err := json.Unmarshal(data, &arr)
580 *ss = make(map[string]struct{}, len(arr))
581 for _, t := range arr {
582 (*ss)[t] = struct{}{}
586 var hash map[string]struct{}
587 err := json.Unmarshal(data, &hash)
591 *ss = make(map[string]struct{}, len(hash))
592 for t := range hash {
593 (*ss)[t] = struct{}{}
599 type ServiceName string
602 ServiceNameRailsAPI ServiceName = "arvados-api-server"
603 ServiceNameController ServiceName = "arvados-controller"
604 ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
605 ServiceNameDispatchLSF ServiceName = "arvados-dispatch-lsf"
606 ServiceNameHealth ServiceName = "arvados-health"
607 ServiceNameWorkbench1 ServiceName = "arvados-workbench1"
608 ServiceNameWorkbench2 ServiceName = "arvados-workbench2"
609 ServiceNameWebsocket ServiceName = "arvados-ws"
610 ServiceNameKeepbalance ServiceName = "keep-balance"
611 ServiceNameKeepweb ServiceName = "keep-web"
612 ServiceNameKeepproxy ServiceName = "keepproxy"
613 ServiceNameKeepstore ServiceName = "keepstore"
616 // Map returns all services as a map, suitable for iterating over all
617 // services or looking up a service by name.
618 func (svcs Services) Map() map[ServiceName]Service {
619 return map[ServiceName]Service{
620 ServiceNameRailsAPI: svcs.RailsAPI,
621 ServiceNameController: svcs.Controller,
622 ServiceNameDispatchCloud: svcs.DispatchCloud,
623 ServiceNameDispatchLSF: svcs.DispatchLSF,
624 ServiceNameHealth: svcs.Health,
625 ServiceNameWorkbench1: svcs.Workbench1,
626 ServiceNameWorkbench2: svcs.Workbench2,
627 ServiceNameWebsocket: svcs.Websocket,
628 ServiceNameKeepbalance: svcs.Keepbalance,
629 ServiceNameKeepweb: svcs.WebDAV,
630 ServiceNameKeepproxy: svcs.Keepproxy,
631 ServiceNameKeepstore: svcs.Keepstore,