1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: Apache-2.0
15 "git.arvados.org/arvados.git/sdk/go/config"
18 var DefaultConfigFile = func() string {
19 if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
22 return "/etc/arvados/config.yml"
26 Clusters map[string]Cluster
28 SourceTimestamp time.Time
32 // GetConfig returns the current system config, loading it from
33 // configFile if needed.
34 func GetConfig(configFile string) (*Config, error) {
36 err := config.LoadFile(&cfg, configFile)
40 // GetCluster returns the cluster ID and config for the given
41 // cluster, or the default/only configured cluster if clusterID is "".
42 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
44 if len(sc.Clusters) == 0 {
45 return nil, fmt.Errorf("no clusters configured")
46 } else if len(sc.Clusters) > 1 {
47 return nil, fmt.Errorf("multiple clusters configured, cannot choose")
49 for id, cc := range sc.Clusters {
55 cc, ok := sc.Clusters[clusterID]
57 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
59 cc.ClusterID = clusterID
63 type WebDAVCacheConfig struct {
66 MaxCollectionBytes int64
70 type UploadDownloadPermission struct {
75 type UploadDownloadRolePermissions struct {
76 User UploadDownloadPermission
77 Admin UploadDownloadPermission
80 type ManagedProperties map[string]struct {
87 ClusterID string `json:"-"`
88 ManagementToken string
89 SystemRootToken string
91 InstanceTypes InstanceTypeMap
92 Containers ContainersConfig
93 RemoteClusters map[string]RemoteCluster
97 AsyncPermissionsUpdateInterval Duration
98 DisabledAPIs StringSet
99 MaxIndexDatabaseRead int
100 MaxItemsPerResponse int
101 MaxConcurrentRequests int
102 MaxKeepBlobBuffers int
103 MaxRequestAmplification int
105 MaxTokenLifetime Duration
106 RequestTimeout Duration
108 WebsocketClientEventQueue int
109 WebsocketServerEventQueue int
110 KeepServiceRequestTimeout Duration
111 VocabularyPath string
112 FreezeProjectRequiresDescription bool
113 FreezeProjectRequiresProperties StringSet
114 UnfreezeProjectRequiresAdmin bool
119 UnloggedAttributes StringSet
123 BlobSigningKey string
124 BlobSigningTTL Duration
126 BlobTrashLifetime Duration
127 BlobTrashCheckInterval Duration
128 BlobTrashConcurrency int
129 BlobDeleteConcurrency int
130 BlobReplicateConcurrency int
131 CollectionVersioning bool
132 DefaultTrashLifetime Duration
133 DefaultReplication int
134 ManagedProperties ManagedProperties
135 PreserveVersionIfIdle Duration
136 TrashSweepInterval Duration
138 ForwardSlashNameSubstitution string
141 BlobMissingReport string
142 BalancePeriod Duration
143 BalanceCollectionBatch int
144 BalanceCollectionBuffers int
145 BalanceTimeout Duration
146 BalanceUpdateLimit int
148 WebDAVCache WebDAVCacheConfig
150 KeepproxyPermission UploadDownloadRolePermissions
151 WebDAVPermission UploadDownloadRolePermissions
167 SearchAttribute string
168 SearchBindUser string
169 SearchBindPassword string
172 EmailAttribute string
173 UsernameAttribute string
179 AlternateEmailAddresses bool
180 AuthenticationRequestParameters map[string]string
182 OpenIDConnect struct {
188 EmailVerifiedClaim string
190 AcceptAccessToken bool
191 AcceptAccessTokenScope string
192 AuthenticationRequestParameters map[string]string
197 DefaultEmailDomain string
201 Users map[string]TestUser
204 RemoteTokenRefresh Duration
205 TokenLifetime Duration
206 TrustedClients map[string]struct{}
207 IssueTrustedTokens bool
210 MailchimpAPIKey string
211 MailchimpListID string
212 SendUserSetupNotificationEmail bool
213 IssueReporterEmailFrom string
214 IssueReporterEmailTo string
215 SupportEmailAddress string
221 MaxRequestLogParamsSize int
232 ActivatedUsersAreVisibleToOthers bool
233 AnonymousUserToken string
234 AdminNotifierEmailFrom string
235 AutoAdminFirstUser bool
236 AutoAdminUserWithEmail string
237 AutoSetupNewUsers bool
238 AutoSetupNewUsersWithRepository bool
239 AutoSetupNewUsersWithVmUUID string
240 AutoSetupUsernameBlacklist StringSet
241 EmailSubjectPrefix string
242 NewInactiveUserNotificationRecipients StringSet
243 NewUserNotificationRecipients StringSet
244 NewUsersAreActive bool
245 UserNotifierEmailFrom string
246 UserNotifierEmailBcc StringSet
247 UserProfileNotificationAddress string
248 PreferDomainForUsername string
249 UserSetupMailText string
250 RoleGroupsVisibleToAll bool
252 StorageClasses map[string]StorageClassConfig
253 Volumes map[string]Volume
255 ActivationContactLink string
256 APIClientConnectTimeout Duration
257 APIClientReceiveTimeout Duration
258 APIResponseCompression bool
259 ApplicationMimetypesWithViewIcon StringSet
260 ArvadosDocsite string
261 ArvadosPublicDataDocURL string
262 DefaultOpenIdPrefix string
263 DisableSharingURLsUI bool
264 EnableGettingStartedPopup bool
265 EnablePublicProjectsPage bool
266 FileViewersConfigURL string
267 LogViewerMaxBytes ByteSize
268 MultiSiteSearch string
269 ProfilingEnabled bool
271 RepositoryCache string
272 RunningJobLogRecordsToFetch int
274 ShowRecentCollectionsOnDashboard bool
275 ShowUserAgreementInline bool
276 ShowUserNotifications bool
279 UserProfileFormFields map[string]struct {
281 FormFieldTitle string
282 FormFieldDescription string
285 Options map[string]struct{}
287 UserProfileFormMessage string
288 WelcomePageHTML string
289 InactivePageHTML string
290 SSHHelpPageHTML string
291 SSHHelpHostSuffix string
296 type StorageClassConfig struct {
302 AccessViaHosts map[URL]VolumeAccess
305 StorageClasses map[string]bool
307 DriverParameters json.RawMessage
310 type S3VolumeDriverParameters struct {
313 SecretAccessKey string
317 LocationConstraint bool
319 UseAWSS3v2Driver bool
321 ConnectTimeout Duration
328 type AzureVolumeDriverParameters struct {
329 StorageAccountName string
330 StorageAccountKey string
331 StorageBaseURL string
333 RequestTimeout Duration
334 ListBlobsRetryDelay Duration
335 ListBlobsMaxAttempts int
338 type DirectoryVolumeDriverParameters struct {
343 type VolumeAccess struct {
347 type Services struct {
350 DispatchCloud Service
352 DispatchSLURM Service
360 WebDAVDownload Service
368 type Service struct {
369 InternalURLs map[URL]ServiceInstance
373 type TestUser struct {
378 // URL is a url.URL that is also usable as a JSON key/value.
381 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
382 // used as a JSON key/value.
383 func (su *URL) UnmarshalText(text []byte) error {
384 u, err := url.Parse(string(text))
387 if su.Path == "" && su.Host != "" {
388 // http://example really means http://example/
395 func (su URL) MarshalText() ([]byte, error) {
396 return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
399 func (su URL) String() string {
400 return (*url.URL)(&su).String()
403 type ServiceInstance struct {
405 Rendezvous string `json:",omitempty"`
408 type PostgreSQL struct {
409 Connection PostgreSQLConnection
413 type PostgreSQLConnection map[string]string
415 type RemoteCluster struct {
423 type CUDAFeatures struct {
425 HardwareCapability string
429 type InstanceType struct {
430 Name string `json:"-"`
434 Scratch ByteSize `json:"-"`
435 IncludedScratch ByteSize
436 AddedScratch ByteSize
442 type ContainersConfig struct {
443 CloudVMs CloudVMsConfig
444 CrunchRunCommand string
445 CrunchRunArgumentsList []string
446 DefaultKeepCacheRAM ByteSize
447 DispatchPrivateKey string
448 LogReuseDecisions bool
450 MaxDispatchAttempts int
452 MinRetryPeriod Duration
453 ReserveExtraRAM ByteSize
454 StaleLockTimeout Duration
455 SupportedDockerImageFormats StringSet
456 AlwaysUsePreemptibleInstances bool
457 PreemptiblePriceFactor float64
459 LocalKeepBlobBuffersPerVCPU int
460 LocalKeepLogsToContainerLog string
464 GitInternalDir string
469 LogSecondsBetweenEvents Duration
470 LogThrottlePeriod Duration
473 LimitLogBytesPerJob int
474 LogPartialLineThrottlePeriod Duration
475 LogUpdatePeriod Duration
476 LogUpdateSize ByteSize
484 SbatchArgumentsList []string
485 SbatchEnvironmentVariables map[string]string
487 DNSServerConfDir string
488 DNSServerConfTemplate string
489 DNSServerReloadCommand string
490 DNSServerUpdateCommand string
491 ComputeNodeDomain string
492 ComputeNodeNameservers StringSet
493 AssignNodeHostname string
498 BsubArgumentsList []string
499 BsubCUDAArguments []string
503 type CloudVMsConfig struct {
506 BootProbeCommand string
507 DeployRunnerBinary string
509 MaxCloudOpsPerSecond int
510 MaxProbesPerSecond int
511 MaxConcurrentInstanceCreateOps int
512 PollInterval Duration
513 ProbeInterval Duration
515 SyncInterval Duration
516 TimeoutBooting Duration
518 TimeoutProbe Duration
519 TimeoutShutdown Duration
520 TimeoutSignal Duration
521 TimeoutStaleRunLock Duration
523 ResourceTags map[string]string
527 DriverParameters json.RawMessage
530 type InstanceTypeMap map[string]InstanceType
532 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
534 // UnmarshalJSON does special handling of InstanceTypes:
535 // * populate computed fields (Name and Scratch)
536 // * error out if InstancesTypes are populated as an array, which was
537 // deprecated in Arvados 1.2.0
538 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
539 fixup := func(t InstanceType) (InstanceType, error) {
540 if t.ProviderType == "" {
541 t.ProviderType = t.Name
543 // If t.Scratch is set in the configuration file, it will be ignored and overwritten.
544 // It will also generate a "deprecated or unknown config entry" warning.
545 t.Scratch = t.IncludedScratch + t.AddedScratch
549 if len(data) > 0 && data[0] == '[' {
550 return fmt.Errorf("InstanceTypes must be specified as a map, not an array, see https://doc.arvados.org/admin/config.html")
552 var hash map[string]InstanceType
553 err := json.Unmarshal(data, &hash)
557 // Fill in Name field (and ProviderType field, if not
558 // specified) using hash key.
559 *it = InstanceTypeMap(hash)
560 for name, t := range *it {
571 type StringSet map[string]struct{}
573 // UnmarshalJSON handles old config files that provide an array of
574 // instance types instead of a hash.
575 func (ss *StringSet) UnmarshalJSON(data []byte) error {
576 if len(data) > 0 && data[0] == '[' {
578 err := json.Unmarshal(data, &arr)
586 *ss = make(map[string]struct{}, len(arr))
587 for _, t := range arr {
588 (*ss)[t] = struct{}{}
592 var hash map[string]struct{}
593 err := json.Unmarshal(data, &hash)
597 *ss = make(map[string]struct{}, len(hash))
598 for t := range hash {
599 (*ss)[t] = struct{}{}
605 type ServiceName string
608 ServiceNameController ServiceName = "arvados-controller"
609 ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
610 ServiceNameDispatchLSF ServiceName = "arvados-dispatch-lsf"
611 ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm"
612 ServiceNameGitHTTP ServiceName = "arvados-git-httpd"
613 ServiceNameHealth ServiceName = "arvados-health"
614 ServiceNameKeepbalance ServiceName = "keep-balance"
615 ServiceNameKeepproxy ServiceName = "keepproxy"
616 ServiceNameKeepstore ServiceName = "keepstore"
617 ServiceNameKeepweb ServiceName = "keep-web"
618 ServiceNameRailsAPI ServiceName = "arvados-api-server"
619 ServiceNameWebsocket ServiceName = "arvados-ws"
620 ServiceNameWorkbench1 ServiceName = "arvados-workbench1"
621 ServiceNameWorkbench2 ServiceName = "arvados-workbench2"
624 // Map returns all services as a map, suitable for iterating over all
625 // services or looking up a service by name.
626 func (svcs Services) Map() map[ServiceName]Service {
627 return map[ServiceName]Service{
628 ServiceNameController: svcs.Controller,
629 ServiceNameDispatchCloud: svcs.DispatchCloud,
630 ServiceNameDispatchLSF: svcs.DispatchLSF,
631 ServiceNameDispatchSLURM: svcs.DispatchSLURM,
632 ServiceNameGitHTTP: svcs.GitHTTP,
633 ServiceNameHealth: svcs.Health,
634 ServiceNameKeepbalance: svcs.Keepbalance,
635 ServiceNameKeepproxy: svcs.Keepproxy,
636 ServiceNameKeepstore: svcs.Keepstore,
637 ServiceNameKeepweb: svcs.WebDAV,
638 ServiceNameRailsAPI: svcs.RailsAPI,
639 ServiceNameWebsocket: svcs.Websocket,
640 ServiceNameWorkbench1: svcs.Workbench1,
641 ServiceNameWorkbench2: svcs.Workbench2,