1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: Apache-2.0
15 "git.arvados.org/arvados.git/sdk/go/config"
18 var DefaultConfigFile = func() string {
19 if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
22 return "/etc/arvados/config.yml"
26 Clusters map[string]Cluster
28 SourceTimestamp time.Time
32 // GetConfig returns the current system config, loading it from
33 // configFile if needed.
34 func GetConfig(configFile string) (*Config, error) {
36 err := config.LoadFile(&cfg, configFile)
40 // GetCluster returns the cluster ID and config for the given
41 // cluster, or the default/only configured cluster if clusterID is "".
42 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
44 if len(sc.Clusters) == 0 {
45 return nil, fmt.Errorf("no clusters configured")
46 } else if len(sc.Clusters) > 1 {
47 return nil, fmt.Errorf("multiple clusters configured, cannot choose")
49 for id, cc := range sc.Clusters {
55 cc, ok := sc.Clusters[clusterID]
57 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
59 cc.ClusterID = clusterID
63 type WebDAVCacheConfig struct {
67 MaxCollectionEntries int
68 MaxCollectionBytes int64
73 type UploadDownloadPermission struct {
78 type UploadDownloadRolePermissions struct {
79 User UploadDownloadPermission
80 Admin UploadDownloadPermission
83 type ManagedProperties map[string]struct {
90 ClusterID string `json:"-"`
91 ManagementToken string
92 SystemRootToken string
94 InstanceTypes InstanceTypeMap
95 Containers ContainersConfig
96 RemoteClusters map[string]RemoteCluster
100 AsyncPermissionsUpdateInterval Duration
101 DisabledAPIs StringSet
102 MaxIndexDatabaseRead int
103 MaxItemsPerResponse int
104 MaxConcurrentRequests int
105 MaxKeepBlobBuffers int
106 MaxRequestAmplification int
108 MaxTokenLifetime Duration
109 RequestTimeout Duration
111 WebsocketClientEventQueue int
112 WebsocketServerEventQueue int
113 KeepServiceRequestTimeout Duration
114 VocabularyPath string
115 FreezeProjectRequiresDescription bool
116 FreezeProjectRequiresProperties StringSet
117 UnfreezeProjectRequiresAdmin bool
122 UnloggedAttributes StringSet
126 BlobSigningKey string
127 BlobSigningTTL Duration
129 BlobTrashLifetime Duration
130 BlobTrashCheckInterval Duration
131 BlobTrashConcurrency int
132 BlobDeleteConcurrency int
133 BlobReplicateConcurrency int
134 CollectionVersioning bool
135 DefaultTrashLifetime Duration
136 DefaultReplication int
137 ManagedProperties ManagedProperties
138 PreserveVersionIfIdle Duration
139 TrashSweepInterval Duration
141 ForwardSlashNameSubstitution string
144 BlobMissingReport string
145 BalancePeriod Duration
146 BalanceCollectionBatch int
147 BalanceCollectionBuffers int
148 BalanceTimeout Duration
149 BalanceUpdateLimit int
151 WebDAVCache WebDAVCacheConfig
153 KeepproxyPermission UploadDownloadRolePermissions
154 WebDAVPermission UploadDownloadRolePermissions
170 SearchAttribute string
171 SearchBindUser string
172 SearchBindPassword string
175 EmailAttribute string
176 UsernameAttribute string
182 AlternateEmailAddresses bool
183 AuthenticationRequestParameters map[string]string
185 OpenIDConnect struct {
191 EmailVerifiedClaim string
193 AcceptAccessToken bool
194 AcceptAccessTokenScope string
195 AuthenticationRequestParameters map[string]string
200 DefaultEmailDomain string
204 Users map[string]TestUser
207 RemoteTokenRefresh Duration
208 TokenLifetime Duration
209 TrustedClients map[string]struct{}
210 IssueTrustedTokens bool
213 MailchimpAPIKey string
214 MailchimpListID string
215 SendUserSetupNotificationEmail bool
216 IssueReporterEmailFrom string
217 IssueReporterEmailTo string
218 SupportEmailAddress string
224 MaxRequestLogParamsSize int
235 ActivatedUsersAreVisibleToOthers bool
236 AnonymousUserToken string
237 AdminNotifierEmailFrom string
238 AutoAdminFirstUser bool
239 AutoAdminUserWithEmail string
240 AutoSetupNewUsers bool
241 AutoSetupNewUsersWithRepository bool
242 AutoSetupNewUsersWithVmUUID string
243 AutoSetupUsernameBlacklist StringSet
244 EmailSubjectPrefix string
245 NewInactiveUserNotificationRecipients StringSet
246 NewUserNotificationRecipients StringSet
247 NewUsersAreActive bool
248 UserNotifierEmailFrom string
249 UserNotifierEmailBcc StringSet
250 UserProfileNotificationAddress string
251 PreferDomainForUsername string
252 UserSetupMailText string
253 RoleGroupsVisibleToAll bool
255 StorageClasses map[string]StorageClassConfig
256 Volumes map[string]Volume
258 ActivationContactLink string
259 APIClientConnectTimeout Duration
260 APIClientReceiveTimeout Duration
261 APIResponseCompression bool
262 ApplicationMimetypesWithViewIcon StringSet
263 ArvadosDocsite string
264 ArvadosPublicDataDocURL string
265 DefaultOpenIdPrefix string
266 DisableSharingURLsUI bool
267 EnableGettingStartedPopup bool
268 EnablePublicProjectsPage bool
269 FileViewersConfigURL string
270 LogViewerMaxBytes ByteSize
271 MultiSiteSearch string
272 ProfilingEnabled bool
274 RepositoryCache string
275 RunningJobLogRecordsToFetch int
277 ShowRecentCollectionsOnDashboard bool
278 ShowUserAgreementInline bool
279 ShowUserNotifications bool
282 UserProfileFormFields map[string]struct {
284 FormFieldTitle string
285 FormFieldDescription string
288 Options map[string]struct{}
290 UserProfileFormMessage string
291 WelcomePageHTML string
292 InactivePageHTML string
293 SSHHelpPageHTML string
294 SSHHelpHostSuffix string
299 type StorageClassConfig struct {
305 AccessViaHosts map[URL]VolumeAccess
308 StorageClasses map[string]bool
310 DriverParameters json.RawMessage
313 type S3VolumeDriverParameters struct {
316 SecretAccessKey string
320 LocationConstraint bool
322 UseAWSS3v2Driver bool
324 ConnectTimeout Duration
331 type AzureVolumeDriverParameters struct {
332 StorageAccountName string
333 StorageAccountKey string
334 StorageBaseURL string
336 RequestTimeout Duration
337 ListBlobsRetryDelay Duration
338 ListBlobsMaxAttempts int
341 type DirectoryVolumeDriverParameters struct {
346 type VolumeAccess struct {
350 type Services struct {
353 DispatchCloud Service
355 DispatchSLURM Service
363 WebDAVDownload Service
371 type Service struct {
372 InternalURLs map[URL]ServiceInstance
376 type TestUser struct {
381 // URL is a url.URL that is also usable as a JSON key/value.
384 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
385 // used as a JSON key/value.
386 func (su *URL) UnmarshalText(text []byte) error {
387 u, err := url.Parse(string(text))
390 if su.Path == "" && su.Host != "" {
391 // http://example really means http://example/
398 func (su URL) MarshalText() ([]byte, error) {
399 return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
402 func (su URL) String() string {
403 return (*url.URL)(&su).String()
406 type ServiceInstance struct {
408 Rendezvous string `json:",omitempty"`
411 type PostgreSQL struct {
412 Connection PostgreSQLConnection
416 type PostgreSQLConnection map[string]string
418 type RemoteCluster struct {
426 type CUDAFeatures struct {
428 HardwareCapability string
432 type InstanceType struct {
433 Name string `json:"-"`
437 Scratch ByteSize `json:"-"`
438 IncludedScratch ByteSize
439 AddedScratch ByteSize
445 type ContainersConfig struct {
446 CloudVMs CloudVMsConfig
447 CrunchRunCommand string
448 CrunchRunArgumentsList []string
449 DefaultKeepCacheRAM ByteSize
450 DispatchPrivateKey string
451 LogReuseDecisions bool
453 MaxDispatchAttempts int
455 MinRetryPeriod Duration
456 ReserveExtraRAM ByteSize
457 StaleLockTimeout Duration
458 SupportedDockerImageFormats StringSet
459 AlwaysUsePreemptibleInstances bool
460 PreemptiblePriceFactor float64
462 LocalKeepBlobBuffersPerVCPU int
463 LocalKeepLogsToContainerLog string
467 GitInternalDir string
472 LogSecondsBetweenEvents Duration
473 LogThrottlePeriod Duration
476 LimitLogBytesPerJob int
477 LogPartialLineThrottlePeriod Duration
478 LogUpdatePeriod Duration
479 LogUpdateSize ByteSize
487 SbatchArgumentsList []string
488 SbatchEnvironmentVariables map[string]string
490 DNSServerConfDir string
491 DNSServerConfTemplate string
492 DNSServerReloadCommand string
493 DNSServerUpdateCommand string
494 ComputeNodeDomain string
495 ComputeNodeNameservers StringSet
496 AssignNodeHostname string
501 BsubArgumentsList []string
502 BsubCUDAArguments []string
506 type CloudVMsConfig struct {
509 BootProbeCommand string
510 DeployRunnerBinary string
512 MaxCloudOpsPerSecond int
513 MaxProbesPerSecond int
514 MaxConcurrentInstanceCreateOps int
515 PollInterval Duration
516 ProbeInterval Duration
518 SyncInterval Duration
519 TimeoutBooting Duration
521 TimeoutProbe Duration
522 TimeoutShutdown Duration
523 TimeoutSignal Duration
524 TimeoutStaleRunLock Duration
526 ResourceTags map[string]string
530 DriverParameters json.RawMessage
533 type InstanceTypeMap map[string]InstanceType
535 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
537 // UnmarshalJSON does special handling of InstanceTypes:
538 // * populate computed fields (Name and Scratch)
539 // * error out if InstancesTypes are populated as an array, which was
540 // deprecated in Arvados 1.2.0
541 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
542 fixup := func(t InstanceType) (InstanceType, error) {
543 if t.ProviderType == "" {
544 t.ProviderType = t.Name
546 // If t.Scratch is set in the configuration file, it will be ignored and overwritten.
547 // It will also generate a "deprecated or unknown config entry" warning.
548 t.Scratch = t.IncludedScratch + t.AddedScratch
552 if len(data) > 0 && data[0] == '[' {
553 return fmt.Errorf("InstanceTypes must be specified as a map, not an array, see https://doc.arvados.org/admin/config.html")
555 var hash map[string]InstanceType
556 err := json.Unmarshal(data, &hash)
560 // Fill in Name field (and ProviderType field, if not
561 // specified) using hash key.
562 *it = InstanceTypeMap(hash)
563 for name, t := range *it {
574 type StringSet map[string]struct{}
576 // UnmarshalJSON handles old config files that provide an array of
577 // instance types instead of a hash.
578 func (ss *StringSet) UnmarshalJSON(data []byte) error {
579 if len(data) > 0 && data[0] == '[' {
581 err := json.Unmarshal(data, &arr)
589 *ss = make(map[string]struct{}, len(arr))
590 for _, t := range arr {
591 (*ss)[t] = struct{}{}
595 var hash map[string]struct{}
596 err := json.Unmarshal(data, &hash)
600 *ss = make(map[string]struct{}, len(hash))
601 for t := range hash {
602 (*ss)[t] = struct{}{}
608 type ServiceName string
611 ServiceNameController ServiceName = "arvados-controller"
612 ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
613 ServiceNameDispatchLSF ServiceName = "arvados-dispatch-lsf"
614 ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm"
615 ServiceNameGitHTTP ServiceName = "arvados-git-httpd"
616 ServiceNameHealth ServiceName = "arvados-health"
617 ServiceNameKeepbalance ServiceName = "keep-balance"
618 ServiceNameKeepproxy ServiceName = "keepproxy"
619 ServiceNameKeepstore ServiceName = "keepstore"
620 ServiceNameKeepweb ServiceName = "keep-web"
621 ServiceNameRailsAPI ServiceName = "arvados-api-server"
622 ServiceNameWebsocket ServiceName = "arvados-ws"
623 ServiceNameWorkbench1 ServiceName = "arvados-workbench1"
624 ServiceNameWorkbench2 ServiceName = "arvados-workbench2"
627 // Map returns all services as a map, suitable for iterating over all
628 // services or looking up a service by name.
629 func (svcs Services) Map() map[ServiceName]Service {
630 return map[ServiceName]Service{
631 ServiceNameController: svcs.Controller,
632 ServiceNameDispatchCloud: svcs.DispatchCloud,
633 ServiceNameDispatchLSF: svcs.DispatchLSF,
634 ServiceNameDispatchSLURM: svcs.DispatchSLURM,
635 ServiceNameGitHTTP: svcs.GitHTTP,
636 ServiceNameHealth: svcs.Health,
637 ServiceNameKeepbalance: svcs.Keepbalance,
638 ServiceNameKeepproxy: svcs.Keepproxy,
639 ServiceNameKeepstore: svcs.Keepstore,
640 ServiceNameKeepweb: svcs.WebDAV,
641 ServiceNameRailsAPI: svcs.RailsAPI,
642 ServiceNameWebsocket: svcs.Websocket,
643 ServiceNameWorkbench1: svcs.Workbench1,
644 ServiceNameWorkbench2: svcs.Workbench2,