1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: Apache-2.0
15 "git.arvados.org/arvados.git/sdk/go/config"
18 var DefaultConfigFile = func() string {
19 if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
22 return "/etc/arvados/config.yml"
26 Clusters map[string]Cluster
28 SourceTimestamp time.Time
32 // GetConfig returns the current system config, loading it from
33 // configFile if needed.
34 func GetConfig(configFile string) (*Config, error) {
36 err := config.LoadFile(&cfg, configFile)
40 // GetCluster returns the cluster ID and config for the given
41 // cluster, or the default/only configured cluster if clusterID is "".
42 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
44 if len(sc.Clusters) == 0 {
45 return nil, fmt.Errorf("no clusters configured")
46 } else if len(sc.Clusters) > 1 {
47 return nil, fmt.Errorf("multiple clusters configured, cannot choose")
49 for id, cc := range sc.Clusters {
55 cc, ok := sc.Clusters[clusterID]
57 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
59 cc.ClusterID = clusterID
63 type WebDAVCacheConfig struct {
66 MaxCollectionBytes int64
70 type UploadDownloadPermission struct {
75 type UploadDownloadRolePermissions struct {
76 User UploadDownloadPermission
77 Admin UploadDownloadPermission
80 type ManagedProperties map[string]struct {
87 ClusterID string `json:"-"`
88 ManagementToken string
89 SystemRootToken string
91 InstanceTypes InstanceTypeMap
92 Containers ContainersConfig
93 RemoteClusters map[string]RemoteCluster
97 AsyncPermissionsUpdateInterval Duration
98 DisabledAPIs StringSet
99 MaxIndexDatabaseRead int
100 MaxItemsPerResponse int
101 MaxConcurrentRequests int
102 MaxKeepBlobBuffers int
103 MaxRequestAmplification int
105 MaxTokenLifetime Duration
106 RequestTimeout Duration
108 WebsocketClientEventQueue int
109 WebsocketServerEventQueue int
110 KeepServiceRequestTimeout Duration
111 VocabularyPath string
112 FreezeProjectRequiresDescription bool
113 FreezeProjectRequiresProperties StringSet
114 UnfreezeProjectRequiresAdmin bool
119 UnloggedAttributes StringSet
123 BlobSigningKey string
124 BlobSigningTTL Duration
126 BlobTrashLifetime Duration
127 BlobTrashCheckInterval Duration
128 BlobTrashConcurrency int
129 BlobDeleteConcurrency int
130 BlobReplicateConcurrency int
131 CollectionVersioning bool
132 DefaultTrashLifetime Duration
133 DefaultReplication int
134 ManagedProperties ManagedProperties
135 PreserveVersionIfIdle Duration
136 TrashSweepInterval Duration
138 ForwardSlashNameSubstitution string
141 BlobMissingReport string
142 BalancePeriod Duration
143 BalanceCollectionBatch int
144 BalanceCollectionBuffers int
145 BalanceTimeout Duration
146 BalanceUpdateLimit int
148 WebDAVCache WebDAVCacheConfig
150 KeepproxyPermission UploadDownloadRolePermissions
151 WebDAVPermission UploadDownloadRolePermissions
167 SearchAttribute string
168 SearchBindUser string
169 SearchBindPassword string
172 EmailAttribute string
173 UsernameAttribute string
179 AlternateEmailAddresses bool
180 AuthenticationRequestParameters map[string]string
182 OpenIDConnect struct {
188 EmailVerifiedClaim string
190 AcceptAccessToken bool
191 AcceptAccessTokenScope string
192 AuthenticationRequestParameters map[string]string
197 DefaultEmailDomain string
201 Users map[string]TestUser
204 RemoteTokenRefresh Duration
205 TokenLifetime Duration
206 TrustedClients map[URL]struct{}
207 TrustPrivateNetworks bool
208 IssueTrustedTokens bool
211 MailchimpAPIKey string
212 MailchimpListID string
213 SendUserSetupNotificationEmail bool
214 IssueReporterEmailFrom string
215 IssueReporterEmailTo string
216 SupportEmailAddress string
222 MaxRequestLogParamsSize int
233 ActivatedUsersAreVisibleToOthers bool
234 AnonymousUserToken string
235 AdminNotifierEmailFrom string
236 AutoAdminFirstUser bool
237 AutoAdminUserWithEmail string
238 AutoSetupNewUsers bool
239 AutoSetupNewUsersWithRepository bool
240 AutoSetupNewUsersWithVmUUID string
241 AutoSetupUsernameBlacklist StringSet
242 EmailSubjectPrefix string
243 NewInactiveUserNotificationRecipients StringSet
244 NewUserNotificationRecipients StringSet
245 NewUsersAreActive bool
246 UserNotifierEmailFrom string
247 UserNotifierEmailBcc StringSet
248 UserProfileNotificationAddress string
249 PreferDomainForUsername string
250 UserSetupMailText string
251 RoleGroupsVisibleToAll bool
252 ActivityLoggingPeriod Duration
254 StorageClasses map[string]StorageClassConfig
255 Volumes map[string]Volume
257 ActivationContactLink string
258 APIClientConnectTimeout Duration
259 APIClientReceiveTimeout Duration
260 APIResponseCompression bool
261 ApplicationMimetypesWithViewIcon StringSet
262 ArvadosDocsite string
263 ArvadosPublicDataDocURL string
264 DefaultOpenIdPrefix string
265 DisableSharingURLsUI bool
266 EnableGettingStartedPopup bool
267 EnablePublicProjectsPage bool
268 FileViewersConfigURL string
269 LogViewerMaxBytes ByteSize
270 MultiSiteSearch string
271 ProfilingEnabled bool
273 RepositoryCache string
274 RunningJobLogRecordsToFetch int
276 ShowRecentCollectionsOnDashboard bool
277 ShowUserAgreementInline bool
278 ShowUserNotifications bool
281 UserProfileFormFields map[string]struct {
283 FormFieldTitle string
284 FormFieldDescription string
287 Options map[string]struct{}
289 UserProfileFormMessage string
290 WelcomePageHTML string
291 InactivePageHTML string
292 SSHHelpPageHTML string
293 SSHHelpHostSuffix string
299 type StorageClassConfig struct {
305 AccessViaHosts map[URL]VolumeAccess
308 StorageClasses map[string]bool
310 DriverParameters json.RawMessage
313 type S3VolumeDriverParameters struct {
316 SecretAccessKey string
320 LocationConstraint bool
322 UseAWSS3v2Driver bool
324 ConnectTimeout Duration
331 type AzureVolumeDriverParameters struct {
332 StorageAccountName string
333 StorageAccountKey string
334 StorageBaseURL string
336 RequestTimeout Duration
337 ListBlobsRetryDelay Duration
338 ListBlobsMaxAttempts int
341 type DirectoryVolumeDriverParameters struct {
346 type VolumeAccess struct {
350 type Services struct {
353 DispatchCloud Service
355 DispatchSLURM Service
363 WebDAVDownload Service
371 type Service struct {
372 InternalURLs map[URL]ServiceInstance
376 type TestUser struct {
381 // URL is a url.URL that is also usable as a JSON key/value.
384 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
385 // used as a JSON key/value.
386 func (su *URL) UnmarshalText(text []byte) error {
387 u, err := url.Parse(string(text))
390 if su.Path == "" && su.Host != "" {
391 // http://example really means http://example/
398 func (su URL) MarshalText() ([]byte, error) {
399 return []byte(su.String()), nil
402 func (su URL) String() string {
403 return (*url.URL)(&su).String()
406 type ServiceInstance struct {
408 Rendezvous string `json:",omitempty"`
411 type PostgreSQL struct {
412 Connection PostgreSQLConnection
416 type PostgreSQLConnection map[string]string
418 type RemoteCluster struct {
426 type CUDAFeatures struct {
428 HardwareCapability string
432 type InstanceType struct {
433 Name string `json:"-"`
437 Scratch ByteSize `json:"-"`
438 IncludedScratch ByteSize
439 AddedScratch ByteSize
445 type ContainersConfig struct {
446 CloudVMs CloudVMsConfig
447 CrunchRunCommand string
448 CrunchRunArgumentsList []string
449 DefaultKeepCacheRAM ByteSize
450 DispatchPrivateKey string
451 LogReuseDecisions bool
453 MaxDispatchAttempts int
455 MinRetryPeriod Duration
456 ReserveExtraRAM ByteSize
457 StaleLockTimeout Duration
458 SupportedDockerImageFormats StringSet
459 AlwaysUsePreemptibleInstances bool
460 PreemptiblePriceFactor float64
462 LocalKeepBlobBuffersPerVCPU int
463 LocalKeepLogsToContainerLog string
467 GitInternalDir string
471 SweepInterval Duration
473 LogSecondsBetweenEvents Duration
474 LogThrottlePeriod Duration
477 LimitLogBytesPerJob int
478 LogPartialLineThrottlePeriod Duration
479 LogUpdatePeriod Duration
480 LogUpdateSize ByteSize
488 SbatchArgumentsList []string
489 SbatchEnvironmentVariables map[string]string
491 DNSServerConfDir string
492 DNSServerConfTemplate string
493 DNSServerReloadCommand string
494 DNSServerUpdateCommand string
495 ComputeNodeDomain string
496 ComputeNodeNameservers StringSet
497 AssignNodeHostname string
502 BsubArgumentsList []string
503 BsubCUDAArguments []string
507 type CloudVMsConfig struct {
510 BootProbeCommand string
511 DeployRunnerBinary string
513 MaxCloudOpsPerSecond int
514 MaxProbesPerSecond int
515 MaxConcurrentInstanceCreateOps int
516 PollInterval Duration
517 ProbeInterval Duration
519 SyncInterval Duration
520 TimeoutBooting Duration
522 TimeoutProbe Duration
523 TimeoutShutdown Duration
524 TimeoutSignal Duration
525 TimeoutStaleRunLock Duration
527 ResourceTags map[string]string
531 DriverParameters json.RawMessage
534 type InstanceTypeMap map[string]InstanceType
536 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
538 // UnmarshalJSON does special handling of InstanceTypes:
540 // - populate computed fields (Name and Scratch)
542 // - error out if InstancesTypes are populated as an array, which was
543 // deprecated in Arvados 1.2.0
544 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
545 fixup := func(t InstanceType) (InstanceType, error) {
546 if t.ProviderType == "" {
547 t.ProviderType = t.Name
549 // If t.Scratch is set in the configuration file, it will be ignored and overwritten.
550 // It will also generate a "deprecated or unknown config entry" warning.
551 t.Scratch = t.IncludedScratch + t.AddedScratch
555 if len(data) > 0 && data[0] == '[' {
556 return fmt.Errorf("InstanceTypes must be specified as a map, not an array, see https://doc.arvados.org/admin/config.html")
558 var hash map[string]InstanceType
559 err := json.Unmarshal(data, &hash)
563 // Fill in Name field (and ProviderType field, if not
564 // specified) using hash key.
565 *it = InstanceTypeMap(hash)
566 for name, t := range *it {
577 type StringSet map[string]struct{}
579 // UnmarshalJSON handles old config files that provide an array of
580 // instance types instead of a hash.
581 func (ss *StringSet) UnmarshalJSON(data []byte) error {
582 if len(data) > 0 && data[0] == '[' {
584 err := json.Unmarshal(data, &arr)
592 *ss = make(map[string]struct{}, len(arr))
593 for _, t := range arr {
594 (*ss)[t] = struct{}{}
598 var hash map[string]struct{}
599 err := json.Unmarshal(data, &hash)
603 *ss = make(map[string]struct{}, len(hash))
604 for t := range hash {
605 (*ss)[t] = struct{}{}
611 type ServiceName string
614 ServiceNameController ServiceName = "arvados-controller"
615 ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
616 ServiceNameDispatchLSF ServiceName = "arvados-dispatch-lsf"
617 ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm"
618 ServiceNameGitHTTP ServiceName = "arvados-git-httpd"
619 ServiceNameHealth ServiceName = "arvados-health"
620 ServiceNameKeepbalance ServiceName = "keep-balance"
621 ServiceNameKeepproxy ServiceName = "keepproxy"
622 ServiceNameKeepstore ServiceName = "keepstore"
623 ServiceNameKeepweb ServiceName = "keep-web"
624 ServiceNameRailsAPI ServiceName = "arvados-api-server"
625 ServiceNameWebsocket ServiceName = "arvados-ws"
626 ServiceNameWorkbench1 ServiceName = "arvados-workbench1"
627 ServiceNameWorkbench2 ServiceName = "arvados-workbench2"
630 // Map returns all services as a map, suitable for iterating over all
631 // services or looking up a service by name.
632 func (svcs Services) Map() map[ServiceName]Service {
633 return map[ServiceName]Service{
634 ServiceNameController: svcs.Controller,
635 ServiceNameDispatchCloud: svcs.DispatchCloud,
636 ServiceNameDispatchLSF: svcs.DispatchLSF,
637 ServiceNameDispatchSLURM: svcs.DispatchSLURM,
638 ServiceNameGitHTTP: svcs.GitHTTP,
639 ServiceNameHealth: svcs.Health,
640 ServiceNameKeepbalance: svcs.Keepbalance,
641 ServiceNameKeepproxy: svcs.Keepproxy,
642 ServiceNameKeepstore: svcs.Keepstore,
643 ServiceNameKeepweb: svcs.WebDAV,
644 ServiceNameRailsAPI: svcs.RailsAPI,
645 ServiceNameWebsocket: svcs.Websocket,
646 ServiceNameWorkbench1: svcs.Workbench1,
647 ServiceNameWorkbench2: svcs.Workbench2,