1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: Apache-2.0
16 "git.arvados.org/arvados.git/sdk/go/config"
19 var DefaultConfigFile = func() string {
20 if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
23 return "/etc/arvados/config.yml"
27 Clusters map[string]Cluster
29 SourceTimestamp time.Time
33 // GetConfig returns the current system config, loading it from
34 // configFile if needed.
35 func GetConfig(configFile string) (*Config, error) {
37 err := config.LoadFile(&cfg, configFile)
41 // GetCluster returns the cluster ID and config for the given
42 // cluster, or the default/only configured cluster if clusterID is "".
43 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
45 if len(sc.Clusters) == 0 {
46 return nil, fmt.Errorf("no clusters configured")
47 } else if len(sc.Clusters) > 1 {
48 return nil, fmt.Errorf("multiple clusters configured, cannot choose")
50 for id, cc := range sc.Clusters {
56 cc, ok := sc.Clusters[clusterID]
58 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
60 cc.ClusterID = clusterID
64 type WebDAVCacheConfig struct {
67 MaxCollectionBytes int64
71 type UploadDownloadPermission struct {
76 type UploadDownloadRolePermissions struct {
77 User UploadDownloadPermission
78 Admin UploadDownloadPermission
81 type ManagedProperties map[string]struct {
88 ClusterID string `json:"-"`
89 ManagementToken string
90 SystemRootToken string
92 InstanceTypes InstanceTypeMap
93 Containers ContainersConfig
94 RemoteClusters map[string]RemoteCluster
98 AsyncPermissionsUpdateInterval Duration
99 DisabledAPIs StringSet
100 MaxIndexDatabaseRead int
101 MaxItemsPerResponse int
102 MaxConcurrentRequests int
103 MaxKeepBlobBuffers int
104 MaxRequestAmplification int
106 MaxTokenLifetime Duration
107 RequestTimeout Duration
109 WebsocketClientEventQueue int
110 WebsocketServerEventQueue int
111 KeepServiceRequestTimeout Duration
112 VocabularyPath string
113 FreezeProjectRequiresDescription bool
114 FreezeProjectRequiresProperties StringSet
115 UnfreezeProjectRequiresAdmin bool
116 LockBeforeUpdate bool
121 UnloggedAttributes StringSet
125 BlobSigningKey string
126 BlobSigningTTL Duration
128 BlobTrashLifetime Duration
129 BlobTrashCheckInterval Duration
130 BlobTrashConcurrency int
131 BlobDeleteConcurrency int
132 BlobReplicateConcurrency int
133 CollectionVersioning bool
134 DefaultTrashLifetime Duration
135 DefaultReplication int
136 ManagedProperties ManagedProperties
137 PreserveVersionIfIdle Duration
138 TrashSweepInterval Duration
140 ForwardSlashNameSubstitution string
143 BlobMissingReport string
144 BalancePeriod Duration
145 BalanceCollectionBatch int
146 BalanceCollectionBuffers int
147 BalanceTimeout Duration
148 BalanceUpdateLimit int
150 WebDAVCache WebDAVCacheConfig
152 KeepproxyPermission UploadDownloadRolePermissions
153 WebDAVPermission UploadDownloadRolePermissions
167 MinTLSVersion TLSVersion
170 SearchAttribute string
171 SearchBindUser string
172 SearchBindPassword string
175 EmailAttribute string
176 UsernameAttribute string
182 AlternateEmailAddresses bool
183 AuthenticationRequestParameters map[string]string
185 OpenIDConnect struct {
191 EmailVerifiedClaim string
193 AcceptAccessToken bool
194 AcceptAccessTokenScope string
195 AuthenticationRequestParameters map[string]string
200 DefaultEmailDomain string
204 Users map[string]TestUser
207 RemoteTokenRefresh Duration
208 TokenLifetime Duration
209 TrustedClients map[URL]struct{}
210 TrustPrivateNetworks bool
211 IssueTrustedTokens bool
214 MailchimpAPIKey string
215 MailchimpListID string
216 SendUserSetupNotificationEmail bool
217 IssueReporterEmailFrom string
218 IssueReporterEmailTo string
219 SupportEmailAddress string
225 MaxRequestLogParamsSize int
236 ActivatedUsersAreVisibleToOthers bool
237 AnonymousUserToken string
238 AdminNotifierEmailFrom string
239 AutoAdminFirstUser bool
240 AutoAdminUserWithEmail string
241 AutoSetupNewUsers bool
242 AutoSetupNewUsersWithRepository bool
243 AutoSetupNewUsersWithVmUUID string
244 AutoSetupUsernameBlacklist StringSet
245 EmailSubjectPrefix string
246 NewInactiveUserNotificationRecipients StringSet
247 NewUserNotificationRecipients StringSet
248 NewUsersAreActive bool
249 UserNotifierEmailFrom string
250 UserNotifierEmailBcc StringSet
251 UserProfileNotificationAddress string
252 PreferDomainForUsername string
253 UserSetupMailText string
254 RoleGroupsVisibleToAll bool
255 CanCreateRoleGroups bool
256 ActivityLoggingPeriod Duration
258 StorageClasses map[string]StorageClassConfig
259 Volumes map[string]Volume
261 ActivationContactLink string
262 APIClientConnectTimeout Duration
263 APIClientReceiveTimeout Duration
264 APIResponseCompression bool
265 ApplicationMimetypesWithViewIcon StringSet
266 ArvadosDocsite string
267 ArvadosPublicDataDocURL string
268 DefaultOpenIdPrefix string
269 DisableSharingURLsUI bool
270 EnableGettingStartedPopup bool
271 EnablePublicProjectsPage bool
272 FileViewersConfigURL string
273 LogViewerMaxBytes ByteSize
274 MultiSiteSearch string
275 ProfilingEnabled bool
277 RepositoryCache string
278 RunningJobLogRecordsToFetch int
280 ShowRecentCollectionsOnDashboard bool
281 ShowUserAgreementInline bool
282 ShowUserNotifications bool
285 UserProfileFormFields map[string]struct {
287 FormFieldTitle string
288 FormFieldDescription string
291 Options map[string]struct{}
293 UserProfileFormMessage string
294 WelcomePageHTML string
295 InactivePageHTML string
296 SSHHelpPageHTML string
297 SSHHelpHostSuffix string
303 type StorageClassConfig struct {
309 AccessViaHosts map[URL]VolumeAccess
312 StorageClasses map[string]bool
314 DriverParameters json.RawMessage
317 type S3VolumeDriverParameters struct {
320 SecretAccessKey string
324 LocationConstraint bool
326 UseAWSS3v2Driver bool
328 ConnectTimeout Duration
335 type AzureVolumeDriverParameters struct {
336 StorageAccountName string
337 StorageAccountKey string
338 StorageBaseURL string
340 RequestTimeout Duration
341 ListBlobsRetryDelay Duration
342 ListBlobsMaxAttempts int
345 type DirectoryVolumeDriverParameters struct {
350 type VolumeAccess struct {
354 type Services struct {
357 DispatchCloud Service
359 DispatchSLURM Service
367 WebDAVDownload Service
375 type Service struct {
376 InternalURLs map[URL]ServiceInstance
380 type TestUser struct {
385 // URL is a url.URL that is also usable as a JSON key/value.
388 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
389 // used as a JSON key/value.
390 func (su *URL) UnmarshalText(text []byte) error {
391 u, err := url.Parse(string(text))
394 if su.Path == "" && su.Host != "" {
395 // http://example really means http://example/
402 func (su URL) MarshalText() ([]byte, error) {
403 return []byte(su.String()), nil
406 func (su URL) String() string {
407 return (*url.URL)(&su).String()
410 type TLSVersion uint16
412 func (v TLSVersion) MarshalText() ([]byte, error) {
416 case tls.VersionTLS10:
417 return []byte("1.0"), nil
418 case tls.VersionTLS11:
419 return []byte("1.1"), nil
420 case tls.VersionTLS12:
421 return []byte("1.2"), nil
422 case tls.VersionTLS13:
423 return []byte("1.3"), nil
425 return nil, fmt.Errorf("unsupported TLSVersion %x", v)
429 func (v *TLSVersion) UnmarshalJSON(text []byte) error {
430 if len(text) > 0 && text[0] == '"' {
432 err := json.Unmarshal(text, &s)
438 switch string(text) {
442 *v = tls.VersionTLS10
444 *v = tls.VersionTLS11
446 *v = tls.VersionTLS12
448 *v = tls.VersionTLS13
450 return fmt.Errorf("unsupported TLSVersion %q", text)
455 type ServiceInstance struct {
457 Rendezvous string `json:",omitempty"`
460 type PostgreSQL struct {
461 Connection PostgreSQLConnection
465 type PostgreSQLConnection map[string]string
467 type RemoteCluster struct {
475 type CUDAFeatures struct {
477 HardwareCapability string
481 type InstanceType struct {
482 Name string `json:"-"`
486 Scratch ByteSize `json:"-"`
487 IncludedScratch ByteSize
488 AddedScratch ByteSize
494 type ContainersConfig struct {
495 CloudVMs CloudVMsConfig
496 CrunchRunCommand string
497 CrunchRunArgumentsList []string
498 DefaultKeepCacheRAM ByteSize
499 DispatchPrivateKey string
500 LogReuseDecisions bool
501 MaxDispatchAttempts int
503 MinRetryPeriod Duration
504 ReserveExtraRAM ByteSize
505 StaleLockTimeout Duration
506 SupportedDockerImageFormats StringSet
507 AlwaysUsePreemptibleInstances bool
508 PreemptiblePriceFactor float64
509 SupervisorFraction float64
511 LocalKeepBlobBuffersPerVCPU int
512 LocalKeepLogsToContainerLog string
516 GitInternalDir string
520 SweepInterval Duration
522 LogSecondsBetweenEvents Duration
523 LogThrottlePeriod Duration
526 LimitLogBytesPerJob int
527 LogPartialLineThrottlePeriod Duration
528 LogUpdatePeriod Duration
529 LogUpdateSize ByteSize
537 SbatchArgumentsList []string
538 SbatchEnvironmentVariables map[string]string
540 DNSServerConfDir string
541 DNSServerConfTemplate string
542 DNSServerReloadCommand string
543 DNSServerUpdateCommand string
544 ComputeNodeDomain string
545 ComputeNodeNameservers StringSet
546 AssignNodeHostname string
551 BsubArgumentsList []string
552 BsubCUDAArguments []string
556 type CloudVMsConfig struct {
559 BootProbeCommand string
560 DeployRunnerBinary string
562 MaxCloudOpsPerSecond int
563 MaxProbesPerSecond int
564 MaxConcurrentInstanceCreateOps int
566 PollInterval Duration
567 ProbeInterval Duration
569 SyncInterval Duration
570 TimeoutBooting Duration
572 TimeoutProbe Duration
573 TimeoutShutdown Duration
574 TimeoutSignal Duration
575 TimeoutStaleRunLock Duration
577 ResourceTags map[string]string
581 DriverParameters json.RawMessage
584 type InstanceTypeMap map[string]InstanceType
586 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
588 // UnmarshalJSON does special handling of InstanceTypes:
590 // - populate computed fields (Name and Scratch)
592 // - error out if InstancesTypes are populated as an array, which was
593 // deprecated in Arvados 1.2.0
594 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
595 fixup := func(t InstanceType) (InstanceType, error) {
596 if t.ProviderType == "" {
597 t.ProviderType = t.Name
599 // If t.Scratch is set in the configuration file, it will be ignored and overwritten.
600 // It will also generate a "deprecated or unknown config entry" warning.
601 t.Scratch = t.IncludedScratch + t.AddedScratch
605 if len(data) > 0 && data[0] == '[' {
606 return fmt.Errorf("InstanceTypes must be specified as a map, not an array, see https://doc.arvados.org/admin/config.html")
608 var hash map[string]InstanceType
609 err := json.Unmarshal(data, &hash)
613 // Fill in Name field (and ProviderType field, if not
614 // specified) using hash key.
615 *it = InstanceTypeMap(hash)
616 for name, t := range *it {
627 type StringSet map[string]struct{}
629 // UnmarshalJSON handles old config files that provide an array of
630 // instance types instead of a hash.
631 func (ss *StringSet) UnmarshalJSON(data []byte) error {
632 if len(data) > 0 && data[0] == '[' {
634 err := json.Unmarshal(data, &arr)
642 *ss = make(map[string]struct{}, len(arr))
643 for _, t := range arr {
644 (*ss)[t] = struct{}{}
648 var hash map[string]struct{}
649 err := json.Unmarshal(data, &hash)
653 *ss = make(map[string]struct{}, len(hash))
654 for t := range hash {
655 (*ss)[t] = struct{}{}
661 type ServiceName string
664 ServiceNameController ServiceName = "arvados-controller"
665 ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
666 ServiceNameDispatchLSF ServiceName = "arvados-dispatch-lsf"
667 ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm"
668 ServiceNameGitHTTP ServiceName = "arvados-git-httpd"
669 ServiceNameHealth ServiceName = "arvados-health"
670 ServiceNameKeepbalance ServiceName = "keep-balance"
671 ServiceNameKeepproxy ServiceName = "keepproxy"
672 ServiceNameKeepstore ServiceName = "keepstore"
673 ServiceNameKeepweb ServiceName = "keep-web"
674 ServiceNameRailsAPI ServiceName = "arvados-api-server"
675 ServiceNameWebsocket ServiceName = "arvados-ws"
676 ServiceNameWorkbench1 ServiceName = "arvados-workbench1"
677 ServiceNameWorkbench2 ServiceName = "arvados-workbench2"
680 // Map returns all services as a map, suitable for iterating over all
681 // services or looking up a service by name.
682 func (svcs Services) Map() map[ServiceName]Service {
683 return map[ServiceName]Service{
684 ServiceNameController: svcs.Controller,
685 ServiceNameDispatchCloud: svcs.DispatchCloud,
686 ServiceNameDispatchLSF: svcs.DispatchLSF,
687 ServiceNameDispatchSLURM: svcs.DispatchSLURM,
688 ServiceNameGitHTTP: svcs.GitHTTP,
689 ServiceNameHealth: svcs.Health,
690 ServiceNameKeepbalance: svcs.Keepbalance,
691 ServiceNameKeepproxy: svcs.Keepproxy,
692 ServiceNameKeepstore: svcs.Keepstore,
693 ServiceNameKeepweb: svcs.WebDAV,
694 ServiceNameRailsAPI: svcs.RailsAPI,
695 ServiceNameWebsocket: svcs.Websocket,
696 ServiceNameWorkbench1: svcs.Workbench1,
697 ServiceNameWorkbench2: svcs.Workbench2,