Merge branch '17983-fuse-unlock-relock' into main refs #17983
[arvados.git] / sdk / go / arvados / config.go
1 // Copyright (C) The Arvados Authors. All rights reserved.
2 //
3 // SPDX-License-Identifier: Apache-2.0
4
5 package arvados
6
7 import (
8         "encoding/json"
9         "errors"
10         "fmt"
11         "net/url"
12         "os"
13
14         "git.arvados.org/arvados.git/sdk/go/config"
15 )
16
17 var DefaultConfigFile = func() string {
18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
19                 return path
20         }
21         return "/etc/arvados/config.yml"
22 }()
23
24 type Config struct {
25         Clusters         map[string]Cluster
26         AutoReloadConfig bool
27 }
28
29 // GetConfig returns the current system config, loading it from
30 // configFile if needed.
31 func GetConfig(configFile string) (*Config, error) {
32         var cfg Config
33         err := config.LoadFile(&cfg, configFile)
34         return &cfg, err
35 }
36
37 // GetCluster returns the cluster ID and config for the given
38 // cluster, or the default/only configured cluster if clusterID is "".
39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
40         if clusterID == "" {
41                 if len(sc.Clusters) == 0 {
42                         return nil, fmt.Errorf("no clusters configured")
43                 } else if len(sc.Clusters) > 1 {
44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
45                 } else {
46                         for id, cc := range sc.Clusters {
47                                 cc.ClusterID = id
48                                 return &cc, nil
49                         }
50                 }
51         }
52         cc, ok := sc.Clusters[clusterID]
53         if !ok {
54                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
55         }
56         cc.ClusterID = clusterID
57         return &cc, nil
58 }
59
60 type WebDAVCacheConfig struct {
61         TTL                  Duration
62         UUIDTTL              Duration
63         MaxBlockEntries      int
64         MaxCollectionEntries int
65         MaxCollectionBytes   int64
66         MaxPermissionEntries int
67         MaxUUIDEntries       int
68         MaxSessions          int
69 }
70
71 type UploadDownloadPermission struct {
72         Upload   bool
73         Download bool
74 }
75
76 type UploadDownloadRolePermissions struct {
77         User  UploadDownloadPermission
78         Admin UploadDownloadPermission
79 }
80
81 type Cluster struct {
82         ClusterID       string `json:"-"`
83         ManagementToken string
84         SystemRootToken string
85         Services        Services
86         InstanceTypes   InstanceTypeMap
87         Containers      ContainersConfig
88         RemoteClusters  map[string]RemoteCluster
89         PostgreSQL      PostgreSQL
90
91         API struct {
92                 AsyncPermissionsUpdateInterval Duration
93                 DisabledAPIs                   StringSet
94                 MaxIndexDatabaseRead           int
95                 MaxItemsPerResponse            int
96                 MaxConcurrentRequests          int
97                 MaxKeepBlobBuffers             int
98                 MaxRequestAmplification        int
99                 MaxRequestSize                 int
100                 MaxTokenLifetime               Duration
101                 RequestTimeout                 Duration
102                 SendTimeout                    Duration
103                 WebsocketClientEventQueue      int
104                 WebsocketServerEventQueue      int
105                 KeepServiceRequestTimeout      Duration
106         }
107         AuditLogs struct {
108                 MaxAge             Duration
109                 MaxDeleteBatch     int
110                 UnloggedAttributes StringSet
111         }
112         Collections struct {
113                 BlobSigning              bool
114                 BlobSigningKey           string
115                 BlobSigningTTL           Duration
116                 BlobTrash                bool
117                 BlobTrashLifetime        Duration
118                 BlobTrashCheckInterval   Duration
119                 BlobTrashConcurrency     int
120                 BlobDeleteConcurrency    int
121                 BlobReplicateConcurrency int
122                 CollectionVersioning     bool
123                 DefaultTrashLifetime     Duration
124                 DefaultReplication       int
125                 ManagedProperties        map[string]struct {
126                         Value     interface{}
127                         Function  string
128                         Protected bool
129                 }
130                 PreserveVersionIfIdle        Duration
131                 TrashSweepInterval           Duration
132                 TrustAllContent              bool
133                 ForwardSlashNameSubstitution string
134                 S3FolderObjects              bool
135
136                 BlobMissingReport        string
137                 BalancePeriod            Duration
138                 BalanceCollectionBatch   int
139                 BalanceCollectionBuffers int
140                 BalanceTimeout           Duration
141                 BalanceUpdateLimit       int
142
143                 WebDAVCache WebDAVCacheConfig
144
145                 KeepproxyPermission UploadDownloadRolePermissions
146                 WebDAVPermission    UploadDownloadRolePermissions
147                 WebDAVLogEvents     bool
148         }
149         Git struct {
150                 GitCommand   string
151                 GitoliteHome string
152                 Repositories string
153         }
154         Login struct {
155                 LDAP struct {
156                         Enable             bool
157                         URL                URL
158                         StartTLS           bool
159                         InsecureTLS        bool
160                         StripDomain        string
161                         AppendDomain       string
162                         SearchAttribute    string
163                         SearchBindUser     string
164                         SearchBindPassword string
165                         SearchBase         string
166                         SearchFilters      string
167                         EmailAttribute     string
168                         UsernameAttribute  string
169                 }
170                 Google struct {
171                         Enable                          bool
172                         ClientID                        string
173                         ClientSecret                    string
174                         AlternateEmailAddresses         bool
175                         AuthenticationRequestParameters map[string]string
176                 }
177                 OpenIDConnect struct {
178                         Enable                          bool
179                         Issuer                          string
180                         ClientID                        string
181                         ClientSecret                    string
182                         EmailClaim                      string
183                         EmailVerifiedClaim              string
184                         UsernameClaim                   string
185                         AcceptAccessToken               bool
186                         AcceptAccessTokenScope          string
187                         AuthenticationRequestParameters map[string]string
188                 }
189                 PAM struct {
190                         Enable             bool
191                         Service            string
192                         DefaultEmailDomain string
193                 }
194                 Test struct {
195                         Enable bool
196                         Users  map[string]TestUser
197                 }
198                 LoginCluster       string
199                 RemoteTokenRefresh Duration
200                 TokenLifetime      Duration
201                 TrustedClients     map[string]struct{}
202                 IssueTrustedTokens bool
203         }
204         Mail struct {
205                 MailchimpAPIKey                string
206                 MailchimpListID                string
207                 SendUserSetupNotificationEmail bool
208                 IssueReporterEmailFrom         string
209                 IssueReporterEmailTo           string
210                 SupportEmailAddress            string
211                 EmailFrom                      string
212         }
213         SystemLogs struct {
214                 LogLevel                string
215                 Format                  string
216                 MaxRequestLogParamsSize int
217         }
218         TLS struct {
219                 Certificate string
220                 Key         string
221                 Insecure    bool
222         }
223         Users struct {
224                 AnonymousUserToken                    string
225                 AdminNotifierEmailFrom                string
226                 AutoAdminFirstUser                    bool
227                 AutoAdminUserWithEmail                string
228                 AutoSetupNewUsers                     bool
229                 AutoSetupNewUsersWithRepository       bool
230                 AutoSetupNewUsersWithVmUUID           string
231                 AutoSetupUsernameBlacklist            StringSet
232                 EmailSubjectPrefix                    string
233                 NewInactiveUserNotificationRecipients StringSet
234                 NewUserNotificationRecipients         StringSet
235                 NewUsersAreActive                     bool
236                 UserNotifierEmailFrom                 string
237                 UserNotifierEmailBcc                  StringSet
238                 UserProfileNotificationAddress        string
239                 PreferDomainForUsername               string
240                 UserSetupMailText                     string
241         }
242         StorageClasses map[string]StorageClassConfig
243         Volumes        map[string]Volume
244         Workbench      struct {
245                 ActivationContactLink            string
246                 APIClientConnectTimeout          Duration
247                 APIClientReceiveTimeout          Duration
248                 APIResponseCompression           bool
249                 ApplicationMimetypesWithViewIcon StringSet
250                 ArvadosDocsite                   string
251                 ArvadosPublicDataDocURL          string
252                 DefaultOpenIdPrefix              string
253                 EnableGettingStartedPopup        bool
254                 EnablePublicProjectsPage         bool
255                 FileViewersConfigURL             string
256                 LogViewerMaxBytes                ByteSize
257                 MultiSiteSearch                  string
258                 ProfilingEnabled                 bool
259                 Repositories                     bool
260                 RepositoryCache                  string
261                 RunningJobLogRecordsToFetch      int
262                 SecretKeyBase                    string
263                 ShowRecentCollectionsOnDashboard bool
264                 ShowUserAgreementInline          bool
265                 ShowUserNotifications            bool
266                 SiteName                         string
267                 Theme                            string
268                 UserProfileFormFields            map[string]struct {
269                         Type                 string
270                         FormFieldTitle       string
271                         FormFieldDescription string
272                         Required             bool
273                         Position             int
274                         Options              map[string]struct{}
275                 }
276                 UserProfileFormMessage string
277                 VocabularyURL          string
278                 WelcomePageHTML        string
279                 InactivePageHTML       string
280                 SSHHelpPageHTML        string
281                 SSHHelpHostSuffix      string
282                 IdleTimeout            Duration
283         }
284 }
285
286 type StorageClassConfig struct {
287         Default  bool
288         Priority int
289 }
290
291 type Volume struct {
292         AccessViaHosts   map[URL]VolumeAccess
293         ReadOnly         bool
294         Replication      int
295         StorageClasses   map[string]bool
296         Driver           string
297         DriverParameters json.RawMessage
298 }
299
300 type S3VolumeDriverParameters struct {
301         IAMRole            string
302         AccessKeyID        string
303         SecretAccessKey    string
304         Endpoint           string
305         Region             string
306         Bucket             string
307         LocationConstraint bool
308         V2Signature        bool
309         UseAWSS3v2Driver   bool
310         IndexPageSize      int
311         ConnectTimeout     Duration
312         ReadTimeout        Duration
313         RaceWindow         Duration
314         UnsafeDelete       bool
315 }
316
317 type AzureVolumeDriverParameters struct {
318         StorageAccountName   string
319         StorageAccountKey    string
320         StorageBaseURL       string
321         ContainerName        string
322         RequestTimeout       Duration
323         ListBlobsRetryDelay  Duration
324         ListBlobsMaxAttempts int
325 }
326
327 type DirectoryVolumeDriverParameters struct {
328         Root      string
329         Serialize bool
330 }
331
332 type VolumeAccess struct {
333         ReadOnly bool
334 }
335
336 type Services struct {
337         Composer       Service
338         Controller     Service
339         DispatchCloud  Service
340         DispatchLSF    Service
341         GitHTTP        Service
342         GitSSH         Service
343         Health         Service
344         Keepbalance    Service
345         Keepproxy      Service
346         Keepstore      Service
347         RailsAPI       Service
348         WebDAVDownload Service
349         WebDAV         Service
350         WebShell       Service
351         Websocket      Service
352         Workbench1     Service
353         Workbench2     Service
354 }
355
356 type Service struct {
357         InternalURLs map[URL]ServiceInstance
358         ExternalURL  URL
359 }
360
361 type TestUser struct {
362         Email    string
363         Password string
364 }
365
366 // URL is a url.URL that is also usable as a JSON key/value.
367 type URL url.URL
368
369 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
370 // used as a JSON key/value.
371 func (su *URL) UnmarshalText(text []byte) error {
372         u, err := url.Parse(string(text))
373         if err == nil {
374                 *su = URL(*u)
375                 if su.Path == "" && su.Host != "" {
376                         // http://example really means http://example/
377                         su.Path = "/"
378                 }
379         }
380         return err
381 }
382
383 func (su URL) MarshalText() ([]byte, error) {
384         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
385 }
386
387 func (su URL) String() string {
388         return (*url.URL)(&su).String()
389 }
390
391 type ServiceInstance struct {
392         Rendezvous string `json:",omitempty"`
393 }
394
395 type PostgreSQL struct {
396         Connection     PostgreSQLConnection
397         ConnectionPool int
398 }
399
400 type PostgreSQLConnection map[string]string
401
402 type RemoteCluster struct {
403         Host          string
404         Proxy         bool
405         Scheme        string
406         Insecure      bool
407         ActivateUsers bool
408 }
409
410 type InstanceType struct {
411         Name            string
412         ProviderType    string
413         VCPUs           int
414         RAM             ByteSize
415         Scratch         ByteSize
416         IncludedScratch ByteSize
417         AddedScratch    ByteSize
418         Price           float64
419         Preemptible     bool
420 }
421
422 type ContainersConfig struct {
423         CloudVMs                    CloudVMsConfig
424         CrunchRunCommand            string
425         CrunchRunArgumentsList      []string
426         DefaultKeepCacheRAM         ByteSize
427         DispatchPrivateKey          string
428         LogReuseDecisions           bool
429         MaxComputeVMs               int
430         MaxDispatchAttempts         int
431         MaxRetryAttempts            int
432         MinRetryPeriod              Duration
433         ReserveExtraRAM             ByteSize
434         StaleLockTimeout            Duration
435         SupportedDockerImageFormats StringSet
436         UsePreemptibleInstances     bool
437         RuntimeEngine               string
438
439         JobsAPI struct {
440                 Enable         string
441                 GitInternalDir string
442         }
443         Logging struct {
444                 MaxAge                       Duration
445                 LogBytesPerEvent             int
446                 LogSecondsBetweenEvents      Duration
447                 LogThrottlePeriod            Duration
448                 LogThrottleBytes             int
449                 LogThrottleLines             int
450                 LimitLogBytesPerJob          int
451                 LogPartialLineThrottlePeriod Duration
452                 LogUpdatePeriod              Duration
453                 LogUpdateSize                ByteSize
454         }
455         ShellAccess struct {
456                 Admin bool
457                 User  bool
458         }
459         SLURM struct {
460                 PrioritySpread             int64
461                 SbatchArgumentsList        []string
462                 SbatchEnvironmentVariables map[string]string
463                 Managed                    struct {
464                         DNSServerConfDir       string
465                         DNSServerConfTemplate  string
466                         DNSServerReloadCommand string
467                         DNSServerUpdateCommand string
468                         ComputeNodeDomain      string
469                         ComputeNodeNameservers StringSet
470                         AssignNodeHostname     string
471                 }
472         }
473         LSF struct {
474                 BsubSudoUser      string
475                 BsubArgumentsList []string
476         }
477 }
478
479 type CloudVMsConfig struct {
480         Enable bool
481
482         BootProbeCommand               string
483         DeployRunnerBinary             string
484         ImageID                        string
485         MaxCloudOpsPerSecond           int
486         MaxProbesPerSecond             int
487         MaxConcurrentInstanceCreateOps int
488         PollInterval                   Duration
489         ProbeInterval                  Duration
490         SSHPort                        string
491         SyncInterval                   Duration
492         TimeoutBooting                 Duration
493         TimeoutIdle                    Duration
494         TimeoutProbe                   Duration
495         TimeoutShutdown                Duration
496         TimeoutSignal                  Duration
497         TimeoutStaleRunLock            Duration
498         TimeoutTERM                    Duration
499         ResourceTags                   map[string]string
500         TagKeyPrefix                   string
501
502         Driver           string
503         DriverParameters json.RawMessage
504 }
505
506 type InstanceTypeMap map[string]InstanceType
507
508 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
509
510 // UnmarshalJSON handles old config files that provide an array of
511 // instance types instead of a hash.
512 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
513         fixup := func(t InstanceType) (InstanceType, error) {
514                 if t.ProviderType == "" {
515                         t.ProviderType = t.Name
516                 }
517                 if t.Scratch == 0 {
518                         t.Scratch = t.IncludedScratch + t.AddedScratch
519                 } else if t.AddedScratch == 0 {
520                         t.AddedScratch = t.Scratch - t.IncludedScratch
521                 } else if t.IncludedScratch == 0 {
522                         t.IncludedScratch = t.Scratch - t.AddedScratch
523                 }
524
525                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
526                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
527                 }
528                 return t, nil
529         }
530
531         if len(data) > 0 && data[0] == '[' {
532                 var arr []InstanceType
533                 err := json.Unmarshal(data, &arr)
534                 if err != nil {
535                         return err
536                 }
537                 if len(arr) == 0 {
538                         *it = nil
539                         return nil
540                 }
541                 *it = make(map[string]InstanceType, len(arr))
542                 for _, t := range arr {
543                         if _, ok := (*it)[t.Name]; ok {
544                                 return errDuplicateInstanceTypeName
545                         }
546                         t, err := fixup(t)
547                         if err != nil {
548                                 return err
549                         }
550                         (*it)[t.Name] = t
551                 }
552                 return nil
553         }
554         var hash map[string]InstanceType
555         err := json.Unmarshal(data, &hash)
556         if err != nil {
557                 return err
558         }
559         // Fill in Name field (and ProviderType field, if not
560         // specified) using hash key.
561         *it = InstanceTypeMap(hash)
562         for name, t := range *it {
563                 t.Name = name
564                 t, err := fixup(t)
565                 if err != nil {
566                         return err
567                 }
568                 (*it)[name] = t
569         }
570         return nil
571 }
572
573 type StringSet map[string]struct{}
574
575 // UnmarshalJSON handles old config files that provide an array of
576 // instance types instead of a hash.
577 func (ss *StringSet) UnmarshalJSON(data []byte) error {
578         if len(data) > 0 && data[0] == '[' {
579                 var arr []string
580                 err := json.Unmarshal(data, &arr)
581                 if err != nil {
582                         return err
583                 }
584                 if len(arr) == 0 {
585                         *ss = nil
586                         return nil
587                 }
588                 *ss = make(map[string]struct{}, len(arr))
589                 for _, t := range arr {
590                         (*ss)[t] = struct{}{}
591                 }
592                 return nil
593         }
594         var hash map[string]struct{}
595         err := json.Unmarshal(data, &hash)
596         if err != nil {
597                 return err
598         }
599         *ss = make(map[string]struct{}, len(hash))
600         for t := range hash {
601                 (*ss)[t] = struct{}{}
602         }
603
604         return nil
605 }
606
607 type ServiceName string
608
609 const (
610         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
611         ServiceNameController    ServiceName = "arvados-controller"
612         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
613         ServiceNameDispatchLSF   ServiceName = "arvados-dispatch-lsf"
614         ServiceNameHealth        ServiceName = "arvados-health"
615         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
616         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
617         ServiceNameWebsocket     ServiceName = "arvados-ws"
618         ServiceNameKeepbalance   ServiceName = "keep-balance"
619         ServiceNameKeepweb       ServiceName = "keep-web"
620         ServiceNameKeepproxy     ServiceName = "keepproxy"
621         ServiceNameKeepstore     ServiceName = "keepstore"
622 )
623
624 // Map returns all services as a map, suitable for iterating over all
625 // services or looking up a service by name.
626 func (svcs Services) Map() map[ServiceName]Service {
627         return map[ServiceName]Service{
628                 ServiceNameRailsAPI:      svcs.RailsAPI,
629                 ServiceNameController:    svcs.Controller,
630                 ServiceNameDispatchCloud: svcs.DispatchCloud,
631                 ServiceNameDispatchLSF:   svcs.DispatchLSF,
632                 ServiceNameHealth:        svcs.Health,
633                 ServiceNameWorkbench1:    svcs.Workbench1,
634                 ServiceNameWorkbench2:    svcs.Workbench2,
635                 ServiceNameWebsocket:     svcs.Websocket,
636                 ServiceNameKeepbalance:   svcs.Keepbalance,
637                 ServiceNameKeepweb:       svcs.WebDAV,
638                 ServiceNameKeepproxy:     svcs.Keepproxy,
639                 ServiceNameKeepstore:     svcs.Keepstore,
640         }
641 }