Merge branch '18122-select-param'
[arvados.git] / sdk / go / arvados / config.go
1 // Copyright (C) The Arvados Authors. All rights reserved.
2 //
3 // SPDX-License-Identifier: Apache-2.0
4
5 package arvados
6
7 import (
8         "encoding/json"
9         "errors"
10         "fmt"
11         "net/url"
12         "os"
13
14         "git.arvados.org/arvados.git/sdk/go/config"
15 )
16
17 var DefaultConfigFile = func() string {
18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
19                 return path
20         }
21         return "/etc/arvados/config.yml"
22 }()
23
24 type Config struct {
25         Clusters         map[string]Cluster
26         AutoReloadConfig bool
27 }
28
29 // GetConfig returns the current system config, loading it from
30 // configFile if needed.
31 func GetConfig(configFile string) (*Config, error) {
32         var cfg Config
33         err := config.LoadFile(&cfg, configFile)
34         return &cfg, err
35 }
36
37 // GetCluster returns the cluster ID and config for the given
38 // cluster, or the default/only configured cluster if clusterID is "".
39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
40         if clusterID == "" {
41                 if len(sc.Clusters) == 0 {
42                         return nil, fmt.Errorf("no clusters configured")
43                 } else if len(sc.Clusters) > 1 {
44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
45                 } else {
46                         for id, cc := range sc.Clusters {
47                                 cc.ClusterID = id
48                                 return &cc, nil
49                         }
50                 }
51         }
52         cc, ok := sc.Clusters[clusterID]
53         if !ok {
54                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
55         }
56         cc.ClusterID = clusterID
57         return &cc, nil
58 }
59
60 type WebDAVCacheConfig struct {
61         TTL                  Duration
62         UUIDTTL              Duration
63         MaxBlockEntries      int
64         MaxCollectionEntries int
65         MaxCollectionBytes   int64
66         MaxUUIDEntries       int
67         MaxSessions          int
68 }
69
70 type UploadDownloadPermission struct {
71         Upload   bool
72         Download bool
73 }
74
75 type UploadDownloadRolePermissions struct {
76         User  UploadDownloadPermission
77         Admin UploadDownloadPermission
78 }
79
80 type Cluster struct {
81         ClusterID       string `json:"-"`
82         ManagementToken string
83         SystemRootToken string
84         Services        Services
85         InstanceTypes   InstanceTypeMap
86         Containers      ContainersConfig
87         RemoteClusters  map[string]RemoteCluster
88         PostgreSQL      PostgreSQL
89
90         API struct {
91                 AsyncPermissionsUpdateInterval Duration
92                 DisabledAPIs                   StringSet
93                 MaxIndexDatabaseRead           int
94                 MaxItemsPerResponse            int
95                 MaxConcurrentRequests          int
96                 MaxKeepBlobBuffers             int
97                 MaxRequestAmplification        int
98                 MaxRequestSize                 int
99                 MaxTokenLifetime               Duration
100                 RequestTimeout                 Duration
101                 SendTimeout                    Duration
102                 WebsocketClientEventQueue      int
103                 WebsocketServerEventQueue      int
104                 KeepServiceRequestTimeout      Duration
105         }
106         AuditLogs struct {
107                 MaxAge             Duration
108                 MaxDeleteBatch     int
109                 UnloggedAttributes StringSet
110         }
111         Collections struct {
112                 BlobSigning              bool
113                 BlobSigningKey           string
114                 BlobSigningTTL           Duration
115                 BlobTrash                bool
116                 BlobTrashLifetime        Duration
117                 BlobTrashCheckInterval   Duration
118                 BlobTrashConcurrency     int
119                 BlobDeleteConcurrency    int
120                 BlobReplicateConcurrency int
121                 CollectionVersioning     bool
122                 DefaultTrashLifetime     Duration
123                 DefaultReplication       int
124                 ManagedProperties        map[string]struct {
125                         Value     interface{}
126                         Function  string
127                         Protected bool
128                 }
129                 PreserveVersionIfIdle        Duration
130                 TrashSweepInterval           Duration
131                 TrustAllContent              bool
132                 ForwardSlashNameSubstitution string
133                 S3FolderObjects              bool
134
135                 BlobMissingReport        string
136                 BalancePeriod            Duration
137                 BalanceCollectionBatch   int
138                 BalanceCollectionBuffers int
139                 BalanceTimeout           Duration
140                 BalanceUpdateLimit       int
141
142                 WebDAVCache WebDAVCacheConfig
143
144                 KeepproxyPermission UploadDownloadRolePermissions
145                 WebDAVPermission    UploadDownloadRolePermissions
146                 WebDAVLogEvents     bool
147         }
148         Git struct {
149                 GitCommand   string
150                 GitoliteHome string
151                 Repositories string
152         }
153         Login struct {
154                 LDAP struct {
155                         Enable             bool
156                         URL                URL
157                         StartTLS           bool
158                         InsecureTLS        bool
159                         StripDomain        string
160                         AppendDomain       string
161                         SearchAttribute    string
162                         SearchBindUser     string
163                         SearchBindPassword string
164                         SearchBase         string
165                         SearchFilters      string
166                         EmailAttribute     string
167                         UsernameAttribute  string
168                 }
169                 Google struct {
170                         Enable                          bool
171                         ClientID                        string
172                         ClientSecret                    string
173                         AlternateEmailAddresses         bool
174                         AuthenticationRequestParameters map[string]string
175                 }
176                 OpenIDConnect struct {
177                         Enable                          bool
178                         Issuer                          string
179                         ClientID                        string
180                         ClientSecret                    string
181                         EmailClaim                      string
182                         EmailVerifiedClaim              string
183                         UsernameClaim                   string
184                         AcceptAccessToken               bool
185                         AcceptAccessTokenScope          string
186                         AuthenticationRequestParameters map[string]string
187                 }
188                 PAM struct {
189                         Enable             bool
190                         Service            string
191                         DefaultEmailDomain string
192                 }
193                 Test struct {
194                         Enable bool
195                         Users  map[string]TestUser
196                 }
197                 LoginCluster       string
198                 RemoteTokenRefresh Duration
199                 TokenLifetime      Duration
200                 TrustedClients     map[string]struct{}
201                 IssueTrustedTokens bool
202         }
203         Mail struct {
204                 MailchimpAPIKey                string
205                 MailchimpListID                string
206                 SendUserSetupNotificationEmail bool
207                 IssueReporterEmailFrom         string
208                 IssueReporterEmailTo           string
209                 SupportEmailAddress            string
210                 EmailFrom                      string
211         }
212         SystemLogs struct {
213                 LogLevel                string
214                 Format                  string
215                 MaxRequestLogParamsSize int
216         }
217         TLS struct {
218                 Certificate string
219                 Key         string
220                 Insecure    bool
221         }
222         Users struct {
223                 AnonymousUserToken                    string
224                 AdminNotifierEmailFrom                string
225                 AutoAdminFirstUser                    bool
226                 AutoAdminUserWithEmail                string
227                 AutoSetupNewUsers                     bool
228                 AutoSetupNewUsersWithRepository       bool
229                 AutoSetupNewUsersWithVmUUID           string
230                 AutoSetupUsernameBlacklist            StringSet
231                 EmailSubjectPrefix                    string
232                 NewInactiveUserNotificationRecipients StringSet
233                 NewUserNotificationRecipients         StringSet
234                 NewUsersAreActive                     bool
235                 UserNotifierEmailFrom                 string
236                 UserNotifierEmailBcc                  StringSet
237                 UserProfileNotificationAddress        string
238                 PreferDomainForUsername               string
239                 UserSetupMailText                     string
240         }
241         StorageClasses map[string]StorageClassConfig
242         Volumes        map[string]Volume
243         Workbench      struct {
244                 ActivationContactLink            string
245                 APIClientConnectTimeout          Duration
246                 APIClientReceiveTimeout          Duration
247                 APIResponseCompression           bool
248                 ApplicationMimetypesWithViewIcon StringSet
249                 ArvadosDocsite                   string
250                 ArvadosPublicDataDocURL          string
251                 DefaultOpenIdPrefix              string
252                 EnableGettingStartedPopup        bool
253                 EnablePublicProjectsPage         bool
254                 FileViewersConfigURL             string
255                 LogViewerMaxBytes                ByteSize
256                 MultiSiteSearch                  string
257                 ProfilingEnabled                 bool
258                 Repositories                     bool
259                 RepositoryCache                  string
260                 RunningJobLogRecordsToFetch      int
261                 SecretKeyBase                    string
262                 ShowRecentCollectionsOnDashboard bool
263                 ShowUserAgreementInline          bool
264                 ShowUserNotifications            bool
265                 SiteName                         string
266                 Theme                            string
267                 UserProfileFormFields            map[string]struct {
268                         Type                 string
269                         FormFieldTitle       string
270                         FormFieldDescription string
271                         Required             bool
272                         Position             int
273                         Options              map[string]struct{}
274                 }
275                 UserProfileFormMessage string
276                 VocabularyURL          string
277                 WelcomePageHTML        string
278                 InactivePageHTML       string
279                 SSHHelpPageHTML        string
280                 SSHHelpHostSuffix      string
281                 IdleTimeout            Duration
282         }
283 }
284
285 type StorageClassConfig struct {
286         Default  bool
287         Priority int
288 }
289
290 type Volume struct {
291         AccessViaHosts   map[URL]VolumeAccess
292         ReadOnly         bool
293         Replication      int
294         StorageClasses   map[string]bool
295         Driver           string
296         DriverParameters json.RawMessage
297 }
298
299 type S3VolumeDriverParameters struct {
300         IAMRole            string
301         AccessKeyID        string
302         SecretAccessKey    string
303         Endpoint           string
304         Region             string
305         Bucket             string
306         LocationConstraint bool
307         V2Signature        bool
308         UseAWSS3v2Driver   bool
309         IndexPageSize      int
310         ConnectTimeout     Duration
311         ReadTimeout        Duration
312         RaceWindow         Duration
313         UnsafeDelete       bool
314 }
315
316 type AzureVolumeDriverParameters struct {
317         StorageAccountName   string
318         StorageAccountKey    string
319         StorageBaseURL       string
320         ContainerName        string
321         RequestTimeout       Duration
322         ListBlobsRetryDelay  Duration
323         ListBlobsMaxAttempts int
324 }
325
326 type DirectoryVolumeDriverParameters struct {
327         Root      string
328         Serialize bool
329 }
330
331 type VolumeAccess struct {
332         ReadOnly bool
333 }
334
335 type Services struct {
336         Composer       Service
337         Controller     Service
338         DispatchCloud  Service
339         DispatchLSF    Service
340         GitHTTP        Service
341         GitSSH         Service
342         Health         Service
343         Keepbalance    Service
344         Keepproxy      Service
345         Keepstore      Service
346         RailsAPI       Service
347         WebDAVDownload Service
348         WebDAV         Service
349         WebShell       Service
350         Websocket      Service
351         Workbench1     Service
352         Workbench2     Service
353 }
354
355 type Service struct {
356         InternalURLs map[URL]ServiceInstance
357         ExternalURL  URL
358 }
359
360 type TestUser struct {
361         Email    string
362         Password string
363 }
364
365 // URL is a url.URL that is also usable as a JSON key/value.
366 type URL url.URL
367
368 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
369 // used as a JSON key/value.
370 func (su *URL) UnmarshalText(text []byte) error {
371         u, err := url.Parse(string(text))
372         if err == nil {
373                 *su = URL(*u)
374                 if su.Path == "" && su.Host != "" {
375                         // http://example really means http://example/
376                         su.Path = "/"
377                 }
378         }
379         return err
380 }
381
382 func (su URL) MarshalText() ([]byte, error) {
383         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
384 }
385
386 func (su URL) String() string {
387         return (*url.URL)(&su).String()
388 }
389
390 type ServiceInstance struct {
391         Rendezvous string `json:",omitempty"`
392 }
393
394 type PostgreSQL struct {
395         Connection     PostgreSQLConnection
396         ConnectionPool int
397 }
398
399 type PostgreSQLConnection map[string]string
400
401 type RemoteCluster struct {
402         Host          string
403         Proxy         bool
404         Scheme        string
405         Insecure      bool
406         ActivateUsers bool
407 }
408
409 type InstanceType struct {
410         Name            string
411         ProviderType    string
412         VCPUs           int
413         RAM             ByteSize
414         Scratch         ByteSize
415         IncludedScratch ByteSize
416         AddedScratch    ByteSize
417         Price           float64
418         Preemptible     bool
419 }
420
421 type ContainersConfig struct {
422         CloudVMs                    CloudVMsConfig
423         CrunchRunCommand            string
424         CrunchRunArgumentsList      []string
425         DefaultKeepCacheRAM         ByteSize
426         DispatchPrivateKey          string
427         LogReuseDecisions           bool
428         MaxComputeVMs               int
429         MaxDispatchAttempts         int
430         MaxRetryAttempts            int
431         MinRetryPeriod              Duration
432         ReserveExtraRAM             ByteSize
433         StaleLockTimeout            Duration
434         SupportedDockerImageFormats StringSet
435         UsePreemptibleInstances     bool
436         RuntimeEngine               string
437
438         JobsAPI struct {
439                 Enable         string
440                 GitInternalDir string
441         }
442         Logging struct {
443                 MaxAge                       Duration
444                 LogBytesPerEvent             int
445                 LogSecondsBetweenEvents      Duration
446                 LogThrottlePeriod            Duration
447                 LogThrottleBytes             int
448                 LogThrottleLines             int
449                 LimitLogBytesPerJob          int
450                 LogPartialLineThrottlePeriod Duration
451                 LogUpdatePeriod              Duration
452                 LogUpdateSize                ByteSize
453         }
454         ShellAccess struct {
455                 Admin bool
456                 User  bool
457         }
458         SLURM struct {
459                 PrioritySpread             int64
460                 SbatchArgumentsList        []string
461                 SbatchEnvironmentVariables map[string]string
462                 Managed                    struct {
463                         DNSServerConfDir       string
464                         DNSServerConfTemplate  string
465                         DNSServerReloadCommand string
466                         DNSServerUpdateCommand string
467                         ComputeNodeDomain      string
468                         ComputeNodeNameservers StringSet
469                         AssignNodeHostname     string
470                 }
471         }
472         LSF struct {
473                 BsubSudoUser      string
474                 BsubArgumentsList []string
475         }
476 }
477
478 type CloudVMsConfig struct {
479         Enable bool
480
481         BootProbeCommand               string
482         DeployRunnerBinary             string
483         ImageID                        string
484         MaxCloudOpsPerSecond           int
485         MaxProbesPerSecond             int
486         MaxConcurrentInstanceCreateOps int
487         PollInterval                   Duration
488         ProbeInterval                  Duration
489         SSHPort                        string
490         SyncInterval                   Duration
491         TimeoutBooting                 Duration
492         TimeoutIdle                    Duration
493         TimeoutProbe                   Duration
494         TimeoutShutdown                Duration
495         TimeoutSignal                  Duration
496         TimeoutStaleRunLock            Duration
497         TimeoutTERM                    Duration
498         ResourceTags                   map[string]string
499         TagKeyPrefix                   string
500
501         Driver           string
502         DriverParameters json.RawMessage
503 }
504
505 type InstanceTypeMap map[string]InstanceType
506
507 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
508
509 // UnmarshalJSON handles old config files that provide an array of
510 // instance types instead of a hash.
511 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
512         fixup := func(t InstanceType) (InstanceType, error) {
513                 if t.ProviderType == "" {
514                         t.ProviderType = t.Name
515                 }
516                 if t.Scratch == 0 {
517                         t.Scratch = t.IncludedScratch + t.AddedScratch
518                 } else if t.AddedScratch == 0 {
519                         t.AddedScratch = t.Scratch - t.IncludedScratch
520                 } else if t.IncludedScratch == 0 {
521                         t.IncludedScratch = t.Scratch - t.AddedScratch
522                 }
523
524                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
525                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
526                 }
527                 return t, nil
528         }
529
530         if len(data) > 0 && data[0] == '[' {
531                 var arr []InstanceType
532                 err := json.Unmarshal(data, &arr)
533                 if err != nil {
534                         return err
535                 }
536                 if len(arr) == 0 {
537                         *it = nil
538                         return nil
539                 }
540                 *it = make(map[string]InstanceType, len(arr))
541                 for _, t := range arr {
542                         if _, ok := (*it)[t.Name]; ok {
543                                 return errDuplicateInstanceTypeName
544                         }
545                         t, err := fixup(t)
546                         if err != nil {
547                                 return err
548                         }
549                         (*it)[t.Name] = t
550                 }
551                 return nil
552         }
553         var hash map[string]InstanceType
554         err := json.Unmarshal(data, &hash)
555         if err != nil {
556                 return err
557         }
558         // Fill in Name field (and ProviderType field, if not
559         // specified) using hash key.
560         *it = InstanceTypeMap(hash)
561         for name, t := range *it {
562                 t.Name = name
563                 t, err := fixup(t)
564                 if err != nil {
565                         return err
566                 }
567                 (*it)[name] = t
568         }
569         return nil
570 }
571
572 type StringSet map[string]struct{}
573
574 // UnmarshalJSON handles old config files that provide an array of
575 // instance types instead of a hash.
576 func (ss *StringSet) UnmarshalJSON(data []byte) error {
577         if len(data) > 0 && data[0] == '[' {
578                 var arr []string
579                 err := json.Unmarshal(data, &arr)
580                 if err != nil {
581                         return err
582                 }
583                 if len(arr) == 0 {
584                         *ss = nil
585                         return nil
586                 }
587                 *ss = make(map[string]struct{}, len(arr))
588                 for _, t := range arr {
589                         (*ss)[t] = struct{}{}
590                 }
591                 return nil
592         }
593         var hash map[string]struct{}
594         err := json.Unmarshal(data, &hash)
595         if err != nil {
596                 return err
597         }
598         *ss = make(map[string]struct{}, len(hash))
599         for t := range hash {
600                 (*ss)[t] = struct{}{}
601         }
602
603         return nil
604 }
605
606 type ServiceName string
607
608 const (
609         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
610         ServiceNameController    ServiceName = "arvados-controller"
611         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
612         ServiceNameDispatchLSF   ServiceName = "arvados-dispatch-lsf"
613         ServiceNameHealth        ServiceName = "arvados-health"
614         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
615         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
616         ServiceNameWebsocket     ServiceName = "arvados-ws"
617         ServiceNameKeepbalance   ServiceName = "keep-balance"
618         ServiceNameKeepweb       ServiceName = "keep-web"
619         ServiceNameKeepproxy     ServiceName = "keepproxy"
620         ServiceNameKeepstore     ServiceName = "keepstore"
621 )
622
623 // Map returns all services as a map, suitable for iterating over all
624 // services or looking up a service by name.
625 func (svcs Services) Map() map[ServiceName]Service {
626         return map[ServiceName]Service{
627                 ServiceNameRailsAPI:      svcs.RailsAPI,
628                 ServiceNameController:    svcs.Controller,
629                 ServiceNameDispatchCloud: svcs.DispatchCloud,
630                 ServiceNameDispatchLSF:   svcs.DispatchLSF,
631                 ServiceNameHealth:        svcs.Health,
632                 ServiceNameWorkbench1:    svcs.Workbench1,
633                 ServiceNameWorkbench2:    svcs.Workbench2,
634                 ServiceNameWebsocket:     svcs.Websocket,
635                 ServiceNameKeepbalance:   svcs.Keepbalance,
636                 ServiceNameKeepweb:       svcs.WebDAV,
637                 ServiceNameKeepproxy:     svcs.Keepproxy,
638                 ServiceNameKeepstore:     svcs.Keepstore,
639         }
640 }