Fix some golint warnings.
[arvados.git] / sdk / go / arvados / config.go
1 // Copyright (C) The Arvados Authors. All rights reserved.
2 //
3 // SPDX-License-Identifier: Apache-2.0
4
5 package arvados
6
7 import (
8         "encoding/json"
9         "errors"
10         "fmt"
11         "net/url"
12         "os"
13
14         "git.arvados.org/arvados.git/sdk/go/config"
15 )
16
17 var DefaultConfigFile = func() string {
18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
19                 return path
20         }
21         return "/etc/arvados/config.yml"
22 }()
23
24 type Config struct {
25         Clusters         map[string]Cluster
26         AutoReloadConfig bool
27 }
28
29 // GetConfig returns the current system config, loading it from
30 // configFile if needed.
31 func GetConfig(configFile string) (*Config, error) {
32         var cfg Config
33         err := config.LoadFile(&cfg, configFile)
34         return &cfg, err
35 }
36
37 // GetCluster returns the cluster ID and config for the given
38 // cluster, or the default/only configured cluster if clusterID is "".
39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
40         if clusterID == "" {
41                 if len(sc.Clusters) == 0 {
42                         return nil, fmt.Errorf("no clusters configured")
43                 } else if len(sc.Clusters) > 1 {
44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
45                 } else {
46                         for id, cc := range sc.Clusters {
47                                 cc.ClusterID = id
48                                 return &cc, nil
49                         }
50                 }
51         }
52         if cc, ok := sc.Clusters[clusterID]; !ok {
53                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
54         } else {
55                 cc.ClusterID = clusterID
56                 return &cc, nil
57         }
58 }
59
60 type WebDAVCacheConfig struct {
61         TTL                  Duration
62         UUIDTTL              Duration
63         MaxBlockEntries      int
64         MaxCollectionEntries int
65         MaxCollectionBytes   int64
66         MaxPermissionEntries int
67         MaxUUIDEntries       int
68 }
69
70 type Cluster struct {
71         ClusterID       string `json:"-"`
72         ManagementToken string
73         SystemRootToken string
74         Services        Services
75         InstanceTypes   InstanceTypeMap
76         Containers      ContainersConfig
77         RemoteClusters  map[string]RemoteCluster
78         PostgreSQL      PostgreSQL
79
80         API struct {
81                 AsyncPermissionsUpdateInterval Duration
82                 DisabledAPIs                   StringSet
83                 MaxIndexDatabaseRead           int
84                 MaxItemsPerResponse            int
85                 MaxConcurrentRequests          int
86                 MaxKeepBlobBuffers             int
87                 MaxRequestAmplification        int
88                 MaxRequestSize                 int
89                 RailsSessionSecretToken        string
90                 RequestTimeout                 Duration
91                 SendTimeout                    Duration
92                 WebsocketClientEventQueue      int
93                 WebsocketServerEventQueue      int
94                 KeepServiceRequestTimeout      Duration
95         }
96         AuditLogs struct {
97                 MaxAge             Duration
98                 MaxDeleteBatch     int
99                 UnloggedAttributes StringSet
100         }
101         Collections struct {
102                 BlobSigning              bool
103                 BlobSigningKey           string
104                 BlobSigningTTL           Duration
105                 BlobTrash                bool
106                 BlobTrashLifetime        Duration
107                 BlobTrashCheckInterval   Duration
108                 BlobTrashConcurrency     int
109                 BlobDeleteConcurrency    int
110                 BlobReplicateConcurrency int
111                 CollectionVersioning     bool
112                 DefaultTrashLifetime     Duration
113                 DefaultReplication       int
114                 ManagedProperties        map[string]struct {
115                         Value     interface{}
116                         Function  string
117                         Protected bool
118                 }
119                 PreserveVersionIfIdle        Duration
120                 TrashSweepInterval           Duration
121                 TrustAllContent              bool
122                 ForwardSlashNameSubstitution string
123                 S3FolderObjects              bool
124
125                 BlobMissingReport        string
126                 BalancePeriod            Duration
127                 BalanceCollectionBatch   int
128                 BalanceCollectionBuffers int
129                 BalanceTimeout           Duration
130
131                 WebDAVCache WebDAVCacheConfig
132         }
133         Git struct {
134                 GitCommand   string
135                 GitoliteHome string
136                 Repositories string
137         }
138         Login struct {
139                 LDAP struct {
140                         Enable             bool
141                         URL                URL
142                         StartTLS           bool
143                         InsecureTLS        bool
144                         StripDomain        string
145                         AppendDomain       string
146                         SearchAttribute    string
147                         SearchBindUser     string
148                         SearchBindPassword string
149                         SearchBase         string
150                         SearchFilters      string
151                         EmailAttribute     string
152                         UsernameAttribute  string
153                 }
154                 Google struct {
155                         Enable                  bool
156                         ClientID                string
157                         ClientSecret            string
158                         AlternateEmailAddresses bool
159                 }
160                 OpenIDConnect struct {
161                         Enable             bool
162                         Issuer             string
163                         ClientID           string
164                         ClientSecret       string
165                         EmailClaim         string
166                         EmailVerifiedClaim string
167                         UsernameClaim      string
168                 }
169                 PAM struct {
170                         Enable             bool
171                         Service            string
172                         DefaultEmailDomain string
173                 }
174                 SSO struct {
175                         Enable            bool
176                         ProviderAppID     string
177                         ProviderAppSecret string
178                 }
179                 Test struct {
180                         Enable bool
181                         Users  map[string]TestUser
182                 }
183                 LoginCluster       string
184                 RemoteTokenRefresh Duration
185                 TokenLifetime      Duration
186         }
187         Mail struct {
188                 MailchimpAPIKey                string
189                 MailchimpListID                string
190                 SendUserSetupNotificationEmail bool
191                 IssueReporterEmailFrom         string
192                 IssueReporterEmailTo           string
193                 SupportEmailAddress            string
194                 EmailFrom                      string
195         }
196         SystemLogs struct {
197                 LogLevel                string
198                 Format                  string
199                 MaxRequestLogParamsSize int
200         }
201         TLS struct {
202                 Certificate string
203                 Key         string
204                 Insecure    bool
205         }
206         Users struct {
207                 AnonymousUserToken                    string
208                 AdminNotifierEmailFrom                string
209                 AutoAdminFirstUser                    bool
210                 AutoAdminUserWithEmail                string
211                 AutoSetupNewUsers                     bool
212                 AutoSetupNewUsersWithRepository       bool
213                 AutoSetupNewUsersWithVmUUID           string
214                 AutoSetupUsernameBlacklist            StringSet
215                 EmailSubjectPrefix                    string
216                 NewInactiveUserNotificationRecipients StringSet
217                 NewUserNotificationRecipients         StringSet
218                 NewUsersAreActive                     bool
219                 UserNotifierEmailFrom                 string
220                 UserProfileNotificationAddress        string
221                 PreferDomainForUsername               string
222         }
223         Volumes   map[string]Volume
224         Workbench struct {
225                 ActivationContactLink            string
226                 APIClientConnectTimeout          Duration
227                 APIClientReceiveTimeout          Duration
228                 APIResponseCompression           bool
229                 ApplicationMimetypesWithViewIcon StringSet
230                 ArvadosDocsite                   string
231                 ArvadosPublicDataDocURL          string
232                 DefaultOpenIdPrefix              string
233                 EnableGettingStartedPopup        bool
234                 EnablePublicProjectsPage         bool
235                 FileViewersConfigURL             string
236                 LogViewerMaxBytes                ByteSize
237                 MultiSiteSearch                  string
238                 ProfilingEnabled                 bool
239                 Repositories                     bool
240                 RepositoryCache                  string
241                 RunningJobLogRecordsToFetch      int
242                 SecretKeyBase                    string
243                 ShowRecentCollectionsOnDashboard bool
244                 ShowUserAgreementInline          bool
245                 ShowUserNotifications            bool
246                 SiteName                         string
247                 Theme                            string
248                 UserProfileFormFields            map[string]struct {
249                         Type                 string
250                         FormFieldTitle       string
251                         FormFieldDescription string
252                         Required             bool
253                         Position             int
254                         Options              map[string]struct{}
255                 }
256                 UserProfileFormMessage string
257                 VocabularyURL          string
258                 WelcomePageHTML        string
259                 InactivePageHTML       string
260                 SSHHelpPageHTML        string
261                 SSHHelpHostSuffix      string
262         }
263
264         ForceLegacyAPI14 bool
265 }
266
267 type Volume struct {
268         AccessViaHosts   map[URL]VolumeAccess
269         ReadOnly         bool
270         Replication      int
271         StorageClasses   map[string]bool
272         Driver           string
273         DriverParameters json.RawMessage
274 }
275
276 type S3VolumeDriverParameters struct {
277         IAMRole            string
278         AccessKey          string
279         SecretKey          string
280         Endpoint           string
281         Region             string
282         Bucket             string
283         LocationConstraint bool
284         V2Signature        bool
285         UseAWSS3v2Driver   bool
286         IndexPageSize      int
287         ConnectTimeout     Duration
288         ReadTimeout        Duration
289         RaceWindow         Duration
290         UnsafeDelete       bool
291 }
292
293 type AzureVolumeDriverParameters struct {
294         StorageAccountName   string
295         StorageAccountKey    string
296         StorageBaseURL       string
297         ContainerName        string
298         RequestTimeout       Duration
299         ListBlobsRetryDelay  Duration
300         ListBlobsMaxAttempts int
301 }
302
303 type DirectoryVolumeDriverParameters struct {
304         Root      string
305         Serialize bool
306 }
307
308 type VolumeAccess struct {
309         ReadOnly bool
310 }
311
312 type Services struct {
313         Composer       Service
314         Controller     Service
315         DispatchCloud  Service
316         GitHTTP        Service
317         GitSSH         Service
318         Health         Service
319         Keepbalance    Service
320         Keepproxy      Service
321         Keepstore      Service
322         RailsAPI       Service
323         SSO            Service
324         WebDAVDownload Service
325         WebDAV         Service
326         WebShell       Service
327         Websocket      Service
328         Workbench1     Service
329         Workbench2     Service
330 }
331
332 type Service struct {
333         InternalURLs map[URL]ServiceInstance
334         ExternalURL  URL
335 }
336
337 type TestUser struct {
338         Email    string
339         Password string
340 }
341
342 // URL is a url.URL that is also usable as a JSON key/value.
343 type URL url.URL
344
345 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
346 // used as a JSON key/value.
347 func (su *URL) UnmarshalText(text []byte) error {
348         u, err := url.Parse(string(text))
349         if err == nil {
350                 *su = URL(*u)
351                 if su.Path == "" && su.Host != "" {
352                         // http://example really means http://example/
353                         su.Path = "/"
354                 }
355         }
356         return err
357 }
358
359 func (su URL) MarshalText() ([]byte, error) {
360         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
361 }
362
363 func (su URL) String() string {
364         return (*url.URL)(&su).String()
365 }
366
367 type ServiceInstance struct {
368         Rendezvous string `json:",omitempty"`
369 }
370
371 type PostgreSQL struct {
372         Connection     PostgreSQLConnection
373         ConnectionPool int
374 }
375
376 type PostgreSQLConnection map[string]string
377
378 type RemoteCluster struct {
379         Host          string
380         Proxy         bool
381         Scheme        string
382         Insecure      bool
383         ActivateUsers bool
384 }
385
386 type InstanceType struct {
387         Name            string
388         ProviderType    string
389         VCPUs           int
390         RAM             ByteSize
391         Scratch         ByteSize
392         IncludedScratch ByteSize
393         AddedScratch    ByteSize
394         Price           float64
395         Preemptible     bool
396 }
397
398 type ContainersConfig struct {
399         CloudVMs                    CloudVMsConfig
400         CrunchRunCommand            string
401         CrunchRunArgumentsList      []string
402         DefaultKeepCacheRAM         ByteSize
403         DispatchPrivateKey          string
404         LogReuseDecisions           bool
405         MaxComputeVMs               int
406         MaxDispatchAttempts         int
407         MaxRetryAttempts            int
408         MinRetryPeriod              Duration
409         ReserveExtraRAM             ByteSize
410         StaleLockTimeout            Duration
411         SupportedDockerImageFormats StringSet
412         UsePreemptibleInstances     bool
413
414         JobsAPI struct {
415                 Enable         string
416                 GitInternalDir string
417         }
418         Logging struct {
419                 MaxAge                       Duration
420                 LogBytesPerEvent             int
421                 LogSecondsBetweenEvents      Duration
422                 LogThrottlePeriod            Duration
423                 LogThrottleBytes             int
424                 LogThrottleLines             int
425                 LimitLogBytesPerJob          int
426                 LogPartialLineThrottlePeriod Duration
427                 LogUpdatePeriod              Duration
428                 LogUpdateSize                ByteSize
429         }
430         SLURM struct {
431                 PrioritySpread             int64
432                 SbatchArgumentsList        []string
433                 SbatchEnvironmentVariables map[string]string
434                 Managed                    struct {
435                         DNSServerConfDir       string
436                         DNSServerConfTemplate  string
437                         DNSServerReloadCommand string
438                         DNSServerUpdateCommand string
439                         ComputeNodeDomain      string
440                         ComputeNodeNameservers StringSet
441                         AssignNodeHostname     string
442                 }
443         }
444 }
445
446 type CloudVMsConfig struct {
447         Enable bool
448
449         BootProbeCommand     string
450         DeployRunnerBinary   string
451         ImageID              string
452         MaxCloudOpsPerSecond int
453         MaxProbesPerSecond   int
454         PollInterval         Duration
455         ProbeInterval        Duration
456         SSHPort              string
457         SyncInterval         Duration
458         TimeoutBooting       Duration
459         TimeoutIdle          Duration
460         TimeoutProbe         Duration
461         TimeoutShutdown      Duration
462         TimeoutSignal        Duration
463         TimeoutTERM          Duration
464         ResourceTags         map[string]string
465         TagKeyPrefix         string
466
467         Driver           string
468         DriverParameters json.RawMessage
469 }
470
471 type InstanceTypeMap map[string]InstanceType
472
473 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
474
475 // UnmarshalJSON handles old config files that provide an array of
476 // instance types instead of a hash.
477 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
478         fixup := func(t InstanceType) (InstanceType, error) {
479                 if t.ProviderType == "" {
480                         t.ProviderType = t.Name
481                 }
482                 if t.Scratch == 0 {
483                         t.Scratch = t.IncludedScratch + t.AddedScratch
484                 } else if t.AddedScratch == 0 {
485                         t.AddedScratch = t.Scratch - t.IncludedScratch
486                 } else if t.IncludedScratch == 0 {
487                         t.IncludedScratch = t.Scratch - t.AddedScratch
488                 }
489
490                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
491                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
492                 }
493                 return t, nil
494         }
495
496         if len(data) > 0 && data[0] == '[' {
497                 var arr []InstanceType
498                 err := json.Unmarshal(data, &arr)
499                 if err != nil {
500                         return err
501                 }
502                 if len(arr) == 0 {
503                         *it = nil
504                         return nil
505                 }
506                 *it = make(map[string]InstanceType, len(arr))
507                 for _, t := range arr {
508                         if _, ok := (*it)[t.Name]; ok {
509                                 return errDuplicateInstanceTypeName
510                         }
511                         t, err := fixup(t)
512                         if err != nil {
513                                 return err
514                         }
515                         (*it)[t.Name] = t
516                 }
517                 return nil
518         }
519         var hash map[string]InstanceType
520         err := json.Unmarshal(data, &hash)
521         if err != nil {
522                 return err
523         }
524         // Fill in Name field (and ProviderType field, if not
525         // specified) using hash key.
526         *it = InstanceTypeMap(hash)
527         for name, t := range *it {
528                 t.Name = name
529                 t, err := fixup(t)
530                 if err != nil {
531                         return err
532                 }
533                 (*it)[name] = t
534         }
535         return nil
536 }
537
538 type StringSet map[string]struct{}
539
540 // UnmarshalJSON handles old config files that provide an array of
541 // instance types instead of a hash.
542 func (ss *StringSet) UnmarshalJSON(data []byte) error {
543         if len(data) > 0 && data[0] == '[' {
544                 var arr []string
545                 err := json.Unmarshal(data, &arr)
546                 if err != nil {
547                         return err
548                 }
549                 if len(arr) == 0 {
550                         *ss = nil
551                         return nil
552                 }
553                 *ss = make(map[string]struct{}, len(arr))
554                 for _, t := range arr {
555                         (*ss)[t] = struct{}{}
556                 }
557                 return nil
558         }
559         var hash map[string]struct{}
560         err := json.Unmarshal(data, &hash)
561         if err != nil {
562                 return err
563         }
564         *ss = make(map[string]struct{}, len(hash))
565         for t := range hash {
566                 (*ss)[t] = struct{}{}
567         }
568
569         return nil
570 }
571
572 type ServiceName string
573
574 const (
575         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
576         ServiceNameController    ServiceName = "arvados-controller"
577         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
578         ServiceNameHealth        ServiceName = "arvados-health"
579         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
580         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
581         ServiceNameWebsocket     ServiceName = "arvados-ws"
582         ServiceNameKeepbalance   ServiceName = "keep-balance"
583         ServiceNameKeepweb       ServiceName = "keep-web"
584         ServiceNameKeepproxy     ServiceName = "keepproxy"
585         ServiceNameKeepstore     ServiceName = "keepstore"
586 )
587
588 // Map returns all services as a map, suitable for iterating over all
589 // services or looking up a service by name.
590 func (svcs Services) Map() map[ServiceName]Service {
591         return map[ServiceName]Service{
592                 ServiceNameRailsAPI:      svcs.RailsAPI,
593                 ServiceNameController:    svcs.Controller,
594                 ServiceNameDispatchCloud: svcs.DispatchCloud,
595                 ServiceNameHealth:        svcs.Health,
596                 ServiceNameWorkbench1:    svcs.Workbench1,
597                 ServiceNameWorkbench2:    svcs.Workbench2,
598                 ServiceNameWebsocket:     svcs.Websocket,
599                 ServiceNameKeepbalance:   svcs.Keepbalance,
600                 ServiceNameKeepweb:       svcs.WebDAV,
601                 ServiceNameKeepproxy:     svcs.Keepproxy,
602                 ServiceNameKeepstore:     svcs.Keepstore,
603         }
604 }