Merge branch '16580-remove-python2-packages'
[arvados.git] / sdk / go / arvados / config.go
1 // Copyright (C) The Arvados Authors. All rights reserved.
2 //
3 // SPDX-License-Identifier: Apache-2.0
4
5 package arvados
6
7 import (
8         "encoding/json"
9         "errors"
10         "fmt"
11         "net/url"
12         "os"
13
14         "git.arvados.org/arvados.git/sdk/go/config"
15 )
16
17 var DefaultConfigFile = func() string {
18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
19                 return path
20         } else {
21                 return "/etc/arvados/config.yml"
22         }
23 }()
24
25 type Config struct {
26         Clusters         map[string]Cluster
27         AutoReloadConfig bool
28 }
29
30 // GetConfig returns the current system config, loading it from
31 // configFile if needed.
32 func GetConfig(configFile string) (*Config, error) {
33         var cfg Config
34         err := config.LoadFile(&cfg, configFile)
35         return &cfg, err
36 }
37
38 // GetCluster returns the cluster ID and config for the given
39 // cluster, or the default/only configured cluster if clusterID is "".
40 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
41         if clusterID == "" {
42                 if len(sc.Clusters) == 0 {
43                         return nil, fmt.Errorf("no clusters configured")
44                 } else if len(sc.Clusters) > 1 {
45                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
46                 } else {
47                         for id, cc := range sc.Clusters {
48                                 cc.ClusterID = id
49                                 return &cc, nil
50                         }
51                 }
52         }
53         if cc, ok := sc.Clusters[clusterID]; !ok {
54                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
55         } else {
56                 cc.ClusterID = clusterID
57                 return &cc, nil
58         }
59 }
60
61 type WebDAVCacheConfig struct {
62         TTL                  Duration
63         UUIDTTL              Duration
64         MaxBlockEntries      int
65         MaxCollectionEntries int
66         MaxCollectionBytes   int64
67         MaxPermissionEntries int
68         MaxUUIDEntries       int
69 }
70
71 type Cluster struct {
72         ClusterID       string `json:"-"`
73         ManagementToken string
74         SystemRootToken string
75         Services        Services
76         InstanceTypes   InstanceTypeMap
77         Containers      ContainersConfig
78         RemoteClusters  map[string]RemoteCluster
79         PostgreSQL      PostgreSQL
80
81         API struct {
82                 AsyncPermissionsUpdateInterval Duration
83                 DisabledAPIs                   StringSet
84                 MaxIndexDatabaseRead           int
85                 MaxItemsPerResponse            int
86                 MaxConcurrentRequests          int
87                 MaxKeepBlobBuffers             int
88                 MaxRequestAmplification        int
89                 MaxRequestSize                 int
90                 RailsSessionSecretToken        string
91                 RequestTimeout                 Duration
92                 SendTimeout                    Duration
93                 WebsocketClientEventQueue      int
94                 WebsocketServerEventQueue      int
95                 KeepServiceRequestTimeout      Duration
96         }
97         AuditLogs struct {
98                 MaxAge             Duration
99                 MaxDeleteBatch     int
100                 UnloggedAttributes StringSet
101         }
102         Collections struct {
103                 BlobSigning              bool
104                 BlobSigningKey           string
105                 BlobSigningTTL           Duration
106                 BlobTrash                bool
107                 BlobTrashLifetime        Duration
108                 BlobTrashCheckInterval   Duration
109                 BlobTrashConcurrency     int
110                 BlobDeleteConcurrency    int
111                 BlobReplicateConcurrency int
112                 CollectionVersioning     bool
113                 DefaultTrashLifetime     Duration
114                 DefaultReplication       int
115                 ManagedProperties        map[string]struct {
116                         Value     interface{}
117                         Function  string
118                         Protected bool
119                 }
120                 PreserveVersionIfIdle        Duration
121                 TrashSweepInterval           Duration
122                 TrustAllContent              bool
123                 ForwardSlashNameSubstitution string
124
125                 BlobMissingReport        string
126                 BalancePeriod            Duration
127                 BalanceCollectionBatch   int
128                 BalanceCollectionBuffers int
129                 BalanceTimeout           Duration
130
131                 WebDAVCache WebDAVCacheConfig
132         }
133         Git struct {
134                 GitCommand   string
135                 GitoliteHome string
136                 Repositories string
137         }
138         Login struct {
139                 LDAP struct {
140                         Enable             bool
141                         URL                URL
142                         StartTLS           bool
143                         InsecureTLS        bool
144                         StripDomain        string
145                         AppendDomain       string
146                         SearchAttribute    string
147                         SearchBindUser     string
148                         SearchBindPassword string
149                         SearchBase         string
150                         SearchFilters      string
151                         EmailAttribute     string
152                         UsernameAttribute  string
153                 }
154                 Google struct {
155                         Enable                  bool
156                         ClientID                string
157                         ClientSecret            string
158                         AlternateEmailAddresses bool
159                 }
160                 OpenIDConnect struct {
161                         Enable             bool
162                         Issuer             string
163                         ClientID           string
164                         ClientSecret       string
165                         EmailClaim         string
166                         EmailVerifiedClaim string
167                         UsernameClaim      string
168                 }
169                 PAM struct {
170                         Enable             bool
171                         Service            string
172                         DefaultEmailDomain string
173                 }
174                 SSO struct {
175                         Enable            bool
176                         ProviderAppID     string
177                         ProviderAppSecret string
178                 }
179                 LoginCluster       string
180                 RemoteTokenRefresh Duration
181         }
182         Mail struct {
183                 MailchimpAPIKey                string
184                 MailchimpListID                string
185                 SendUserSetupNotificationEmail bool
186                 IssueReporterEmailFrom         string
187                 IssueReporterEmailTo           string
188                 SupportEmailAddress            string
189                 EmailFrom                      string
190         }
191         SystemLogs struct {
192                 LogLevel                string
193                 Format                  string
194                 MaxRequestLogParamsSize int
195         }
196         TLS struct {
197                 Certificate string
198                 Key         string
199                 Insecure    bool
200         }
201         Users struct {
202                 AnonymousUserToken                    string
203                 AdminNotifierEmailFrom                string
204                 AutoAdminFirstUser                    bool
205                 AutoAdminUserWithEmail                string
206                 AutoSetupNewUsers                     bool
207                 AutoSetupNewUsersWithRepository       bool
208                 AutoSetupNewUsersWithVmUUID           string
209                 AutoSetupUsernameBlacklist            StringSet
210                 EmailSubjectPrefix                    string
211                 NewInactiveUserNotificationRecipients StringSet
212                 NewUserNotificationRecipients         StringSet
213                 NewUsersAreActive                     bool
214                 UserNotifierEmailFrom                 string
215                 UserProfileNotificationAddress        string
216                 PreferDomainForUsername               string
217         }
218         Volumes   map[string]Volume
219         Workbench struct {
220                 ActivationContactLink            string
221                 APIClientConnectTimeout          Duration
222                 APIClientReceiveTimeout          Duration
223                 APIResponseCompression           bool
224                 ApplicationMimetypesWithViewIcon StringSet
225                 ArvadosDocsite                   string
226                 ArvadosPublicDataDocURL          string
227                 DefaultOpenIdPrefix              string
228                 EnableGettingStartedPopup        bool
229                 EnablePublicProjectsPage         bool
230                 FileViewersConfigURL             string
231                 LogViewerMaxBytes                ByteSize
232                 MultiSiteSearch                  string
233                 ProfilingEnabled                 bool
234                 Repositories                     bool
235                 RepositoryCache                  string
236                 RunningJobLogRecordsToFetch      int
237                 SecretKeyBase                    string
238                 ShowRecentCollectionsOnDashboard bool
239                 ShowUserAgreementInline          bool
240                 ShowUserNotifications            bool
241                 SiteName                         string
242                 Theme                            string
243                 UserProfileFormFields            map[string]struct {
244                         Type                 string
245                         FormFieldTitle       string
246                         FormFieldDescription string
247                         Required             bool
248                         Position             int
249                         Options              map[string]struct{}
250                 }
251                 UserProfileFormMessage string
252                 VocabularyURL          string
253                 WelcomePageHTML        string
254                 InactivePageHTML       string
255                 SSHHelpPageHTML        string
256                 SSHHelpHostSuffix      string
257         }
258
259         ForceLegacyAPI14 bool
260 }
261
262 type Volume struct {
263         AccessViaHosts   map[URL]VolumeAccess
264         ReadOnly         bool
265         Replication      int
266         StorageClasses   map[string]bool
267         Driver           string
268         DriverParameters json.RawMessage
269 }
270
271 type S3VolumeDriverParameters struct {
272         IAMRole            string
273         AccessKey          string
274         SecretKey          string
275         Endpoint           string
276         Region             string
277         Bucket             string
278         LocationConstraint bool
279         V2Signature        bool
280         UseAWSS3v2Driver   bool
281         IndexPageSize      int
282         ConnectTimeout     Duration
283         ReadTimeout        Duration
284         RaceWindow         Duration
285         UnsafeDelete       bool
286 }
287
288 type AzureVolumeDriverParameters struct {
289         StorageAccountName   string
290         StorageAccountKey    string
291         StorageBaseURL       string
292         ContainerName        string
293         RequestTimeout       Duration
294         ListBlobsRetryDelay  Duration
295         ListBlobsMaxAttempts int
296 }
297
298 type DirectoryVolumeDriverParameters struct {
299         Root      string
300         Serialize bool
301 }
302
303 type VolumeAccess struct {
304         ReadOnly bool
305 }
306
307 type Services struct {
308         Composer       Service
309         Controller     Service
310         DispatchCloud  Service
311         GitHTTP        Service
312         GitSSH         Service
313         Health         Service
314         Keepbalance    Service
315         Keepproxy      Service
316         Keepstore      Service
317         RailsAPI       Service
318         SSO            Service
319         WebDAVDownload Service
320         WebDAV         Service
321         WebShell       Service
322         Websocket      Service
323         Workbench1     Service
324         Workbench2     Service
325 }
326
327 type Service struct {
328         InternalURLs map[URL]ServiceInstance
329         ExternalURL  URL
330 }
331
332 // URL is a url.URL that is also usable as a JSON key/value.
333 type URL url.URL
334
335 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
336 // used as a JSON key/value.
337 func (su *URL) UnmarshalText(text []byte) error {
338         u, err := url.Parse(string(text))
339         if err == nil {
340                 *su = URL(*u)
341                 if su.Path == "" && su.Host != "" {
342                         // http://example really means http://example/
343                         su.Path = "/"
344                 }
345         }
346         return err
347 }
348
349 func (su URL) MarshalText() ([]byte, error) {
350         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
351 }
352
353 func (su URL) String() string {
354         return (*url.URL)(&su).String()
355 }
356
357 type ServiceInstance struct {
358         Rendezvous string `json:",omitempty"`
359 }
360
361 type PostgreSQL struct {
362         Connection     PostgreSQLConnection
363         ConnectionPool int
364 }
365
366 type PostgreSQLConnection map[string]string
367
368 type RemoteCluster struct {
369         Host          string
370         Proxy         bool
371         Scheme        string
372         Insecure      bool
373         ActivateUsers bool
374 }
375
376 type InstanceType struct {
377         Name            string
378         ProviderType    string
379         VCPUs           int
380         RAM             ByteSize
381         Scratch         ByteSize
382         IncludedScratch ByteSize
383         AddedScratch    ByteSize
384         Price           float64
385         Preemptible     bool
386 }
387
388 type ContainersConfig struct {
389         CloudVMs                    CloudVMsConfig
390         CrunchRunCommand            string
391         CrunchRunArgumentsList      []string
392         DefaultKeepCacheRAM         ByteSize
393         DispatchPrivateKey          string
394         LogReuseDecisions           bool
395         MaxComputeVMs               int
396         MaxDispatchAttempts         int
397         MaxRetryAttempts            int
398         MinRetryPeriod              Duration
399         ReserveExtraRAM             ByteSize
400         StaleLockTimeout            Duration
401         SupportedDockerImageFormats StringSet
402         UsePreemptibleInstances     bool
403
404         JobsAPI struct {
405                 Enable         string
406                 GitInternalDir string
407         }
408         Logging struct {
409                 MaxAge                       Duration
410                 LogBytesPerEvent             int
411                 LogSecondsBetweenEvents      Duration
412                 LogThrottlePeriod            Duration
413                 LogThrottleBytes             int
414                 LogThrottleLines             int
415                 LimitLogBytesPerJob          int
416                 LogPartialLineThrottlePeriod Duration
417                 LogUpdatePeriod              Duration
418                 LogUpdateSize                ByteSize
419         }
420         SLURM struct {
421                 PrioritySpread             int64
422                 SbatchArgumentsList        []string
423                 SbatchEnvironmentVariables map[string]string
424                 Managed                    struct {
425                         DNSServerConfDir       string
426                         DNSServerConfTemplate  string
427                         DNSServerReloadCommand string
428                         DNSServerUpdateCommand string
429                         ComputeNodeDomain      string
430                         ComputeNodeNameservers StringSet
431                         AssignNodeHostname     string
432                 }
433         }
434 }
435
436 type CloudVMsConfig struct {
437         Enable bool
438
439         BootProbeCommand     string
440         DeployRunnerBinary   string
441         ImageID              string
442         MaxCloudOpsPerSecond int
443         MaxProbesPerSecond   int
444         PollInterval         Duration
445         ProbeInterval        Duration
446         SSHPort              string
447         SyncInterval         Duration
448         TimeoutBooting       Duration
449         TimeoutIdle          Duration
450         TimeoutProbe         Duration
451         TimeoutShutdown      Duration
452         TimeoutSignal        Duration
453         TimeoutTERM          Duration
454         ResourceTags         map[string]string
455         TagKeyPrefix         string
456
457         Driver           string
458         DriverParameters json.RawMessage
459 }
460
461 type InstanceTypeMap map[string]InstanceType
462
463 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
464
465 // UnmarshalJSON handles old config files that provide an array of
466 // instance types instead of a hash.
467 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
468         fixup := func(t InstanceType) (InstanceType, error) {
469                 if t.ProviderType == "" {
470                         t.ProviderType = t.Name
471                 }
472                 if t.Scratch == 0 {
473                         t.Scratch = t.IncludedScratch + t.AddedScratch
474                 } else if t.AddedScratch == 0 {
475                         t.AddedScratch = t.Scratch - t.IncludedScratch
476                 } else if t.IncludedScratch == 0 {
477                         t.IncludedScratch = t.Scratch - t.AddedScratch
478                 }
479
480                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
481                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
482                 }
483                 return t, nil
484         }
485
486         if len(data) > 0 && data[0] == '[' {
487                 var arr []InstanceType
488                 err := json.Unmarshal(data, &arr)
489                 if err != nil {
490                         return err
491                 }
492                 if len(arr) == 0 {
493                         *it = nil
494                         return nil
495                 }
496                 *it = make(map[string]InstanceType, len(arr))
497                 for _, t := range arr {
498                         if _, ok := (*it)[t.Name]; ok {
499                                 return errDuplicateInstanceTypeName
500                         }
501                         t, err := fixup(t)
502                         if err != nil {
503                                 return err
504                         }
505                         (*it)[t.Name] = t
506                 }
507                 return nil
508         }
509         var hash map[string]InstanceType
510         err := json.Unmarshal(data, &hash)
511         if err != nil {
512                 return err
513         }
514         // Fill in Name field (and ProviderType field, if not
515         // specified) using hash key.
516         *it = InstanceTypeMap(hash)
517         for name, t := range *it {
518                 t.Name = name
519                 t, err := fixup(t)
520                 if err != nil {
521                         return err
522                 }
523                 (*it)[name] = t
524         }
525         return nil
526 }
527
528 type StringSet map[string]struct{}
529
530 // UnmarshalJSON handles old config files that provide an array of
531 // instance types instead of a hash.
532 func (ss *StringSet) UnmarshalJSON(data []byte) error {
533         if len(data) > 0 && data[0] == '[' {
534                 var arr []string
535                 err := json.Unmarshal(data, &arr)
536                 if err != nil {
537                         return err
538                 }
539                 if len(arr) == 0 {
540                         *ss = nil
541                         return nil
542                 }
543                 *ss = make(map[string]struct{}, len(arr))
544                 for _, t := range arr {
545                         (*ss)[t] = struct{}{}
546                 }
547                 return nil
548         }
549         var hash map[string]struct{}
550         err := json.Unmarshal(data, &hash)
551         if err != nil {
552                 return err
553         }
554         *ss = make(map[string]struct{}, len(hash))
555         for t := range hash {
556                 (*ss)[t] = struct{}{}
557         }
558
559         return nil
560 }
561
562 type ServiceName string
563
564 const (
565         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
566         ServiceNameController    ServiceName = "arvados-controller"
567         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
568         ServiceNameHealth        ServiceName = "arvados-health"
569         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
570         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
571         ServiceNameWebsocket     ServiceName = "arvados-ws"
572         ServiceNameKeepbalance   ServiceName = "keep-balance"
573         ServiceNameKeepweb       ServiceName = "keep-web"
574         ServiceNameKeepproxy     ServiceName = "keepproxy"
575         ServiceNameKeepstore     ServiceName = "keepstore"
576 )
577
578 // Map returns all services as a map, suitable for iterating over all
579 // services or looking up a service by name.
580 func (svcs Services) Map() map[ServiceName]Service {
581         return map[ServiceName]Service{
582                 ServiceNameRailsAPI:      svcs.RailsAPI,
583                 ServiceNameController:    svcs.Controller,
584                 ServiceNameDispatchCloud: svcs.DispatchCloud,
585                 ServiceNameHealth:        svcs.Health,
586                 ServiceNameWorkbench1:    svcs.Workbench1,
587                 ServiceNameWorkbench2:    svcs.Workbench2,
588                 ServiceNameWebsocket:     svcs.Websocket,
589                 ServiceNameKeepbalance:   svcs.Keepbalance,
590                 ServiceNameKeepweb:       svcs.WebDAV,
591                 ServiceNameKeepproxy:     svcs.Keepproxy,
592                 ServiceNameKeepstore:     svcs.Keepstore,
593         }
594 }