16347: Run a dedicated keepstore process for each container.
[arvados.git] / sdk / go / arvados / config.go
1 // Copyright (C) The Arvados Authors. All rights reserved.
2 //
3 // SPDX-License-Identifier: Apache-2.0
4
5 package arvados
6
7 import (
8         "encoding/json"
9         "errors"
10         "fmt"
11         "net/url"
12         "os"
13
14         "git.arvados.org/arvados.git/sdk/go/config"
15 )
16
17 var DefaultConfigFile = func() string {
18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
19                 return path
20         }
21         return "/etc/arvados/config.yml"
22 }()
23
24 type Config struct {
25         Clusters         map[string]Cluster
26         AutoReloadConfig bool
27 }
28
29 // GetConfig returns the current system config, loading it from
30 // configFile if needed.
31 func GetConfig(configFile string) (*Config, error) {
32         var cfg Config
33         err := config.LoadFile(&cfg, configFile)
34         return &cfg, err
35 }
36
37 // GetCluster returns the cluster ID and config for the given
38 // cluster, or the default/only configured cluster if clusterID is "".
39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
40         if clusterID == "" {
41                 if len(sc.Clusters) == 0 {
42                         return nil, fmt.Errorf("no clusters configured")
43                 } else if len(sc.Clusters) > 1 {
44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
45                 } else {
46                         for id, cc := range sc.Clusters {
47                                 cc.ClusterID = id
48                                 return &cc, nil
49                         }
50                 }
51         }
52         cc, ok := sc.Clusters[clusterID]
53         if !ok {
54                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
55         }
56         cc.ClusterID = clusterID
57         return &cc, nil
58 }
59
60 type WebDAVCacheConfig struct {
61         TTL                  Duration
62         UUIDTTL              Duration
63         MaxBlockEntries      int
64         MaxCollectionEntries int
65         MaxCollectionBytes   int64
66         MaxUUIDEntries       int
67         MaxSessions          int
68 }
69
70 type UploadDownloadPermission struct {
71         Upload   bool
72         Download bool
73 }
74
75 type UploadDownloadRolePermissions struct {
76         User  UploadDownloadPermission
77         Admin UploadDownloadPermission
78 }
79
80 type Cluster struct {
81         ClusterID       string `json:"-"`
82         ManagementToken string
83         SystemRootToken string
84         Services        Services
85         InstanceTypes   InstanceTypeMap
86         Containers      ContainersConfig
87         RemoteClusters  map[string]RemoteCluster
88         PostgreSQL      PostgreSQL
89
90         API struct {
91                 AsyncPermissionsUpdateInterval Duration
92                 DisabledAPIs                   StringSet
93                 MaxIndexDatabaseRead           int
94                 MaxItemsPerResponse            int
95                 MaxConcurrentRequests          int
96                 MaxKeepBlobBuffers             int
97                 MaxRequestAmplification        int
98                 MaxRequestSize                 int
99                 MaxTokenLifetime               Duration
100                 RequestTimeout                 Duration
101                 SendTimeout                    Duration
102                 WebsocketClientEventQueue      int
103                 WebsocketServerEventQueue      int
104                 KeepServiceRequestTimeout      Duration
105         }
106         AuditLogs struct {
107                 MaxAge             Duration
108                 MaxDeleteBatch     int
109                 UnloggedAttributes StringSet
110         }
111         Collections struct {
112                 BlobSigning              bool
113                 BlobSigningKey           string
114                 BlobSigningTTL           Duration
115                 BlobTrash                bool
116                 BlobTrashLifetime        Duration
117                 BlobTrashCheckInterval   Duration
118                 BlobTrashConcurrency     int
119                 BlobDeleteConcurrency    int
120                 BlobReplicateConcurrency int
121                 CollectionVersioning     bool
122                 DefaultTrashLifetime     Duration
123                 DefaultReplication       int
124                 ManagedProperties        map[string]struct {
125                         Value     interface{}
126                         Function  string
127                         Protected bool
128                 }
129                 PreserveVersionIfIdle        Duration
130                 TrashSweepInterval           Duration
131                 TrustAllContent              bool
132                 ForwardSlashNameSubstitution string
133                 S3FolderObjects              bool
134
135                 BlobMissingReport        string
136                 BalancePeriod            Duration
137                 BalanceCollectionBatch   int
138                 BalanceCollectionBuffers int
139                 BalanceTimeout           Duration
140                 BalanceUpdateLimit       int
141
142                 WebDAVCache WebDAVCacheConfig
143
144                 KeepproxyPermission UploadDownloadRolePermissions
145                 WebDAVPermission    UploadDownloadRolePermissions
146                 WebDAVLogEvents     bool
147         }
148         Git struct {
149                 GitCommand   string
150                 GitoliteHome string
151                 Repositories string
152         }
153         Login struct {
154                 LDAP struct {
155                         Enable             bool
156                         URL                URL
157                         StartTLS           bool
158                         InsecureTLS        bool
159                         StripDomain        string
160                         AppendDomain       string
161                         SearchAttribute    string
162                         SearchBindUser     string
163                         SearchBindPassword string
164                         SearchBase         string
165                         SearchFilters      string
166                         EmailAttribute     string
167                         UsernameAttribute  string
168                 }
169                 Google struct {
170                         Enable                          bool
171                         ClientID                        string
172                         ClientSecret                    string
173                         AlternateEmailAddresses         bool
174                         AuthenticationRequestParameters map[string]string
175                 }
176                 OpenIDConnect struct {
177                         Enable                          bool
178                         Issuer                          string
179                         ClientID                        string
180                         ClientSecret                    string
181                         EmailClaim                      string
182                         EmailVerifiedClaim              string
183                         UsernameClaim                   string
184                         AcceptAccessToken               bool
185                         AcceptAccessTokenScope          string
186                         AuthenticationRequestParameters map[string]string
187                 }
188                 PAM struct {
189                         Enable             bool
190                         Service            string
191                         DefaultEmailDomain string
192                 }
193                 Test struct {
194                         Enable bool
195                         Users  map[string]TestUser
196                 }
197                 LoginCluster       string
198                 RemoteTokenRefresh Duration
199                 TokenLifetime      Duration
200                 TrustedClients     map[string]struct{}
201                 IssueTrustedTokens bool
202         }
203         Mail struct {
204                 MailchimpAPIKey                string
205                 MailchimpListID                string
206                 SendUserSetupNotificationEmail bool
207                 IssueReporterEmailFrom         string
208                 IssueReporterEmailTo           string
209                 SupportEmailAddress            string
210                 EmailFrom                      string
211         }
212         SystemLogs struct {
213                 LogLevel                string
214                 Format                  string
215                 MaxRequestLogParamsSize int
216         }
217         TLS struct {
218                 Certificate string
219                 Key         string
220                 Insecure    bool
221         }
222         Users struct {
223                 AnonymousUserToken                    string
224                 AdminNotifierEmailFrom                string
225                 AutoAdminFirstUser                    bool
226                 AutoAdminUserWithEmail                string
227                 AutoSetupNewUsers                     bool
228                 AutoSetupNewUsersWithRepository       bool
229                 AutoSetupNewUsersWithVmUUID           string
230                 AutoSetupUsernameBlacklist            StringSet
231                 EmailSubjectPrefix                    string
232                 NewInactiveUserNotificationRecipients StringSet
233                 NewUserNotificationRecipients         StringSet
234                 NewUsersAreActive                     bool
235                 UserNotifierEmailFrom                 string
236                 UserNotifierEmailBcc                  StringSet
237                 UserProfileNotificationAddress        string
238                 PreferDomainForUsername               string
239                 UserSetupMailText                     string
240         }
241         StorageClasses map[string]StorageClassConfig
242         Volumes        map[string]Volume
243         Workbench      struct {
244                 ActivationContactLink            string
245                 APIClientConnectTimeout          Duration
246                 APIClientReceiveTimeout          Duration
247                 APIResponseCompression           bool
248                 ApplicationMimetypesWithViewIcon StringSet
249                 ArvadosDocsite                   string
250                 ArvadosPublicDataDocURL          string
251                 DefaultOpenIdPrefix              string
252                 EnableGettingStartedPopup        bool
253                 EnablePublicProjectsPage         bool
254                 FileViewersConfigURL             string
255                 LogViewerMaxBytes                ByteSize
256                 MultiSiteSearch                  string
257                 ProfilingEnabled                 bool
258                 Repositories                     bool
259                 RepositoryCache                  string
260                 RunningJobLogRecordsToFetch      int
261                 SecretKeyBase                    string
262                 ShowRecentCollectionsOnDashboard bool
263                 ShowUserAgreementInline          bool
264                 ShowUserNotifications            bool
265                 SiteName                         string
266                 Theme                            string
267                 UserProfileFormFields            map[string]struct {
268                         Type                 string
269                         FormFieldTitle       string
270                         FormFieldDescription string
271                         Required             bool
272                         Position             int
273                         Options              map[string]struct{}
274                 }
275                 UserProfileFormMessage string
276                 VocabularyURL          string
277                 WelcomePageHTML        string
278                 InactivePageHTML       string
279                 SSHHelpPageHTML        string
280                 SSHHelpHostSuffix      string
281                 IdleTimeout            Duration
282         }
283 }
284
285 type StorageClassConfig struct {
286         Default  bool
287         Priority int
288 }
289
290 type Volume struct {
291         AccessViaHosts   map[URL]VolumeAccess
292         ReadOnly         bool
293         Replication      int
294         StorageClasses   map[string]bool
295         Driver           string
296         DriverParameters json.RawMessage
297 }
298
299 type S3VolumeDriverParameters struct {
300         IAMRole            string
301         AccessKeyID        string
302         SecretAccessKey    string
303         Endpoint           string
304         Region             string
305         Bucket             string
306         LocationConstraint bool
307         V2Signature        bool
308         UseAWSS3v2Driver   bool
309         IndexPageSize      int
310         ConnectTimeout     Duration
311         ReadTimeout        Duration
312         RaceWindow         Duration
313         UnsafeDelete       bool
314         PrefixLength       int
315 }
316
317 type AzureVolumeDriverParameters struct {
318         StorageAccountName   string
319         StorageAccountKey    string
320         StorageBaseURL       string
321         ContainerName        string
322         RequestTimeout       Duration
323         ListBlobsRetryDelay  Duration
324         ListBlobsMaxAttempts int
325 }
326
327 type DirectoryVolumeDriverParameters struct {
328         Root      string
329         Serialize bool
330 }
331
332 type VolumeAccess struct {
333         ReadOnly bool
334 }
335
336 type Services struct {
337         Composer       Service
338         Controller     Service
339         DispatchCloud  Service
340         DispatchLSF    Service
341         GitHTTP        Service
342         GitSSH         Service
343         Health         Service
344         Keepbalance    Service
345         Keepproxy      Service
346         Keepstore      Service
347         RailsAPI       Service
348         WebDAVDownload Service
349         WebDAV         Service
350         WebShell       Service
351         Websocket      Service
352         Workbench1     Service
353         Workbench2     Service
354 }
355
356 type Service struct {
357         InternalURLs map[URL]ServiceInstance
358         ExternalURL  URL
359 }
360
361 type TestUser struct {
362         Email    string
363         Password string
364 }
365
366 // URL is a url.URL that is also usable as a JSON key/value.
367 type URL url.URL
368
369 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
370 // used as a JSON key/value.
371 func (su *URL) UnmarshalText(text []byte) error {
372         u, err := url.Parse(string(text))
373         if err == nil {
374                 *su = URL(*u)
375                 if su.Path == "" && su.Host != "" {
376                         // http://example really means http://example/
377                         su.Path = "/"
378                 }
379         }
380         return err
381 }
382
383 func (su URL) MarshalText() ([]byte, error) {
384         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
385 }
386
387 func (su URL) String() string {
388         return (*url.URL)(&su).String()
389 }
390
391 type ServiceInstance struct {
392         Rendezvous string `json:",omitempty"`
393 }
394
395 type PostgreSQL struct {
396         Connection     PostgreSQLConnection
397         ConnectionPool int
398 }
399
400 type PostgreSQLConnection map[string]string
401
402 type RemoteCluster struct {
403         Host          string
404         Proxy         bool
405         Scheme        string
406         Insecure      bool
407         ActivateUsers bool
408 }
409
410 type InstanceType struct {
411         Name            string
412         ProviderType    string
413         VCPUs           int
414         RAM             ByteSize
415         Scratch         ByteSize
416         IncludedScratch ByteSize
417         AddedScratch    ByteSize
418         Price           float64
419         Preemptible     bool
420 }
421
422 type ContainersConfig struct {
423         CloudVMs                    CloudVMsConfig
424         CrunchRunCommand            string
425         CrunchRunArgumentsList      []string
426         DefaultKeepCacheRAM         ByteSize
427         DispatchPrivateKey          string
428         LogReuseDecisions           bool
429         MaxComputeVMs               int
430         MaxDispatchAttempts         int
431         MaxRetryAttempts            int
432         MinRetryPeriod              Duration
433         ReserveExtraRAM             ByteSize
434         StaleLockTimeout            Duration
435         SupportedDockerImageFormats StringSet
436         UsePreemptibleInstances     bool
437         RuntimeEngine               string
438         LocalKeepBlobBuffersPerVCPU int
439
440         JobsAPI struct {
441                 Enable         string
442                 GitInternalDir string
443         }
444         Logging struct {
445                 MaxAge                       Duration
446                 LogBytesPerEvent             int
447                 LogSecondsBetweenEvents      Duration
448                 LogThrottlePeriod            Duration
449                 LogThrottleBytes             int
450                 LogThrottleLines             int
451                 LimitLogBytesPerJob          int
452                 LogPartialLineThrottlePeriod Duration
453                 LogUpdatePeriod              Duration
454                 LogUpdateSize                ByteSize
455         }
456         ShellAccess struct {
457                 Admin bool
458                 User  bool
459         }
460         SLURM struct {
461                 PrioritySpread             int64
462                 SbatchArgumentsList        []string
463                 SbatchEnvironmentVariables map[string]string
464                 Managed                    struct {
465                         DNSServerConfDir       string
466                         DNSServerConfTemplate  string
467                         DNSServerReloadCommand string
468                         DNSServerUpdateCommand string
469                         ComputeNodeDomain      string
470                         ComputeNodeNameservers StringSet
471                         AssignNodeHostname     string
472                 }
473         }
474         LSF struct {
475                 BsubSudoUser      string
476                 BsubArgumentsList []string
477         }
478 }
479
480 type CloudVMsConfig struct {
481         Enable bool
482
483         BootProbeCommand               string
484         DeployRunnerBinary             string
485         ImageID                        string
486         MaxCloudOpsPerSecond           int
487         MaxProbesPerSecond             int
488         MaxConcurrentInstanceCreateOps int
489         PollInterval                   Duration
490         ProbeInterval                  Duration
491         SSHPort                        string
492         SyncInterval                   Duration
493         TimeoutBooting                 Duration
494         TimeoutIdle                    Duration
495         TimeoutProbe                   Duration
496         TimeoutShutdown                Duration
497         TimeoutSignal                  Duration
498         TimeoutStaleRunLock            Duration
499         TimeoutTERM                    Duration
500         ResourceTags                   map[string]string
501         TagKeyPrefix                   string
502
503         Driver           string
504         DriverParameters json.RawMessage
505 }
506
507 type InstanceTypeMap map[string]InstanceType
508
509 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
510
511 // UnmarshalJSON handles old config files that provide an array of
512 // instance types instead of a hash.
513 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
514         fixup := func(t InstanceType) (InstanceType, error) {
515                 if t.ProviderType == "" {
516                         t.ProviderType = t.Name
517                 }
518                 if t.Scratch == 0 {
519                         t.Scratch = t.IncludedScratch + t.AddedScratch
520                 } else if t.AddedScratch == 0 {
521                         t.AddedScratch = t.Scratch - t.IncludedScratch
522                 } else if t.IncludedScratch == 0 {
523                         t.IncludedScratch = t.Scratch - t.AddedScratch
524                 }
525
526                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
527                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
528                 }
529                 return t, nil
530         }
531
532         if len(data) > 0 && data[0] == '[' {
533                 var arr []InstanceType
534                 err := json.Unmarshal(data, &arr)
535                 if err != nil {
536                         return err
537                 }
538                 if len(arr) == 0 {
539                         *it = nil
540                         return nil
541                 }
542                 *it = make(map[string]InstanceType, len(arr))
543                 for _, t := range arr {
544                         if _, ok := (*it)[t.Name]; ok {
545                                 return errDuplicateInstanceTypeName
546                         }
547                         t, err := fixup(t)
548                         if err != nil {
549                                 return err
550                         }
551                         (*it)[t.Name] = t
552                 }
553                 return nil
554         }
555         var hash map[string]InstanceType
556         err := json.Unmarshal(data, &hash)
557         if err != nil {
558                 return err
559         }
560         // Fill in Name field (and ProviderType field, if not
561         // specified) using hash key.
562         *it = InstanceTypeMap(hash)
563         for name, t := range *it {
564                 t.Name = name
565                 t, err := fixup(t)
566                 if err != nil {
567                         return err
568                 }
569                 (*it)[name] = t
570         }
571         return nil
572 }
573
574 type StringSet map[string]struct{}
575
576 // UnmarshalJSON handles old config files that provide an array of
577 // instance types instead of a hash.
578 func (ss *StringSet) UnmarshalJSON(data []byte) error {
579         if len(data) > 0 && data[0] == '[' {
580                 var arr []string
581                 err := json.Unmarshal(data, &arr)
582                 if err != nil {
583                         return err
584                 }
585                 if len(arr) == 0 {
586                         *ss = nil
587                         return nil
588                 }
589                 *ss = make(map[string]struct{}, len(arr))
590                 for _, t := range arr {
591                         (*ss)[t] = struct{}{}
592                 }
593                 return nil
594         }
595         var hash map[string]struct{}
596         err := json.Unmarshal(data, &hash)
597         if err != nil {
598                 return err
599         }
600         *ss = make(map[string]struct{}, len(hash))
601         for t := range hash {
602                 (*ss)[t] = struct{}{}
603         }
604
605         return nil
606 }
607
608 type ServiceName string
609
610 const (
611         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
612         ServiceNameController    ServiceName = "arvados-controller"
613         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
614         ServiceNameDispatchLSF   ServiceName = "arvados-dispatch-lsf"
615         ServiceNameHealth        ServiceName = "arvados-health"
616         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
617         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
618         ServiceNameWebsocket     ServiceName = "arvados-ws"
619         ServiceNameKeepbalance   ServiceName = "keep-balance"
620         ServiceNameKeepweb       ServiceName = "keep-web"
621         ServiceNameKeepproxy     ServiceName = "keepproxy"
622         ServiceNameKeepstore     ServiceName = "keepstore"
623 )
624
625 // Map returns all services as a map, suitable for iterating over all
626 // services or looking up a service by name.
627 func (svcs Services) Map() map[ServiceName]Service {
628         return map[ServiceName]Service{
629                 ServiceNameRailsAPI:      svcs.RailsAPI,
630                 ServiceNameController:    svcs.Controller,
631                 ServiceNameDispatchCloud: svcs.DispatchCloud,
632                 ServiceNameDispatchLSF:   svcs.DispatchLSF,
633                 ServiceNameHealth:        svcs.Health,
634                 ServiceNameWorkbench1:    svcs.Workbench1,
635                 ServiceNameWorkbench2:    svcs.Workbench2,
636                 ServiceNameWebsocket:     svcs.Websocket,
637                 ServiceNameKeepbalance:   svcs.Keepbalance,
638                 ServiceNameKeepweb:       svcs.WebDAV,
639                 ServiceNameKeepproxy:     svcs.Keepproxy,
640                 ServiceNameKeepstore:     svcs.Keepstore,
641         }
642 }