Merge branch '21124-max-rails-reqs'
[arvados.git] / sdk / go / arvados / config.go
1 // Copyright (C) The Arvados Authors. All rights reserved.
2 //
3 // SPDX-License-Identifier: Apache-2.0
4
5 package arvados
6
7 import (
8         "crypto/tls"
9         "encoding/json"
10         "errors"
11         "fmt"
12         "net/url"
13         "os"
14         "time"
15
16         "git.arvados.org/arvados.git/sdk/go/config"
17 )
18
19 var DefaultConfigFile = func() string {
20         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
21                 return path
22         }
23         return "/etc/arvados/config.yml"
24 }()
25
26 type Config struct {
27         Clusters         map[string]Cluster
28         AutoReloadConfig bool
29         SourceTimestamp  time.Time
30         SourceSHA256     string
31 }
32
33 // GetConfig returns the current system config, loading it from
34 // configFile if needed.
35 func GetConfig(configFile string) (*Config, error) {
36         var cfg Config
37         err := config.LoadFile(&cfg, configFile)
38         return &cfg, err
39 }
40
41 // GetCluster returns the cluster ID and config for the given
42 // cluster, or the default/only configured cluster if clusterID is "".
43 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
44         if clusterID == "" {
45                 if len(sc.Clusters) == 0 {
46                         return nil, fmt.Errorf("no clusters configured")
47                 } else if len(sc.Clusters) > 1 {
48                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
49                 } else {
50                         for id, cc := range sc.Clusters {
51                                 cc.ClusterID = id
52                                 return &cc, nil
53                         }
54                 }
55         }
56         cc, ok := sc.Clusters[clusterID]
57         if !ok {
58                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
59         }
60         cc.ClusterID = clusterID
61         return &cc, nil
62 }
63
64 type WebDAVCacheConfig struct {
65         TTL                Duration
66         MaxBlockEntries    int
67         MaxCollectionBytes int64
68         MaxSessions        int
69 }
70
71 type UploadDownloadPermission struct {
72         Upload   bool
73         Download bool
74 }
75
76 type UploadDownloadRolePermissions struct {
77         User  UploadDownloadPermission
78         Admin UploadDownloadPermission
79 }
80
81 type ManagedProperties map[string]struct {
82         Value     interface{}
83         Function  string
84         Protected bool
85 }
86
87 type Cluster struct {
88         ClusterID       string `json:"-"`
89         ManagementToken string
90         SystemRootToken string
91         Services        Services
92         InstanceTypes   InstanceTypeMap
93         Containers      ContainersConfig
94         RemoteClusters  map[string]RemoteCluster
95         PostgreSQL      PostgreSQL
96
97         API struct {
98                 AsyncPermissionsUpdateInterval   Duration
99                 DisabledAPIs                     StringSet
100                 MaxIndexDatabaseRead             int
101                 MaxItemsPerResponse              int
102                 MaxConcurrentRailsRequests       int
103                 MaxConcurrentRequests            int
104                 MaxQueuedRequests                int
105                 MaxQueueTimeForLockRequests      Duration
106                 LogCreateRequestFraction         float64
107                 MaxKeepBlobBuffers               int
108                 MaxRequestAmplification          int
109                 MaxRequestSize                   int
110                 MaxTokenLifetime                 Duration
111                 RequestTimeout                   Duration
112                 SendTimeout                      Duration
113                 WebsocketClientEventQueue        int
114                 WebsocketServerEventQueue        int
115                 KeepServiceRequestTimeout        Duration
116                 VocabularyPath                   string
117                 FreezeProjectRequiresDescription bool
118                 FreezeProjectRequiresProperties  StringSet
119                 UnfreezeProjectRequiresAdmin     bool
120                 LockBeforeUpdate                 bool
121         }
122         AuditLogs struct {
123                 MaxAge             Duration
124                 MaxDeleteBatch     int
125                 UnloggedAttributes StringSet
126         }
127         Collections struct {
128                 BlobSigning                  bool
129                 BlobSigningKey               string
130                 BlobSigningTTL               Duration
131                 BlobTrash                    bool
132                 BlobTrashLifetime            Duration
133                 BlobTrashCheckInterval       Duration
134                 BlobTrashConcurrency         int
135                 BlobDeleteConcurrency        int
136                 BlobReplicateConcurrency     int
137                 CollectionVersioning         bool
138                 DefaultTrashLifetime         Duration
139                 DefaultReplication           int
140                 ManagedProperties            ManagedProperties
141                 PreserveVersionIfIdle        Duration
142                 TrashSweepInterval           Duration
143                 TrustAllContent              bool
144                 ForwardSlashNameSubstitution string
145                 S3FolderObjects              bool
146
147                 BlobMissingReport        string
148                 BalancePeriod            Duration
149                 BalanceCollectionBatch   int
150                 BalanceCollectionBuffers int
151                 BalanceTimeout           Duration
152                 BalanceUpdateLimit       int
153
154                 WebDAVCache WebDAVCacheConfig
155
156                 KeepproxyPermission UploadDownloadRolePermissions
157                 WebDAVPermission    UploadDownloadRolePermissions
158                 WebDAVLogEvents     bool
159         }
160         Git struct {
161                 GitCommand   string
162                 GitoliteHome string
163                 Repositories string
164         }
165         Login struct {
166                 LDAP struct {
167                         Enable             bool
168                         URL                URL
169                         StartTLS           bool
170                         InsecureTLS        bool
171                         MinTLSVersion      TLSVersion
172                         StripDomain        string
173                         AppendDomain       string
174                         SearchAttribute    string
175                         SearchBindUser     string
176                         SearchBindPassword string
177                         SearchBase         string
178                         SearchFilters      string
179                         EmailAttribute     string
180                         UsernameAttribute  string
181                 }
182                 Google struct {
183                         Enable                          bool
184                         ClientID                        string
185                         ClientSecret                    string
186                         AlternateEmailAddresses         bool
187                         AuthenticationRequestParameters map[string]string
188                 }
189                 OpenIDConnect struct {
190                         Enable                          bool
191                         Issuer                          string
192                         ClientID                        string
193                         ClientSecret                    string
194                         EmailClaim                      string
195                         EmailVerifiedClaim              string
196                         UsernameClaim                   string
197                         AcceptAccessToken               bool
198                         AcceptAccessTokenScope          string
199                         AuthenticationRequestParameters map[string]string
200                 }
201                 PAM struct {
202                         Enable             bool
203                         Service            string
204                         DefaultEmailDomain string
205                 }
206                 Test struct {
207                         Enable bool
208                         Users  map[string]TestUser
209                 }
210                 LoginCluster         string
211                 RemoteTokenRefresh   Duration
212                 TokenLifetime        Duration
213                 TrustedClients       map[URL]struct{}
214                 TrustPrivateNetworks bool
215                 IssueTrustedTokens   bool
216         }
217         Mail struct {
218                 MailchimpAPIKey                string
219                 MailchimpListID                string
220                 SendUserSetupNotificationEmail bool
221                 IssueReporterEmailFrom         string
222                 IssueReporterEmailTo           string
223                 SupportEmailAddress            string
224                 EmailFrom                      string
225         }
226         SystemLogs struct {
227                 LogLevel                  string
228                 Format                    string
229                 MaxRequestLogParamsSize   int
230                 RequestQueueDumpDirectory string
231         }
232         TLS struct {
233                 Certificate string
234                 Key         string
235                 Insecure    bool
236                 ACME        struct {
237                         Server string
238                 }
239         }
240         Users struct {
241                 ActivatedUsersAreVisibleToOthers      bool
242                 AnonymousUserToken                    string
243                 AdminNotifierEmailFrom                string
244                 AutoAdminFirstUser                    bool
245                 AutoAdminUserWithEmail                string
246                 AutoSetupNewUsers                     bool
247                 AutoSetupNewUsersWithRepository       bool
248                 AutoSetupNewUsersWithVmUUID           string
249                 AutoSetupUsernameBlacklist            StringSet
250                 EmailSubjectPrefix                    string
251                 NewInactiveUserNotificationRecipients StringSet
252                 NewUserNotificationRecipients         StringSet
253                 NewUsersAreActive                     bool
254                 UserNotifierEmailFrom                 string
255                 UserNotifierEmailBcc                  StringSet
256                 UserProfileNotificationAddress        string
257                 PreferDomainForUsername               string
258                 UserSetupMailText                     string
259                 RoleGroupsVisibleToAll                bool
260                 CanCreateRoleGroups                   bool
261                 ActivityLoggingPeriod                 Duration
262                 SyncIgnoredGroups                     []string
263                 SyncRequiredGroups                    []string
264                 SyncUserAccounts                      bool
265                 SyncUserAPITokens                     bool
266                 SyncUserGroups                        bool
267                 SyncUserSSHKeys                       bool
268         }
269         StorageClasses map[string]StorageClassConfig
270         Volumes        map[string]Volume
271         Workbench      struct {
272                 ActivationContactLink            string
273                 APIClientConnectTimeout          Duration
274                 APIClientReceiveTimeout          Duration
275                 APIResponseCompression           bool
276                 ApplicationMimetypesWithViewIcon StringSet
277                 ArvadosDocsite                   string
278                 ArvadosPublicDataDocURL          string
279                 DefaultOpenIdPrefix              string
280                 DisableSharingURLsUI             bool
281                 EnableGettingStartedPopup        bool
282                 EnablePublicProjectsPage         bool
283                 FileViewersConfigURL             string
284                 LogViewerMaxBytes                ByteSize
285                 MultiSiteSearch                  string
286                 ProfilingEnabled                 bool
287                 Repositories                     bool
288                 RepositoryCache                  string
289                 RunningJobLogRecordsToFetch      int
290                 SecretKeyBase                    string
291                 ShowRecentCollectionsOnDashboard bool
292                 ShowUserAgreementInline          bool
293                 ShowUserNotifications            bool
294                 SiteName                         string
295                 Theme                            string
296                 UserProfileFormFields            map[string]struct {
297                         Type                 string
298                         FormFieldTitle       string
299                         FormFieldDescription string
300                         Required             bool
301                         Position             int
302                         Options              map[string]struct{}
303                 }
304                 UserProfileFormMessage string
305                 WelcomePageHTML        string
306                 InactivePageHTML       string
307                 SSHHelpPageHTML        string
308                 SSHHelpHostSuffix      string
309                 IdleTimeout            Duration
310                 BannerUUID             string
311         }
312 }
313
314 type StorageClassConfig struct {
315         Default  bool
316         Priority int
317 }
318
319 type Volume struct {
320         AccessViaHosts   map[URL]VolumeAccess
321         ReadOnly         bool
322         Replication      int
323         StorageClasses   map[string]bool
324         Driver           string
325         DriverParameters json.RawMessage
326 }
327
328 type S3VolumeDriverParameters struct {
329         IAMRole            string
330         AccessKeyID        string
331         SecretAccessKey    string
332         Endpoint           string
333         Region             string
334         Bucket             string
335         LocationConstraint bool
336         V2Signature        bool
337         IndexPageSize      int
338         ConnectTimeout     Duration
339         ReadTimeout        Duration
340         RaceWindow         Duration
341         UnsafeDelete       bool
342         PrefixLength       int
343 }
344
345 type AzureVolumeDriverParameters struct {
346         StorageAccountName   string
347         StorageAccountKey    string
348         StorageBaseURL       string
349         ContainerName        string
350         RequestTimeout       Duration
351         ListBlobsRetryDelay  Duration
352         ListBlobsMaxAttempts int
353 }
354
355 type DirectoryVolumeDriverParameters struct {
356         Root      string
357         Serialize bool
358 }
359
360 type VolumeAccess struct {
361         ReadOnly bool
362 }
363
364 type Services struct {
365         Composer       Service
366         Controller     Service
367         DispatchCloud  Service
368         DispatchLSF    Service
369         DispatchSLURM  Service
370         GitHTTP        Service
371         GitSSH         Service
372         Health         Service
373         Keepbalance    Service
374         Keepproxy      Service
375         Keepstore      Service
376         RailsAPI       Service
377         WebDAVDownload Service
378         WebDAV         Service
379         WebShell       Service
380         Websocket      Service
381         Workbench1     Service
382         Workbench2     Service
383 }
384
385 type Service struct {
386         InternalURLs map[URL]ServiceInstance
387         ExternalURL  URL
388 }
389
390 type TestUser struct {
391         Email    string
392         Password string
393 }
394
395 // URL is a url.URL that is also usable as a JSON key/value.
396 type URL url.URL
397
398 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
399 // used as a JSON key/value.
400 func (su *URL) UnmarshalText(text []byte) error {
401         u, err := url.Parse(string(text))
402         if err == nil {
403                 *su = URL(*u)
404                 if su.Path == "" && su.Host != "" {
405                         // http://example really means http://example/
406                         su.Path = "/"
407                 }
408         }
409         return err
410 }
411
412 func (su URL) MarshalText() ([]byte, error) {
413         return []byte(su.String()), nil
414 }
415
416 func (su URL) String() string {
417         return (*url.URL)(&su).String()
418 }
419
420 type TLSVersion uint16
421
422 func (v TLSVersion) MarshalText() ([]byte, error) {
423         switch v {
424         case 0:
425                 return []byte{}, nil
426         case tls.VersionTLS10:
427                 return []byte("1.0"), nil
428         case tls.VersionTLS11:
429                 return []byte("1.1"), nil
430         case tls.VersionTLS12:
431                 return []byte("1.2"), nil
432         case tls.VersionTLS13:
433                 return []byte("1.3"), nil
434         default:
435                 return nil, fmt.Errorf("unsupported TLSVersion %x", v)
436         }
437 }
438
439 func (v *TLSVersion) UnmarshalJSON(text []byte) error {
440         if len(text) > 0 && text[0] == '"' {
441                 var s string
442                 err := json.Unmarshal(text, &s)
443                 if err != nil {
444                         return err
445                 }
446                 text = []byte(s)
447         }
448         switch string(text) {
449         case "":
450                 *v = 0
451         case "1.0":
452                 *v = tls.VersionTLS10
453         case "1.1":
454                 *v = tls.VersionTLS11
455         case "1.2":
456                 *v = tls.VersionTLS12
457         case "1.3":
458                 *v = tls.VersionTLS13
459         default:
460                 return fmt.Errorf("unsupported TLSVersion %q", text)
461         }
462         return nil
463 }
464
465 type ServiceInstance struct {
466         ListenURL  URL
467         Rendezvous string `json:",omitempty"`
468 }
469
470 type PostgreSQL struct {
471         Connection     PostgreSQLConnection
472         ConnectionPool int
473 }
474
475 type PostgreSQLConnection map[string]string
476
477 type RemoteCluster struct {
478         Host          string
479         Proxy         bool
480         Scheme        string
481         Insecure      bool
482         ActivateUsers bool
483 }
484
485 type CUDAFeatures struct {
486         DriverVersion      string
487         HardwareCapability string
488         DeviceCount        int
489 }
490
491 type InstanceType struct {
492         Name            string `json:"-"`
493         ProviderType    string
494         VCPUs           int
495         RAM             ByteSize
496         Scratch         ByteSize `json:"-"`
497         IncludedScratch ByteSize
498         AddedScratch    ByteSize
499         Price           float64
500         Preemptible     bool
501         CUDA            CUDAFeatures
502 }
503
504 type ContainersConfig struct {
505         CloudVMs                      CloudVMsConfig
506         CrunchRunCommand              string
507         CrunchRunArgumentsList        []string
508         DefaultKeepCacheRAM           ByteSize
509         DispatchPrivateKey            string
510         LogReuseDecisions             bool
511         MaxDispatchAttempts           int
512         MaxRetryAttempts              int
513         MinRetryPeriod                Duration
514         ReserveExtraRAM               ByteSize
515         StaleLockTimeout              Duration
516         SupportedDockerImageFormats   StringSet
517         AlwaysUsePreemptibleInstances bool
518         PreemptiblePriceFactor        float64
519         MaximumPriceFactor            float64
520         RuntimeEngine                 string
521         LocalKeepBlobBuffersPerVCPU   int
522         LocalKeepLogsToContainerLog   string
523
524         JobsAPI struct {
525                 Enable         string
526                 GitInternalDir string
527         }
528         Logging struct {
529                 MaxAge                       Duration
530                 SweepInterval                Duration
531                 LogBytesPerEvent             int
532                 LogSecondsBetweenEvents      Duration
533                 LogThrottlePeriod            Duration
534                 LogThrottleBytes             int
535                 LogThrottleLines             int
536                 LimitLogBytesPerJob          int
537                 LogPartialLineThrottlePeriod Duration
538                 LogUpdatePeriod              Duration
539                 LogUpdateSize                ByteSize
540         }
541         ShellAccess struct {
542                 Admin bool
543                 User  bool
544         }
545         SLURM struct {
546                 PrioritySpread             int64
547                 SbatchArgumentsList        []string
548                 SbatchEnvironmentVariables map[string]string
549                 Managed                    struct {
550                         DNSServerConfDir       string
551                         DNSServerConfTemplate  string
552                         DNSServerReloadCommand string
553                         DNSServerUpdateCommand string
554                         ComputeNodeDomain      string
555                         ComputeNodeNameservers StringSet
556                         AssignNodeHostname     string
557                 }
558         }
559         LSF struct {
560                 BsubSudoUser      string
561                 BsubArgumentsList []string
562                 BsubCUDAArguments []string
563         }
564 }
565
566 type CloudVMsConfig struct {
567         Enable bool
568
569         BootProbeCommand               string
570         InstanceInitCommand            string
571         DeployRunnerBinary             string
572         DeployPublicKey                bool
573         ImageID                        string
574         MaxCloudOpsPerSecond           int
575         MaxProbesPerSecond             int
576         MaxConcurrentInstanceCreateOps int
577         MaxInstances                   int
578         InitialQuotaEstimate           int
579         SupervisorFraction             float64
580         PollInterval                   Duration
581         ProbeInterval                  Duration
582         SSHPort                        string
583         SyncInterval                   Duration
584         TimeoutBooting                 Duration
585         TimeoutIdle                    Duration
586         TimeoutProbe                   Duration
587         TimeoutShutdown                Duration
588         TimeoutSignal                  Duration
589         TimeoutStaleRunLock            Duration
590         TimeoutTERM                    Duration
591         ResourceTags                   map[string]string
592         TagKeyPrefix                   string
593
594         Driver           string
595         DriverParameters json.RawMessage
596 }
597
598 type InstanceTypeMap map[string]InstanceType
599
600 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
601
602 // UnmarshalJSON does special handling of InstanceTypes:
603 //
604 // - populate computed fields (Name and Scratch)
605 //
606 // - error out if InstancesTypes are populated as an array, which was
607 // deprecated in Arvados 1.2.0
608 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
609         fixup := func(t InstanceType) (InstanceType, error) {
610                 if t.ProviderType == "" {
611                         t.ProviderType = t.Name
612                 }
613                 // If t.Scratch is set in the configuration file, it will be ignored and overwritten.
614                 // It will also generate a "deprecated or unknown config entry" warning.
615                 t.Scratch = t.IncludedScratch + t.AddedScratch
616                 return t, nil
617         }
618
619         if len(data) > 0 && data[0] == '[' {
620                 return fmt.Errorf("InstanceTypes must be specified as a map, not an array, see https://doc.arvados.org/admin/config.html")
621         }
622         var hash map[string]InstanceType
623         err := json.Unmarshal(data, &hash)
624         if err != nil {
625                 return err
626         }
627         // Fill in Name field (and ProviderType field, if not
628         // specified) using hash key.
629         *it = InstanceTypeMap(hash)
630         for name, t := range *it {
631                 t.Name = name
632                 t, err := fixup(t)
633                 if err != nil {
634                         return err
635                 }
636                 (*it)[name] = t
637         }
638         return nil
639 }
640
641 type StringSet map[string]struct{}
642
643 // UnmarshalJSON handles old config files that provide an array of
644 // instance types instead of a hash.
645 func (ss *StringSet) UnmarshalJSON(data []byte) error {
646         if len(data) > 0 && data[0] == '[' {
647                 var arr []string
648                 err := json.Unmarshal(data, &arr)
649                 if err != nil {
650                         return err
651                 }
652                 if len(arr) == 0 {
653                         *ss = nil
654                         return nil
655                 }
656                 *ss = make(map[string]struct{}, len(arr))
657                 for _, t := range arr {
658                         (*ss)[t] = struct{}{}
659                 }
660                 return nil
661         }
662         var hash map[string]struct{}
663         err := json.Unmarshal(data, &hash)
664         if err != nil {
665                 return err
666         }
667         *ss = make(map[string]struct{}, len(hash))
668         for t := range hash {
669                 (*ss)[t] = struct{}{}
670         }
671
672         return nil
673 }
674
675 type ServiceName string
676
677 const (
678         ServiceNameController    ServiceName = "arvados-controller"
679         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
680         ServiceNameDispatchLSF   ServiceName = "arvados-dispatch-lsf"
681         ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm"
682         ServiceNameGitHTTP       ServiceName = "arvados-git-httpd"
683         ServiceNameHealth        ServiceName = "arvados-health"
684         ServiceNameKeepbalance   ServiceName = "keep-balance"
685         ServiceNameKeepproxy     ServiceName = "keepproxy"
686         ServiceNameKeepstore     ServiceName = "keepstore"
687         ServiceNameKeepweb       ServiceName = "keep-web"
688         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
689         ServiceNameWebsocket     ServiceName = "arvados-ws"
690         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
691         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
692 )
693
694 // Map returns all services as a map, suitable for iterating over all
695 // services or looking up a service by name.
696 func (svcs Services) Map() map[ServiceName]Service {
697         return map[ServiceName]Service{
698                 ServiceNameController:    svcs.Controller,
699                 ServiceNameDispatchCloud: svcs.DispatchCloud,
700                 ServiceNameDispatchLSF:   svcs.DispatchLSF,
701                 ServiceNameDispatchSLURM: svcs.DispatchSLURM,
702                 ServiceNameGitHTTP:       svcs.GitHTTP,
703                 ServiceNameHealth:        svcs.Health,
704                 ServiceNameKeepbalance:   svcs.Keepbalance,
705                 ServiceNameKeepproxy:     svcs.Keepproxy,
706                 ServiceNameKeepstore:     svcs.Keepstore,
707                 ServiceNameKeepweb:       svcs.WebDAV,
708                 ServiceNameRailsAPI:      svcs.RailsAPI,
709                 ServiceNameWebsocket:     svcs.Websocket,
710                 ServiceNameWorkbench1:    svcs.Workbench1,
711                 ServiceNameWorkbench2:    svcs.Workbench2,
712         }
713 }