Merge branch 'main' from workbench2.git
[arvados.git] / sdk / go / arvados / config.go
1 // Copyright (C) The Arvados Authors. All rights reserved.
2 //
3 // SPDX-License-Identifier: Apache-2.0
4
5 package arvados
6
7 import (
8         "crypto/tls"
9         "encoding/json"
10         "errors"
11         "fmt"
12         "net/url"
13         "os"
14         "time"
15
16         "git.arvados.org/arvados.git/sdk/go/config"
17 )
18
19 var DefaultConfigFile = func() string {
20         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
21                 return path
22         }
23         return "/etc/arvados/config.yml"
24 }()
25
26 type Config struct {
27         Clusters         map[string]Cluster
28         AutoReloadConfig bool
29         SourceTimestamp  time.Time
30         SourceSHA256     string
31 }
32
33 // GetConfig returns the current system config, loading it from
34 // configFile if needed.
35 func GetConfig(configFile string) (*Config, error) {
36         var cfg Config
37         err := config.LoadFile(&cfg, configFile)
38         return &cfg, err
39 }
40
41 // GetCluster returns the cluster ID and config for the given
42 // cluster, or the default/only configured cluster if clusterID is "".
43 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
44         if clusterID == "" {
45                 if len(sc.Clusters) == 0 {
46                         return nil, fmt.Errorf("no clusters configured")
47                 } else if len(sc.Clusters) > 1 {
48                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
49                 } else {
50                         for id, cc := range sc.Clusters {
51                                 cc.ClusterID = id
52                                 return &cc, nil
53                         }
54                 }
55         }
56         cc, ok := sc.Clusters[clusterID]
57         if !ok {
58                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
59         }
60         cc.ClusterID = clusterID
61         return &cc, nil
62 }
63
64 type WebDAVCacheConfig struct {
65         TTL                Duration
66         MaxBlockEntries    int
67         MaxCollectionBytes int64
68         MaxSessions        int
69 }
70
71 type UploadDownloadPermission struct {
72         Upload   bool
73         Download bool
74 }
75
76 type UploadDownloadRolePermissions struct {
77         User  UploadDownloadPermission
78         Admin UploadDownloadPermission
79 }
80
81 type ManagedProperties map[string]struct {
82         Value     interface{}
83         Function  string
84         Protected bool
85 }
86
87 type Cluster struct {
88         ClusterID       string `json:"-"`
89         ManagementToken string
90         SystemRootToken string
91         Services        Services
92         InstanceTypes   InstanceTypeMap
93         Containers      ContainersConfig
94         RemoteClusters  map[string]RemoteCluster
95         PostgreSQL      PostgreSQL
96
97         API struct {
98                 AsyncPermissionsUpdateInterval   Duration
99                 DisabledAPIs                     StringSet
100                 MaxIndexDatabaseRead             int
101                 MaxItemsPerResponse              int
102                 MaxConcurrentRailsRequests       int
103                 MaxConcurrentRequests            int
104                 MaxQueuedRequests                int
105                 MaxQueueTimeForLockRequests      Duration
106                 LogCreateRequestFraction         float64
107                 MaxKeepBlobBuffers               int
108                 MaxRequestAmplification          int
109                 MaxRequestSize                   int
110                 MaxTokenLifetime                 Duration
111                 RequestTimeout                   Duration
112                 SendTimeout                      Duration
113                 WebsocketClientEventQueue        int
114                 WebsocketServerEventQueue        int
115                 KeepServiceRequestTimeout        Duration
116                 VocabularyPath                   string
117                 FreezeProjectRequiresDescription bool
118                 FreezeProjectRequiresProperties  StringSet
119                 UnfreezeProjectRequiresAdmin     bool
120                 LockBeforeUpdate                 bool
121         }
122         AuditLogs struct {
123                 MaxAge             Duration
124                 MaxDeleteBatch     int
125                 UnloggedAttributes StringSet
126         }
127         Collections struct {
128                 BlobSigning                  bool
129                 BlobSigningKey               string
130                 BlobSigningTTL               Duration
131                 BlobTrash                    bool
132                 BlobTrashLifetime            Duration
133                 BlobTrashCheckInterval       Duration
134                 BlobTrashConcurrency         int
135                 BlobDeleteConcurrency        int
136                 BlobReplicateConcurrency     int
137                 CollectionVersioning         bool
138                 DefaultTrashLifetime         Duration
139                 DefaultReplication           int
140                 ManagedProperties            ManagedProperties
141                 PreserveVersionIfIdle        Duration
142                 TrashSweepInterval           Duration
143                 TrustAllContent              bool
144                 ForwardSlashNameSubstitution string
145                 S3FolderObjects              bool
146
147                 BlobMissingReport        string
148                 BalancePeriod            Duration
149                 BalanceCollectionBatch   int
150                 BalanceCollectionBuffers int
151                 BalanceTimeout           Duration
152                 BalanceUpdateLimit       int
153
154                 WebDAVCache WebDAVCacheConfig
155
156                 KeepproxyPermission UploadDownloadRolePermissions
157                 WebDAVPermission    UploadDownloadRolePermissions
158                 WebDAVLogEvents     bool
159         }
160         Git struct {
161                 GitCommand   string
162                 GitoliteHome string
163                 Repositories string
164         }
165         Login struct {
166                 LDAP struct {
167                         Enable             bool
168                         URL                URL
169                         StartTLS           bool
170                         InsecureTLS        bool
171                         MinTLSVersion      TLSVersion
172                         StripDomain        string
173                         AppendDomain       string
174                         SearchAttribute    string
175                         SearchBindUser     string
176                         SearchBindPassword string
177                         SearchBase         string
178                         SearchFilters      string
179                         EmailAttribute     string
180                         UsernameAttribute  string
181                 }
182                 Google struct {
183                         Enable                          bool
184                         ClientID                        string
185                         ClientSecret                    string
186                         AlternateEmailAddresses         bool
187                         AuthenticationRequestParameters map[string]string
188                 }
189                 OpenIDConnect struct {
190                         Enable                          bool
191                         Issuer                          string
192                         ClientID                        string
193                         ClientSecret                    string
194                         EmailClaim                      string
195                         EmailVerifiedClaim              string
196                         UsernameClaim                   string
197                         AcceptAccessToken               bool
198                         AcceptAccessTokenScope          string
199                         AuthenticationRequestParameters map[string]string
200                 }
201                 PAM struct {
202                         Enable             bool
203                         Service            string
204                         DefaultEmailDomain string
205                 }
206                 Test struct {
207                         Enable bool
208                         Users  map[string]TestUser
209                 }
210                 LoginCluster         string
211                 RemoteTokenRefresh   Duration
212                 TokenLifetime        Duration
213                 TrustedClients       map[URL]struct{}
214                 TrustPrivateNetworks bool
215                 IssueTrustedTokens   bool
216         }
217         Mail struct {
218                 MailchimpAPIKey                string
219                 MailchimpListID                string
220                 SendUserSetupNotificationEmail bool
221                 IssueReporterEmailFrom         string
222                 IssueReporterEmailTo           string
223                 SupportEmailAddress            string
224                 EmailFrom                      string
225         }
226         SystemLogs struct {
227                 LogLevel                  string
228                 Format                    string
229                 MaxRequestLogParamsSize   int
230                 RequestQueueDumpDirectory string
231         }
232         TLS struct {
233                 Certificate string
234                 Key         string
235                 Insecure    bool
236                 ACME        struct {
237                         Server string
238                 }
239         }
240         Users struct {
241                 ActivatedUsersAreVisibleToOthers      bool
242                 AnonymousUserToken                    string
243                 AdminNotifierEmailFrom                string
244                 AutoAdminFirstUser                    bool
245                 AutoAdminUserWithEmail                string
246                 AutoSetupNewUsers                     bool
247                 AutoSetupNewUsersWithRepository       bool
248                 AutoSetupNewUsersWithVmUUID           string
249                 AutoSetupUsernameBlacklist            StringSet
250                 EmailSubjectPrefix                    string
251                 NewInactiveUserNotificationRecipients StringSet
252                 NewUserNotificationRecipients         StringSet
253                 NewUsersAreActive                     bool
254                 UserNotifierEmailFrom                 string
255                 UserNotifierEmailBcc                  StringSet
256                 UserProfileNotificationAddress        string
257                 PreferDomainForUsername               string
258                 UserSetupMailText                     string
259                 RoleGroupsVisibleToAll                bool
260                 CanCreateRoleGroups                   bool
261                 ActivityLoggingPeriod                 Duration
262                 SyncIgnoredGroups                     []string
263                 SyncRequiredGroups                    []string
264                 SyncUserAccounts                      bool
265                 SyncUserAPITokens                     bool
266                 SyncUserGroups                        bool
267                 SyncUserSSHKeys                       bool
268         }
269         StorageClasses map[string]StorageClassConfig
270         Volumes        map[string]Volume
271         Workbench      struct {
272                 ActivationContactLink            string
273                 APIClientConnectTimeout          Duration
274                 APIClientReceiveTimeout          Duration
275                 APIResponseCompression           bool
276                 ApplicationMimetypesWithViewIcon StringSet
277                 ArvadosDocsite                   string
278                 ArvadosPublicDataDocURL          string
279                 DefaultOpenIdPrefix              string
280                 DisableSharingURLsUI             bool
281                 EnableGettingStartedPopup        bool
282                 EnablePublicProjectsPage         bool
283                 FileViewersConfigURL             string
284                 LogViewerMaxBytes                ByteSize
285                 MultiSiteSearch                  string
286                 ProfilingEnabled                 bool
287                 Repositories                     bool
288                 RepositoryCache                  string
289                 RunningJobLogRecordsToFetch      int
290                 SecretKeyBase                    string
291                 ShowRecentCollectionsOnDashboard bool
292                 ShowUserAgreementInline          bool
293                 ShowUserNotifications            bool
294                 SiteName                         string
295                 Theme                            string
296                 UserProfileFormFields            map[string]struct {
297                         Type                 string
298                         FormFieldTitle       string
299                         FormFieldDescription string
300                         Required             bool
301                         Position             int
302                         Options              map[string]struct{}
303                 }
304                 UserProfileFormMessage string
305                 WelcomePageHTML        string
306                 InactivePageHTML       string
307                 SSHHelpPageHTML        string
308                 SSHHelpHostSuffix      string
309                 IdleTimeout            Duration
310                 BannerUUID             string
311         }
312 }
313
314 type StorageClassConfig struct {
315         Default  bool
316         Priority int
317 }
318
319 type Volume struct {
320         AccessViaHosts         map[URL]VolumeAccess
321         ReadOnly               bool
322         AllowTrashWhenReadOnly bool
323         Replication            int
324         StorageClasses         map[string]bool
325         Driver                 string
326         DriverParameters       json.RawMessage
327 }
328
329 type S3VolumeDriverParameters struct {
330         IAMRole            string
331         AccessKeyID        string
332         SecretAccessKey    string
333         Endpoint           string
334         Region             string
335         Bucket             string
336         LocationConstraint bool
337         V2Signature        bool
338         IndexPageSize      int
339         ConnectTimeout     Duration
340         ReadTimeout        Duration
341         RaceWindow         Duration
342         UnsafeDelete       bool
343         PrefixLength       int
344 }
345
346 type AzureVolumeDriverParameters struct {
347         StorageAccountName   string
348         StorageAccountKey    string
349         StorageBaseURL       string
350         ContainerName        string
351         RequestTimeout       Duration
352         ListBlobsRetryDelay  Duration
353         ListBlobsMaxAttempts int
354 }
355
356 type DirectoryVolumeDriverParameters struct {
357         Root      string
358         Serialize bool
359 }
360
361 type VolumeAccess struct {
362         ReadOnly bool
363 }
364
365 type Services struct {
366         Composer       Service
367         Controller     Service
368         DispatchCloud  Service
369         DispatchLSF    Service
370         DispatchSLURM  Service
371         GitHTTP        Service
372         GitSSH         Service
373         Health         Service
374         Keepbalance    Service
375         Keepproxy      Service
376         Keepstore      Service
377         RailsAPI       Service
378         WebDAVDownload Service
379         WebDAV         Service
380         WebShell       Service
381         Websocket      Service
382         Workbench1     Service
383         Workbench2     Service
384 }
385
386 type Service struct {
387         InternalURLs map[URL]ServiceInstance
388         ExternalURL  URL
389 }
390
391 type TestUser struct {
392         Email    string
393         Password string
394 }
395
396 // URL is a url.URL that is also usable as a JSON key/value.
397 type URL url.URL
398
399 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
400 // used as a JSON key/value.
401 func (su *URL) UnmarshalText(text []byte) error {
402         u, err := url.Parse(string(text))
403         if err == nil {
404                 *su = URL(*u)
405                 if su.Path == "" && su.Host != "" {
406                         // http://example really means http://example/
407                         su.Path = "/"
408                 }
409         }
410         return err
411 }
412
413 func (su URL) MarshalText() ([]byte, error) {
414         return []byte(su.String()), nil
415 }
416
417 func (su URL) String() string {
418         return (*url.URL)(&su).String()
419 }
420
421 type TLSVersion uint16
422
423 func (v TLSVersion) MarshalText() ([]byte, error) {
424         switch v {
425         case 0:
426                 return []byte{}, nil
427         case tls.VersionTLS10:
428                 return []byte("1.0"), nil
429         case tls.VersionTLS11:
430                 return []byte("1.1"), nil
431         case tls.VersionTLS12:
432                 return []byte("1.2"), nil
433         case tls.VersionTLS13:
434                 return []byte("1.3"), nil
435         default:
436                 return nil, fmt.Errorf("unsupported TLSVersion %x", v)
437         }
438 }
439
440 func (v *TLSVersion) UnmarshalJSON(text []byte) error {
441         if len(text) > 0 && text[0] == '"' {
442                 var s string
443                 err := json.Unmarshal(text, &s)
444                 if err != nil {
445                         return err
446                 }
447                 text = []byte(s)
448         }
449         switch string(text) {
450         case "":
451                 *v = 0
452         case "1.0":
453                 *v = tls.VersionTLS10
454         case "1.1":
455                 *v = tls.VersionTLS11
456         case "1.2":
457                 *v = tls.VersionTLS12
458         case "1.3":
459                 *v = tls.VersionTLS13
460         default:
461                 return fmt.Errorf("unsupported TLSVersion %q", text)
462         }
463         return nil
464 }
465
466 type ServiceInstance struct {
467         ListenURL  URL
468         Rendezvous string `json:",omitempty"`
469 }
470
471 type PostgreSQL struct {
472         Connection     PostgreSQLConnection
473         ConnectionPool int
474 }
475
476 type PostgreSQLConnection map[string]string
477
478 type RemoteCluster struct {
479         Host          string
480         Proxy         bool
481         Scheme        string
482         Insecure      bool
483         ActivateUsers bool
484 }
485
486 type CUDAFeatures struct {
487         DriverVersion      string
488         HardwareCapability string
489         DeviceCount        int
490 }
491
492 type InstanceType struct {
493         Name            string `json:"-"`
494         ProviderType    string
495         VCPUs           int
496         RAM             ByteSize
497         Scratch         ByteSize `json:"-"`
498         IncludedScratch ByteSize
499         AddedScratch    ByteSize
500         Price           float64
501         Preemptible     bool
502         CUDA            CUDAFeatures
503 }
504
505 type ContainersConfig struct {
506         CloudVMs                      CloudVMsConfig
507         CrunchRunCommand              string
508         CrunchRunArgumentsList        []string
509         DefaultKeepCacheRAM           ByteSize
510         DispatchPrivateKey            string
511         LogReuseDecisions             bool
512         MaxDispatchAttempts           int
513         MaxRetryAttempts              int
514         MinRetryPeriod                Duration
515         ReserveExtraRAM               ByteSize
516         StaleLockTimeout              Duration
517         SupportedDockerImageFormats   StringSet
518         AlwaysUsePreemptibleInstances bool
519         PreemptiblePriceFactor        float64
520         RuntimeEngine                 string
521         LocalKeepBlobBuffersPerVCPU   int
522         LocalKeepLogsToContainerLog   string
523
524         JobsAPI struct {
525                 Enable         string
526                 GitInternalDir string
527         }
528         Logging struct {
529                 MaxAge                       Duration
530                 SweepInterval                Duration
531                 LogBytesPerEvent             int
532                 LogSecondsBetweenEvents      Duration
533                 LogThrottlePeriod            Duration
534                 LogThrottleBytes             int
535                 LogThrottleLines             int
536                 LimitLogBytesPerJob          int
537                 LogPartialLineThrottlePeriod Duration
538                 LogUpdatePeriod              Duration
539                 LogUpdateSize                ByteSize
540         }
541         ShellAccess struct {
542                 Admin bool
543                 User  bool
544         }
545         SLURM struct {
546                 PrioritySpread             int64
547                 SbatchArgumentsList        []string
548                 SbatchEnvironmentVariables map[string]string
549                 Managed                    struct {
550                         DNSServerConfDir       string
551                         DNSServerConfTemplate  string
552                         DNSServerReloadCommand string
553                         DNSServerUpdateCommand string
554                         ComputeNodeDomain      string
555                         ComputeNodeNameservers StringSet
556                         AssignNodeHostname     string
557                 }
558         }
559         LSF struct {
560                 BsubSudoUser      string
561                 BsubArgumentsList []string
562                 BsubCUDAArguments []string
563         }
564 }
565
566 type CloudVMsConfig struct {
567         Enable bool
568
569         BootProbeCommand               string
570         InstanceInitCommand            string
571         DeployRunnerBinary             string
572         DeployPublicKey                bool
573         ImageID                        string
574         MaxCloudOpsPerSecond           int
575         MaxProbesPerSecond             int
576         MaxConcurrentInstanceCreateOps int
577         MaxInstances                   int
578         InitialQuotaEstimate           int
579         SupervisorFraction             float64
580         PollInterval                   Duration
581         ProbeInterval                  Duration
582         SSHPort                        string
583         SyncInterval                   Duration
584         TimeoutBooting                 Duration
585         TimeoutIdle                    Duration
586         TimeoutProbe                   Duration
587         TimeoutShutdown                Duration
588         TimeoutSignal                  Duration
589         TimeoutStaleRunLock            Duration
590         TimeoutTERM                    Duration
591         ResourceTags                   map[string]string
592         TagKeyPrefix                   string
593
594         Driver           string
595         DriverParameters json.RawMessage
596 }
597
598 type InstanceTypeMap map[string]InstanceType
599
600 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
601
602 // UnmarshalJSON does special handling of InstanceTypes:
603 //
604 // - populate computed fields (Name and Scratch)
605 //
606 // - error out if InstancesTypes are populated as an array, which was
607 // deprecated in Arvados 1.2.0
608 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
609         fixup := func(t InstanceType) (InstanceType, error) {
610                 if t.ProviderType == "" {
611                         t.ProviderType = t.Name
612                 }
613                 // If t.Scratch is set in the configuration file, it will be ignored and overwritten.
614                 // It will also generate a "deprecated or unknown config entry" warning.
615                 t.Scratch = t.IncludedScratch + t.AddedScratch
616                 return t, nil
617         }
618
619         if len(data) > 0 && data[0] == '[' {
620                 return fmt.Errorf("InstanceTypes must be specified as a map, not an array, see https://doc.arvados.org/admin/config.html")
621         }
622         var hash map[string]InstanceType
623         err := json.Unmarshal(data, &hash)
624         if err != nil {
625                 return err
626         }
627         // Fill in Name field (and ProviderType field, if not
628         // specified) using hash key.
629         *it = InstanceTypeMap(hash)
630         for name, t := range *it {
631                 t.Name = name
632                 t, err := fixup(t)
633                 if err != nil {
634                         return err
635                 }
636                 (*it)[name] = t
637         }
638         return nil
639 }
640
641 type StringSet map[string]struct{}
642
643 // UnmarshalJSON handles old config files that provide an array of
644 // instance types instead of a hash.
645 func (ss *StringSet) UnmarshalJSON(data []byte) error {
646         if len(data) > 0 && data[0] == '[' {
647                 var arr []string
648                 err := json.Unmarshal(data, &arr)
649                 if err != nil {
650                         return err
651                 }
652                 if len(arr) == 0 {
653                         *ss = nil
654                         return nil
655                 }
656                 *ss = make(map[string]struct{}, len(arr))
657                 for _, t := range arr {
658                         (*ss)[t] = struct{}{}
659                 }
660                 return nil
661         }
662         var hash map[string]struct{}
663         err := json.Unmarshal(data, &hash)
664         if err != nil {
665                 return err
666         }
667         *ss = make(map[string]struct{}, len(hash))
668         for t := range hash {
669                 (*ss)[t] = struct{}{}
670         }
671
672         return nil
673 }
674
675 type ServiceName string
676
677 const (
678         ServiceNameController    ServiceName = "arvados-controller"
679         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
680         ServiceNameDispatchLSF   ServiceName = "arvados-dispatch-lsf"
681         ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm"
682         ServiceNameGitHTTP       ServiceName = "arvados-git-httpd"
683         ServiceNameHealth        ServiceName = "arvados-health"
684         ServiceNameKeepbalance   ServiceName = "keep-balance"
685         ServiceNameKeepproxy     ServiceName = "keepproxy"
686         ServiceNameKeepstore     ServiceName = "keepstore"
687         ServiceNameKeepweb       ServiceName = "keep-web"
688         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
689         ServiceNameWebsocket     ServiceName = "arvados-ws"
690         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
691         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
692 )
693
694 // Map returns all services as a map, suitable for iterating over all
695 // services or looking up a service by name.
696 func (svcs Services) Map() map[ServiceName]Service {
697         return map[ServiceName]Service{
698                 ServiceNameController:    svcs.Controller,
699                 ServiceNameDispatchCloud: svcs.DispatchCloud,
700                 ServiceNameDispatchLSF:   svcs.DispatchLSF,
701                 ServiceNameDispatchSLURM: svcs.DispatchSLURM,
702                 ServiceNameGitHTTP:       svcs.GitHTTP,
703                 ServiceNameHealth:        svcs.Health,
704                 ServiceNameKeepbalance:   svcs.Keepbalance,
705                 ServiceNameKeepproxy:     svcs.Keepproxy,
706                 ServiceNameKeepstore:     svcs.Keepstore,
707                 ServiceNameKeepweb:       svcs.WebDAV,
708                 ServiceNameRailsAPI:      svcs.RailsAPI,
709                 ServiceNameWebsocket:     svcs.Websocket,
710                 ServiceNameWorkbench1:    svcs.Workbench1,
711                 ServiceNameWorkbench2:    svcs.Workbench2,
712         }
713 }