18368: Add BannerURL config option
[arvados.git] / sdk / go / arvados / config.go
1 // Copyright (C) The Arvados Authors. All rights reserved.
2 //
3 // SPDX-License-Identifier: Apache-2.0
4
5 package arvados
6
7 import (
8         "encoding/json"
9         "errors"
10         "fmt"
11         "net/url"
12         "os"
13         "time"
14
15         "git.arvados.org/arvados.git/sdk/go/config"
16 )
17
18 var DefaultConfigFile = func() string {
19         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
20                 return path
21         }
22         return "/etc/arvados/config.yml"
23 }()
24
25 type Config struct {
26         Clusters         map[string]Cluster
27         AutoReloadConfig bool
28         SourceTimestamp  time.Time
29         SourceSHA256     string
30 }
31
32 // GetConfig returns the current system config, loading it from
33 // configFile if needed.
34 func GetConfig(configFile string) (*Config, error) {
35         var cfg Config
36         err := config.LoadFile(&cfg, configFile)
37         return &cfg, err
38 }
39
40 // GetCluster returns the cluster ID and config for the given
41 // cluster, or the default/only configured cluster if clusterID is "".
42 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
43         if clusterID == "" {
44                 if len(sc.Clusters) == 0 {
45                         return nil, fmt.Errorf("no clusters configured")
46                 } else if len(sc.Clusters) > 1 {
47                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
48                 } else {
49                         for id, cc := range sc.Clusters {
50                                 cc.ClusterID = id
51                                 return &cc, nil
52                         }
53                 }
54         }
55         cc, ok := sc.Clusters[clusterID]
56         if !ok {
57                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
58         }
59         cc.ClusterID = clusterID
60         return &cc, nil
61 }
62
63 type WebDAVCacheConfig struct {
64         TTL                Duration
65         MaxBlockEntries    int
66         MaxCollectionBytes int64
67         MaxSessions        int
68 }
69
70 type UploadDownloadPermission struct {
71         Upload   bool
72         Download bool
73 }
74
75 type UploadDownloadRolePermissions struct {
76         User  UploadDownloadPermission
77         Admin UploadDownloadPermission
78 }
79
80 type ManagedProperties map[string]struct {
81         Value     interface{}
82         Function  string
83         Protected bool
84 }
85
86 type Cluster struct {
87         ClusterID       string `json:"-"`
88         ManagementToken string
89         SystemRootToken string
90         Services        Services
91         InstanceTypes   InstanceTypeMap
92         Containers      ContainersConfig
93         RemoteClusters  map[string]RemoteCluster
94         PostgreSQL      PostgreSQL
95
96         API struct {
97                 AsyncPermissionsUpdateInterval   Duration
98                 DisabledAPIs                     StringSet
99                 MaxIndexDatabaseRead             int
100                 MaxItemsPerResponse              int
101                 MaxConcurrentRequests            int
102                 MaxKeepBlobBuffers               int
103                 MaxRequestAmplification          int
104                 MaxRequestSize                   int
105                 MaxTokenLifetime                 Duration
106                 RequestTimeout                   Duration
107                 SendTimeout                      Duration
108                 WebsocketClientEventQueue        int
109                 WebsocketServerEventQueue        int
110                 KeepServiceRequestTimeout        Duration
111                 VocabularyPath                   string
112                 FreezeProjectRequiresDescription bool
113                 FreezeProjectRequiresProperties  StringSet
114                 UnfreezeProjectRequiresAdmin     bool
115         }
116         AuditLogs struct {
117                 MaxAge             Duration
118                 MaxDeleteBatch     int
119                 UnloggedAttributes StringSet
120         }
121         Collections struct {
122                 BlobSigning                  bool
123                 BlobSigningKey               string
124                 BlobSigningTTL               Duration
125                 BlobTrash                    bool
126                 BlobTrashLifetime            Duration
127                 BlobTrashCheckInterval       Duration
128                 BlobTrashConcurrency         int
129                 BlobDeleteConcurrency        int
130                 BlobReplicateConcurrency     int
131                 CollectionVersioning         bool
132                 DefaultTrashLifetime         Duration
133                 DefaultReplication           int
134                 ManagedProperties            ManagedProperties
135                 PreserveVersionIfIdle        Duration
136                 TrashSweepInterval           Duration
137                 TrustAllContent              bool
138                 ForwardSlashNameSubstitution string
139                 S3FolderObjects              bool
140
141                 BlobMissingReport        string
142                 BalancePeriod            Duration
143                 BalanceCollectionBatch   int
144                 BalanceCollectionBuffers int
145                 BalanceTimeout           Duration
146                 BalanceUpdateLimit       int
147
148                 WebDAVCache WebDAVCacheConfig
149
150                 KeepproxyPermission UploadDownloadRolePermissions
151                 WebDAVPermission    UploadDownloadRolePermissions
152                 WebDAVLogEvents     bool
153         }
154         Git struct {
155                 GitCommand   string
156                 GitoliteHome string
157                 Repositories string
158         }
159         Login struct {
160                 LDAP struct {
161                         Enable             bool
162                         URL                URL
163                         StartTLS           bool
164                         InsecureTLS        bool
165                         StripDomain        string
166                         AppendDomain       string
167                         SearchAttribute    string
168                         SearchBindUser     string
169                         SearchBindPassword string
170                         SearchBase         string
171                         SearchFilters      string
172                         EmailAttribute     string
173                         UsernameAttribute  string
174                 }
175                 Google struct {
176                         Enable                          bool
177                         ClientID                        string
178                         ClientSecret                    string
179                         AlternateEmailAddresses         bool
180                         AuthenticationRequestParameters map[string]string
181                 }
182                 OpenIDConnect struct {
183                         Enable                          bool
184                         Issuer                          string
185                         ClientID                        string
186                         ClientSecret                    string
187                         EmailClaim                      string
188                         EmailVerifiedClaim              string
189                         UsernameClaim                   string
190                         AcceptAccessToken               bool
191                         AcceptAccessTokenScope          string
192                         AuthenticationRequestParameters map[string]string
193                 }
194                 PAM struct {
195                         Enable             bool
196                         Service            string
197                         DefaultEmailDomain string
198                 }
199                 Test struct {
200                         Enable bool
201                         Users  map[string]TestUser
202                 }
203                 LoginCluster       string
204                 RemoteTokenRefresh Duration
205                 TokenLifetime      Duration
206                 TrustedClients     map[string]struct{}
207                 IssueTrustedTokens bool
208         }
209         Mail struct {
210                 MailchimpAPIKey                string
211                 MailchimpListID                string
212                 SendUserSetupNotificationEmail bool
213                 IssueReporterEmailFrom         string
214                 IssueReporterEmailTo           string
215                 SupportEmailAddress            string
216                 EmailFrom                      string
217         }
218         SystemLogs struct {
219                 LogLevel                string
220                 Format                  string
221                 MaxRequestLogParamsSize int
222         }
223         TLS struct {
224                 Certificate string
225                 Key         string
226                 Insecure    bool
227                 ACME        struct {
228                         Server string
229                 }
230         }
231         Users struct {
232                 ActivatedUsersAreVisibleToOthers      bool
233                 AnonymousUserToken                    string
234                 AdminNotifierEmailFrom                string
235                 AutoAdminFirstUser                    bool
236                 AutoAdminUserWithEmail                string
237                 AutoSetupNewUsers                     bool
238                 AutoSetupNewUsersWithRepository       bool
239                 AutoSetupNewUsersWithVmUUID           string
240                 AutoSetupUsernameBlacklist            StringSet
241                 EmailSubjectPrefix                    string
242                 NewInactiveUserNotificationRecipients StringSet
243                 NewUserNotificationRecipients         StringSet
244                 NewUsersAreActive                     bool
245                 UserNotifierEmailFrom                 string
246                 UserNotifierEmailBcc                  StringSet
247                 UserProfileNotificationAddress        string
248                 PreferDomainForUsername               string
249                 UserSetupMailText                     string
250                 RoleGroupsVisibleToAll                bool
251                 ActivityLoggingPeriod                 Duration
252         }
253         StorageClasses map[string]StorageClassConfig
254         Volumes        map[string]Volume
255         Workbench      struct {
256                 ActivationContactLink            string
257                 APIClientConnectTimeout          Duration
258                 APIClientReceiveTimeout          Duration
259                 APIResponseCompression           bool
260                 ApplicationMimetypesWithViewIcon StringSet
261                 ArvadosDocsite                   string
262                 ArvadosPublicDataDocURL          string
263                 DefaultOpenIdPrefix              string
264                 DisableSharingURLsUI             bool
265                 EnableGettingStartedPopup        bool
266                 EnablePublicProjectsPage         bool
267                 FileViewersConfigURL             string
268                 LogViewerMaxBytes                ByteSize
269                 MultiSiteSearch                  string
270                 ProfilingEnabled                 bool
271                 Repositories                     bool
272                 RepositoryCache                  string
273                 RunningJobLogRecordsToFetch      int
274                 SecretKeyBase                    string
275                 ShowRecentCollectionsOnDashboard bool
276                 ShowUserAgreementInline          bool
277                 ShowUserNotifications            bool
278                 SiteName                         string
279                 Theme                            string
280                 UserProfileFormFields            map[string]struct {
281                         Type                 string
282                         FormFieldTitle       string
283                         FormFieldDescription string
284                         Required             bool
285                         Position             int
286                         Options              map[string]struct{}
287                 }
288                 UserProfileFormMessage string
289                 WelcomePageHTML        string
290                 InactivePageHTML       string
291                 SSHHelpPageHTML        string
292                 SSHHelpHostSuffix      string
293                 IdleTimeout            Duration
294                 BannerURL              string
295         }
296 }
297
298 type StorageClassConfig struct {
299         Default  bool
300         Priority int
301 }
302
303 type Volume struct {
304         AccessViaHosts   map[URL]VolumeAccess
305         ReadOnly         bool
306         Replication      int
307         StorageClasses   map[string]bool
308         Driver           string
309         DriverParameters json.RawMessage
310 }
311
312 type S3VolumeDriverParameters struct {
313         IAMRole            string
314         AccessKeyID        string
315         SecretAccessKey    string
316         Endpoint           string
317         Region             string
318         Bucket             string
319         LocationConstraint bool
320         V2Signature        bool
321         UseAWSS3v2Driver   bool
322         IndexPageSize      int
323         ConnectTimeout     Duration
324         ReadTimeout        Duration
325         RaceWindow         Duration
326         UnsafeDelete       bool
327         PrefixLength       int
328 }
329
330 type AzureVolumeDriverParameters struct {
331         StorageAccountName   string
332         StorageAccountKey    string
333         StorageBaseURL       string
334         ContainerName        string
335         RequestTimeout       Duration
336         ListBlobsRetryDelay  Duration
337         ListBlobsMaxAttempts int
338 }
339
340 type DirectoryVolumeDriverParameters struct {
341         Root      string
342         Serialize bool
343 }
344
345 type VolumeAccess struct {
346         ReadOnly bool
347 }
348
349 type Services struct {
350         Composer       Service
351         Controller     Service
352         DispatchCloud  Service
353         DispatchLSF    Service
354         DispatchSLURM  Service
355         GitHTTP        Service
356         GitSSH         Service
357         Health         Service
358         Keepbalance    Service
359         Keepproxy      Service
360         Keepstore      Service
361         RailsAPI       Service
362         WebDAVDownload Service
363         WebDAV         Service
364         WebShell       Service
365         Websocket      Service
366         Workbench1     Service
367         Workbench2     Service
368 }
369
370 type Service struct {
371         InternalURLs map[URL]ServiceInstance
372         ExternalURL  URL
373 }
374
375 type TestUser struct {
376         Email    string
377         Password string
378 }
379
380 // URL is a url.URL that is also usable as a JSON key/value.
381 type URL url.URL
382
383 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
384 // used as a JSON key/value.
385 func (su *URL) UnmarshalText(text []byte) error {
386         u, err := url.Parse(string(text))
387         if err == nil {
388                 *su = URL(*u)
389                 if su.Path == "" && su.Host != "" {
390                         // http://example really means http://example/
391                         su.Path = "/"
392                 }
393         }
394         return err
395 }
396
397 func (su URL) MarshalText() ([]byte, error) {
398         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
399 }
400
401 func (su URL) String() string {
402         return (*url.URL)(&su).String()
403 }
404
405 type ServiceInstance struct {
406         ListenURL  URL
407         Rendezvous string `json:",omitempty"`
408 }
409
410 type PostgreSQL struct {
411         Connection     PostgreSQLConnection
412         ConnectionPool int
413 }
414
415 type PostgreSQLConnection map[string]string
416
417 type RemoteCluster struct {
418         Host          string
419         Proxy         bool
420         Scheme        string
421         Insecure      bool
422         ActivateUsers bool
423 }
424
425 type CUDAFeatures struct {
426         DriverVersion      string
427         HardwareCapability string
428         DeviceCount        int
429 }
430
431 type InstanceType struct {
432         Name            string `json:"-"`
433         ProviderType    string
434         VCPUs           int
435         RAM             ByteSize
436         Scratch         ByteSize `json:"-"`
437         IncludedScratch ByteSize
438         AddedScratch    ByteSize
439         Price           float64
440         Preemptible     bool
441         CUDA            CUDAFeatures
442 }
443
444 type ContainersConfig struct {
445         CloudVMs                      CloudVMsConfig
446         CrunchRunCommand              string
447         CrunchRunArgumentsList        []string
448         DefaultKeepCacheRAM           ByteSize
449         DispatchPrivateKey            string
450         LogReuseDecisions             bool
451         MaxComputeVMs                 int
452         MaxDispatchAttempts           int
453         MaxRetryAttempts              int
454         MinRetryPeriod                Duration
455         ReserveExtraRAM               ByteSize
456         StaleLockTimeout              Duration
457         SupportedDockerImageFormats   StringSet
458         AlwaysUsePreemptibleInstances bool
459         PreemptiblePriceFactor        float64
460         RuntimeEngine                 string
461         LocalKeepBlobBuffersPerVCPU   int
462         LocalKeepLogsToContainerLog   string
463
464         JobsAPI struct {
465                 Enable         string
466                 GitInternalDir string
467         }
468         Logging struct {
469                 MaxAge                       Duration
470                 LogBytesPerEvent             int
471                 LogSecondsBetweenEvents      Duration
472                 LogThrottlePeriod            Duration
473                 LogThrottleBytes             int
474                 LogThrottleLines             int
475                 LimitLogBytesPerJob          int
476                 LogPartialLineThrottlePeriod Duration
477                 LogUpdatePeriod              Duration
478                 LogUpdateSize                ByteSize
479         }
480         ShellAccess struct {
481                 Admin bool
482                 User  bool
483         }
484         SLURM struct {
485                 PrioritySpread             int64
486                 SbatchArgumentsList        []string
487                 SbatchEnvironmentVariables map[string]string
488                 Managed                    struct {
489                         DNSServerConfDir       string
490                         DNSServerConfTemplate  string
491                         DNSServerReloadCommand string
492                         DNSServerUpdateCommand string
493                         ComputeNodeDomain      string
494                         ComputeNodeNameservers StringSet
495                         AssignNodeHostname     string
496                 }
497         }
498         LSF struct {
499                 BsubSudoUser      string
500                 BsubArgumentsList []string
501                 BsubCUDAArguments []string
502         }
503 }
504
505 type CloudVMsConfig struct {
506         Enable bool
507
508         BootProbeCommand               string
509         DeployRunnerBinary             string
510         ImageID                        string
511         MaxCloudOpsPerSecond           int
512         MaxProbesPerSecond             int
513         MaxConcurrentInstanceCreateOps int
514         PollInterval                   Duration
515         ProbeInterval                  Duration
516         SSHPort                        string
517         SyncInterval                   Duration
518         TimeoutBooting                 Duration
519         TimeoutIdle                    Duration
520         TimeoutProbe                   Duration
521         TimeoutShutdown                Duration
522         TimeoutSignal                  Duration
523         TimeoutStaleRunLock            Duration
524         TimeoutTERM                    Duration
525         ResourceTags                   map[string]string
526         TagKeyPrefix                   string
527
528         Driver           string
529         DriverParameters json.RawMessage
530 }
531
532 type InstanceTypeMap map[string]InstanceType
533
534 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
535
536 // UnmarshalJSON does special handling of InstanceTypes:
537 // * populate computed fields (Name and Scratch)
538 // * error out if InstancesTypes are populated as an array, which was
539 //   deprecated in Arvados 1.2.0
540 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
541         fixup := func(t InstanceType) (InstanceType, error) {
542                 if t.ProviderType == "" {
543                         t.ProviderType = t.Name
544                 }
545                 // If t.Scratch is set in the configuration file, it will be ignored and overwritten.
546                 // It will also generate a "deprecated or unknown config entry" warning.
547                 t.Scratch = t.IncludedScratch + t.AddedScratch
548                 return t, nil
549         }
550
551         if len(data) > 0 && data[0] == '[' {
552                 return fmt.Errorf("InstanceTypes must be specified as a map, not an array, see https://doc.arvados.org/admin/config.html")
553         }
554         var hash map[string]InstanceType
555         err := json.Unmarshal(data, &hash)
556         if err != nil {
557                 return err
558         }
559         // Fill in Name field (and ProviderType field, if not
560         // specified) using hash key.
561         *it = InstanceTypeMap(hash)
562         for name, t := range *it {
563                 t.Name = name
564                 t, err := fixup(t)
565                 if err != nil {
566                         return err
567                 }
568                 (*it)[name] = t
569         }
570         return nil
571 }
572
573 type StringSet map[string]struct{}
574
575 // UnmarshalJSON handles old config files that provide an array of
576 // instance types instead of a hash.
577 func (ss *StringSet) UnmarshalJSON(data []byte) error {
578         if len(data) > 0 && data[0] == '[' {
579                 var arr []string
580                 err := json.Unmarshal(data, &arr)
581                 if err != nil {
582                         return err
583                 }
584                 if len(arr) == 0 {
585                         *ss = nil
586                         return nil
587                 }
588                 *ss = make(map[string]struct{}, len(arr))
589                 for _, t := range arr {
590                         (*ss)[t] = struct{}{}
591                 }
592                 return nil
593         }
594         var hash map[string]struct{}
595         err := json.Unmarshal(data, &hash)
596         if err != nil {
597                 return err
598         }
599         *ss = make(map[string]struct{}, len(hash))
600         for t := range hash {
601                 (*ss)[t] = struct{}{}
602         }
603
604         return nil
605 }
606
607 type ServiceName string
608
609 const (
610         ServiceNameController    ServiceName = "arvados-controller"
611         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
612         ServiceNameDispatchLSF   ServiceName = "arvados-dispatch-lsf"
613         ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm"
614         ServiceNameGitHTTP       ServiceName = "arvados-git-httpd"
615         ServiceNameHealth        ServiceName = "arvados-health"
616         ServiceNameKeepbalance   ServiceName = "keep-balance"
617         ServiceNameKeepproxy     ServiceName = "keepproxy"
618         ServiceNameKeepstore     ServiceName = "keepstore"
619         ServiceNameKeepweb       ServiceName = "keep-web"
620         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
621         ServiceNameWebsocket     ServiceName = "arvados-ws"
622         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
623         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
624 )
625
626 // Map returns all services as a map, suitable for iterating over all
627 // services or looking up a service by name.
628 func (svcs Services) Map() map[ServiceName]Service {
629         return map[ServiceName]Service{
630                 ServiceNameController:    svcs.Controller,
631                 ServiceNameDispatchCloud: svcs.DispatchCloud,
632                 ServiceNameDispatchLSF:   svcs.DispatchLSF,
633                 ServiceNameDispatchSLURM: svcs.DispatchSLURM,
634                 ServiceNameGitHTTP:       svcs.GitHTTP,
635                 ServiceNameHealth:        svcs.Health,
636                 ServiceNameKeepbalance:   svcs.Keepbalance,
637                 ServiceNameKeepproxy:     svcs.Keepproxy,
638                 ServiceNameKeepstore:     svcs.Keepstore,
639                 ServiceNameKeepweb:       svcs.WebDAV,
640                 ServiceNameRailsAPI:      svcs.RailsAPI,
641                 ServiceNameWebsocket:     svcs.Websocket,
642                 ServiceNameWorkbench1:    svcs.Workbench1,
643                 ServiceNameWorkbench2:    svcs.Workbench2,
644         }
645 }