6a71078bb150a8b33b6d0c98fe587735f3b64c39
[arvados.git] / sdk / go / arvados / config.go
1 // Copyright (C) The Arvados Authors. All rights reserved.
2 //
3 // SPDX-License-Identifier: Apache-2.0
4
5 package arvados
6
7 import (
8         "crypto/tls"
9         "encoding/json"
10         "errors"
11         "fmt"
12         "net/url"
13         "os"
14         "time"
15
16         "git.arvados.org/arvados.git/sdk/go/config"
17 )
18
19 var DefaultConfigFile = func() string {
20         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
21                 return path
22         }
23         return "/etc/arvados/config.yml"
24 }()
25
26 type Config struct {
27         Clusters         map[string]Cluster
28         AutoReloadConfig bool
29         SourceTimestamp  time.Time
30         SourceSHA256     string
31 }
32
33 // GetConfig returns the current system config, loading it from
34 // configFile if needed.
35 func GetConfig(configFile string) (*Config, error) {
36         var cfg Config
37         err := config.LoadFile(&cfg, configFile)
38         return &cfg, err
39 }
40
41 // GetCluster returns the cluster ID and config for the given
42 // cluster, or the default/only configured cluster if clusterID is "".
43 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
44         if clusterID == "" {
45                 if len(sc.Clusters) == 0 {
46                         return nil, fmt.Errorf("no clusters configured")
47                 } else if len(sc.Clusters) > 1 {
48                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
49                 } else {
50                         for id, cc := range sc.Clusters {
51                                 cc.ClusterID = id
52                                 return &cc, nil
53                         }
54                 }
55         }
56         cc, ok := sc.Clusters[clusterID]
57         if !ok {
58                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
59         }
60         cc.ClusterID = clusterID
61         return &cc, nil
62 }
63
64 type WebDAVCacheConfig struct {
65         TTL                Duration
66         MaxBlockEntries    int
67         MaxCollectionBytes int64
68         MaxSessions        int
69 }
70
71 type UploadDownloadPermission struct {
72         Upload   bool
73         Download bool
74 }
75
76 type UploadDownloadRolePermissions struct {
77         User  UploadDownloadPermission
78         Admin UploadDownloadPermission
79 }
80
81 type ManagedProperties map[string]struct {
82         Value     interface{}
83         Function  string
84         Protected bool
85 }
86
87 type Cluster struct {
88         ClusterID       string `json:"-"`
89         ManagementToken string
90         SystemRootToken string
91         Services        Services
92         InstanceTypes   InstanceTypeMap
93         Containers      ContainersConfig
94         RemoteClusters  map[string]RemoteCluster
95         PostgreSQL      PostgreSQL
96
97         API struct {
98                 AsyncPermissionsUpdateInterval   Duration
99                 DisabledAPIs                     StringSet
100                 MaxIndexDatabaseRead             int
101                 MaxItemsPerResponse              int
102                 MaxConcurrentRailsRequests       int
103                 MaxConcurrentRequests            int
104                 MaxQueuedRequests                int
105                 MaxQueueTimeForLockRequests      Duration
106                 LogCreateRequestFraction         float64
107                 MaxKeepBlobBuffers               int
108                 MaxRequestAmplification          int
109                 MaxRequestSize                   int
110                 MaxTokenLifetime                 Duration
111                 RequestTimeout                   Duration
112                 SendTimeout                      Duration
113                 WebsocketClientEventQueue        int
114                 WebsocketServerEventQueue        int
115                 KeepServiceRequestTimeout        Duration
116                 VocabularyPath                   string
117                 FreezeProjectRequiresDescription bool
118                 FreezeProjectRequiresProperties  StringSet
119                 UnfreezeProjectRequiresAdmin     bool
120                 LockBeforeUpdate                 bool
121         }
122         AuditLogs struct {
123                 MaxAge             Duration
124                 MaxDeleteBatch     int
125                 UnloggedAttributes StringSet
126         }
127         Collections struct {
128                 BlobSigning                  bool
129                 BlobSigningKey               string
130                 BlobSigningTTL               Duration
131                 BlobTrash                    bool
132                 BlobTrashLifetime            Duration
133                 BlobTrashCheckInterval       Duration
134                 BlobTrashConcurrency         int
135                 BlobDeleteConcurrency        int
136                 BlobReplicateConcurrency     int
137                 CollectionVersioning         bool
138                 DefaultTrashLifetime         Duration
139                 DefaultReplication           int
140                 ManagedProperties            ManagedProperties
141                 PreserveVersionIfIdle        Duration
142                 TrashSweepInterval           Duration
143                 TrustAllContent              bool
144                 ForwardSlashNameSubstitution string
145                 S3FolderObjects              bool
146
147                 BlobMissingReport        string
148                 BalancePeriod            Duration
149                 BalanceCollectionBatch   int
150                 BalanceCollectionBuffers int
151                 BalanceTimeout           Duration
152                 BalanceUpdateLimit       int
153                 BalancePullLimit         int
154                 BalanceTrashLimit        int
155
156                 WebDAVCache WebDAVCacheConfig
157
158                 KeepproxyPermission UploadDownloadRolePermissions
159                 WebDAVPermission    UploadDownloadRolePermissions
160                 WebDAVLogEvents     bool
161         }
162         Git struct {
163                 GitCommand   string
164                 GitoliteHome string
165                 Repositories string
166         }
167         Login struct {
168                 LDAP struct {
169                         Enable             bool
170                         URL                URL
171                         StartTLS           bool
172                         InsecureTLS        bool
173                         MinTLSVersion      TLSVersion
174                         StripDomain        string
175                         AppendDomain       string
176                         SearchAttribute    string
177                         SearchBindUser     string
178                         SearchBindPassword string
179                         SearchBase         string
180                         SearchFilters      string
181                         EmailAttribute     string
182                         UsernameAttribute  string
183                 }
184                 Google struct {
185                         Enable                          bool
186                         ClientID                        string
187                         ClientSecret                    string
188                         AlternateEmailAddresses         bool
189                         AuthenticationRequestParameters map[string]string
190                 }
191                 OpenIDConnect struct {
192                         Enable                          bool
193                         Issuer                          string
194                         ClientID                        string
195                         ClientSecret                    string
196                         EmailClaim                      string
197                         EmailVerifiedClaim              string
198                         UsernameClaim                   string
199                         AcceptAccessToken               bool
200                         AcceptAccessTokenScope          string
201                         AuthenticationRequestParameters map[string]string
202                 }
203                 PAM struct {
204                         Enable             bool
205                         Service            string
206                         DefaultEmailDomain string
207                 }
208                 Test struct {
209                         Enable bool
210                         Users  map[string]TestUser
211                 }
212                 LoginCluster         string
213                 RemoteTokenRefresh   Duration
214                 TokenLifetime        Duration
215                 TrustedClients       map[URL]struct{}
216                 TrustPrivateNetworks bool
217                 IssueTrustedTokens   bool
218         }
219         Mail struct {
220                 MailchimpAPIKey                string
221                 MailchimpListID                string
222                 SendUserSetupNotificationEmail bool
223                 IssueReporterEmailFrom         string
224                 IssueReporterEmailTo           string
225                 SupportEmailAddress            string
226                 EmailFrom                      string
227         }
228         SystemLogs struct {
229                 LogLevel                  string
230                 Format                    string
231                 MaxRequestLogParamsSize   int
232                 RequestQueueDumpDirectory string
233         }
234         TLS struct {
235                 Certificate string
236                 Key         string
237                 Insecure    bool
238                 ACME        struct {
239                         Server string
240                 }
241         }
242         Users struct {
243                 ActivatedUsersAreVisibleToOthers      bool
244                 AnonymousUserToken                    string
245                 AdminNotifierEmailFrom                string
246                 AutoAdminFirstUser                    bool
247                 AutoAdminUserWithEmail                string
248                 AutoSetupNewUsers                     bool
249                 AutoSetupNewUsersWithRepository       bool
250                 AutoSetupNewUsersWithVmUUID           string
251                 AutoSetupUsernameBlacklist            StringSet
252                 EmailSubjectPrefix                    string
253                 NewInactiveUserNotificationRecipients StringSet
254                 NewUserNotificationRecipients         StringSet
255                 NewUsersAreActive                     bool
256                 UserNotifierEmailFrom                 string
257                 UserNotifierEmailBcc                  StringSet
258                 UserProfileNotificationAddress        string
259                 PreferDomainForUsername               string
260                 UserSetupMailText                     string
261                 RoleGroupsVisibleToAll                bool
262                 CanCreateRoleGroups                   bool
263                 ActivityLoggingPeriod                 Duration
264                 SyncIgnoredGroups                     []string
265                 SyncRequiredGroups                    []string
266                 SyncUserAccounts                      bool
267                 SyncUserAPITokens                     bool
268                 SyncUserGroups                        bool
269                 SyncUserSSHKeys                       bool
270         }
271         StorageClasses map[string]StorageClassConfig
272         Volumes        map[string]Volume
273         Workbench      struct {
274                 ActivationContactLink            string
275                 APIClientConnectTimeout          Duration
276                 APIClientReceiveTimeout          Duration
277                 APIResponseCompression           bool
278                 ApplicationMimetypesWithViewIcon StringSet
279                 ArvadosDocsite                   string
280                 ArvadosPublicDataDocURL          string
281                 DefaultOpenIdPrefix              string
282                 DisableSharingURLsUI             bool
283                 EnableGettingStartedPopup        bool
284                 EnablePublicProjectsPage         bool
285                 FileViewersConfigURL             string
286                 LogViewerMaxBytes                ByteSize
287                 MultiSiteSearch                  string
288                 ProfilingEnabled                 bool
289                 Repositories                     bool
290                 RepositoryCache                  string
291                 RunningJobLogRecordsToFetch      int
292                 SecretKeyBase                    string
293                 ShowRecentCollectionsOnDashboard bool
294                 ShowUserAgreementInline          bool
295                 ShowUserNotifications            bool
296                 SiteName                         string
297                 Theme                            string
298                 UserProfileFormFields            map[string]struct {
299                         Type                 string
300                         FormFieldTitle       string
301                         FormFieldDescription string
302                         Required             bool
303                         Position             int
304                         Options              map[string]struct{}
305                 }
306                 UserProfileFormMessage string
307                 WelcomePageHTML        string
308                 InactivePageHTML       string
309                 SSHHelpPageHTML        string
310                 SSHHelpHostSuffix      string
311                 IdleTimeout            Duration
312                 BannerUUID             string
313         }
314 }
315
316 type StorageClassConfig struct {
317         Default  bool
318         Priority int
319 }
320
321 type Volume struct {
322         AccessViaHosts   map[URL]VolumeAccess
323         ReadOnly         bool
324         Replication      int
325         StorageClasses   map[string]bool
326         Driver           string
327         DriverParameters json.RawMessage
328 }
329
330 type S3VolumeDriverParameters struct {
331         IAMRole            string
332         AccessKeyID        string
333         SecretAccessKey    string
334         Endpoint           string
335         Region             string
336         Bucket             string
337         LocationConstraint bool
338         V2Signature        bool
339         IndexPageSize      int
340         ConnectTimeout     Duration
341         ReadTimeout        Duration
342         RaceWindow         Duration
343         UnsafeDelete       bool
344         PrefixLength       int
345 }
346
347 type AzureVolumeDriverParameters struct {
348         StorageAccountName   string
349         StorageAccountKey    string
350         StorageBaseURL       string
351         ContainerName        string
352         RequestTimeout       Duration
353         ListBlobsRetryDelay  Duration
354         ListBlobsMaxAttempts int
355 }
356
357 type DirectoryVolumeDriverParameters struct {
358         Root      string
359         Serialize bool
360 }
361
362 type VolumeAccess struct {
363         ReadOnly bool
364 }
365
366 type Services struct {
367         Composer       Service
368         Controller     Service
369         DispatchCloud  Service
370         DispatchLSF    Service
371         DispatchSLURM  Service
372         GitHTTP        Service
373         GitSSH         Service
374         Health         Service
375         Keepbalance    Service
376         Keepproxy      Service
377         Keepstore      Service
378         RailsAPI       Service
379         WebDAVDownload Service
380         WebDAV         Service
381         WebShell       Service
382         Websocket      Service
383         Workbench1     Service
384         Workbench2     Service
385 }
386
387 type Service struct {
388         InternalURLs map[URL]ServiceInstance
389         ExternalURL  URL
390 }
391
392 type TestUser struct {
393         Email    string
394         Password string
395 }
396
397 // URL is a url.URL that is also usable as a JSON key/value.
398 type URL url.URL
399
400 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
401 // used as a JSON key/value.
402 func (su *URL) UnmarshalText(text []byte) error {
403         u, err := url.Parse(string(text))
404         if err == nil {
405                 *su = URL(*u)
406                 if su.Path == "" && su.Host != "" {
407                         // http://example really means http://example/
408                         su.Path = "/"
409                 }
410         }
411         return err
412 }
413
414 func (su URL) MarshalText() ([]byte, error) {
415         return []byte(su.String()), nil
416 }
417
418 func (su URL) String() string {
419         return (*url.URL)(&su).String()
420 }
421
422 type TLSVersion uint16
423
424 func (v TLSVersion) MarshalText() ([]byte, error) {
425         switch v {
426         case 0:
427                 return []byte{}, nil
428         case tls.VersionTLS10:
429                 return []byte("1.0"), nil
430         case tls.VersionTLS11:
431                 return []byte("1.1"), nil
432         case tls.VersionTLS12:
433                 return []byte("1.2"), nil
434         case tls.VersionTLS13:
435                 return []byte("1.3"), nil
436         default:
437                 return nil, fmt.Errorf("unsupported TLSVersion %x", v)
438         }
439 }
440
441 func (v *TLSVersion) UnmarshalJSON(text []byte) error {
442         if len(text) > 0 && text[0] == '"' {
443                 var s string
444                 err := json.Unmarshal(text, &s)
445                 if err != nil {
446                         return err
447                 }
448                 text = []byte(s)
449         }
450         switch string(text) {
451         case "":
452                 *v = 0
453         case "1.0":
454                 *v = tls.VersionTLS10
455         case "1.1":
456                 *v = tls.VersionTLS11
457         case "1.2":
458                 *v = tls.VersionTLS12
459         case "1.3":
460                 *v = tls.VersionTLS13
461         default:
462                 return fmt.Errorf("unsupported TLSVersion %q", text)
463         }
464         return nil
465 }
466
467 type ServiceInstance struct {
468         ListenURL  URL
469         Rendezvous string `json:",omitempty"`
470 }
471
472 type PostgreSQL struct {
473         Connection     PostgreSQLConnection
474         ConnectionPool int
475 }
476
477 type PostgreSQLConnection map[string]string
478
479 type RemoteCluster struct {
480         Host          string
481         Proxy         bool
482         Scheme        string
483         Insecure      bool
484         ActivateUsers bool
485 }
486
487 type CUDAFeatures struct {
488         DriverVersion      string
489         HardwareCapability string
490         DeviceCount        int
491 }
492
493 type InstanceType struct {
494         Name            string `json:"-"`
495         ProviderType    string
496         VCPUs           int
497         RAM             ByteSize
498         Scratch         ByteSize `json:"-"`
499         IncludedScratch ByteSize
500         AddedScratch    ByteSize
501         Price           float64
502         Preemptible     bool
503         CUDA            CUDAFeatures
504 }
505
506 type ContainersConfig struct {
507         CloudVMs                      CloudVMsConfig
508         CrunchRunCommand              string
509         CrunchRunArgumentsList        []string
510         DefaultKeepCacheRAM           ByteSize
511         DispatchPrivateKey            string
512         LogReuseDecisions             bool
513         MaxDispatchAttempts           int
514         MaxRetryAttempts              int
515         MinRetryPeriod                Duration
516         ReserveExtraRAM               ByteSize
517         StaleLockTimeout              Duration
518         SupportedDockerImageFormats   StringSet
519         AlwaysUsePreemptibleInstances bool
520         PreemptiblePriceFactor        float64
521         MaximumPriceFactor            float64
522         RuntimeEngine                 string
523         LocalKeepBlobBuffersPerVCPU   int
524         LocalKeepLogsToContainerLog   string
525
526         JobsAPI struct {
527                 Enable         string
528                 GitInternalDir string
529         }
530         Logging struct {
531                 MaxAge                       Duration
532                 SweepInterval                Duration
533                 LogBytesPerEvent             int
534                 LogSecondsBetweenEvents      Duration
535                 LogThrottlePeriod            Duration
536                 LogThrottleBytes             int
537                 LogThrottleLines             int
538                 LimitLogBytesPerJob          int
539                 LogPartialLineThrottlePeriod Duration
540                 LogUpdatePeriod              Duration
541                 LogUpdateSize                ByteSize
542         }
543         ShellAccess struct {
544                 Admin bool
545                 User  bool
546         }
547         SLURM struct {
548                 PrioritySpread             int64
549                 SbatchArgumentsList        []string
550                 SbatchEnvironmentVariables map[string]string
551                 Managed                    struct {
552                         DNSServerConfDir       string
553                         DNSServerConfTemplate  string
554                         DNSServerReloadCommand string
555                         DNSServerUpdateCommand string
556                         ComputeNodeDomain      string
557                         ComputeNodeNameservers StringSet
558                         AssignNodeHostname     string
559                 }
560         }
561         LSF struct {
562                 BsubSudoUser      string
563                 BsubArgumentsList []string
564                 BsubCUDAArguments []string
565         }
566 }
567
568 type CloudVMsConfig struct {
569         Enable bool
570
571         BootProbeCommand               string
572         InstanceInitCommand            string
573         DeployRunnerBinary             string
574         DeployPublicKey                bool
575         ImageID                        string
576         MaxCloudOpsPerSecond           int
577         MaxProbesPerSecond             int
578         MaxConcurrentInstanceCreateOps int
579         MaxInstances                   int
580         InitialQuotaEstimate           int
581         SupervisorFraction             float64
582         PollInterval                   Duration
583         ProbeInterval                  Duration
584         SSHPort                        string
585         SyncInterval                   Duration
586         TimeoutBooting                 Duration
587         TimeoutIdle                    Duration
588         TimeoutProbe                   Duration
589         TimeoutShutdown                Duration
590         TimeoutSignal                  Duration
591         TimeoutStaleRunLock            Duration
592         TimeoutTERM                    Duration
593         ResourceTags                   map[string]string
594         TagKeyPrefix                   string
595
596         Driver           string
597         DriverParameters json.RawMessage
598 }
599
600 type InstanceTypeMap map[string]InstanceType
601
602 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
603
604 // UnmarshalJSON does special handling of InstanceTypes:
605 //
606 // - populate computed fields (Name and Scratch)
607 //
608 // - error out if InstancesTypes are populated as an array, which was
609 // deprecated in Arvados 1.2.0
610 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
611         fixup := func(t InstanceType) (InstanceType, error) {
612                 if t.ProviderType == "" {
613                         t.ProviderType = t.Name
614                 }
615                 // If t.Scratch is set in the configuration file, it will be ignored and overwritten.
616                 // It will also generate a "deprecated or unknown config entry" warning.
617                 t.Scratch = t.IncludedScratch + t.AddedScratch
618                 return t, nil
619         }
620
621         if len(data) > 0 && data[0] == '[' {
622                 return fmt.Errorf("InstanceTypes must be specified as a map, not an array, see https://doc.arvados.org/admin/config.html")
623         }
624         var hash map[string]InstanceType
625         err := json.Unmarshal(data, &hash)
626         if err != nil {
627                 return err
628         }
629         // Fill in Name field (and ProviderType field, if not
630         // specified) using hash key.
631         *it = InstanceTypeMap(hash)
632         for name, t := range *it {
633                 t.Name = name
634                 t, err := fixup(t)
635                 if err != nil {
636                         return err
637                 }
638                 (*it)[name] = t
639         }
640         return nil
641 }
642
643 type StringSet map[string]struct{}
644
645 // UnmarshalJSON handles old config files that provide an array of
646 // instance types instead of a hash.
647 func (ss *StringSet) UnmarshalJSON(data []byte) error {
648         if len(data) > 0 && data[0] == '[' {
649                 var arr []string
650                 err := json.Unmarshal(data, &arr)
651                 if err != nil {
652                         return err
653                 }
654                 if len(arr) == 0 {
655                         *ss = nil
656                         return nil
657                 }
658                 *ss = make(map[string]struct{}, len(arr))
659                 for _, t := range arr {
660                         (*ss)[t] = struct{}{}
661                 }
662                 return nil
663         }
664         var hash map[string]struct{}
665         err := json.Unmarshal(data, &hash)
666         if err != nil {
667                 return err
668         }
669         *ss = make(map[string]struct{}, len(hash))
670         for t := range hash {
671                 (*ss)[t] = struct{}{}
672         }
673
674         return nil
675 }
676
677 type ServiceName string
678
679 const (
680         ServiceNameController    ServiceName = "arvados-controller"
681         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
682         ServiceNameDispatchLSF   ServiceName = "arvados-dispatch-lsf"
683         ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm"
684         ServiceNameGitHTTP       ServiceName = "arvados-git-httpd"
685         ServiceNameHealth        ServiceName = "arvados-health"
686         ServiceNameKeepbalance   ServiceName = "keep-balance"
687         ServiceNameKeepproxy     ServiceName = "keepproxy"
688         ServiceNameKeepstore     ServiceName = "keepstore"
689         ServiceNameKeepweb       ServiceName = "keep-web"
690         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
691         ServiceNameWebsocket     ServiceName = "arvados-ws"
692         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
693         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
694 )
695
696 // Map returns all services as a map, suitable for iterating over all
697 // services or looking up a service by name.
698 func (svcs Services) Map() map[ServiceName]Service {
699         return map[ServiceName]Service{
700                 ServiceNameController:    svcs.Controller,
701                 ServiceNameDispatchCloud: svcs.DispatchCloud,
702                 ServiceNameDispatchLSF:   svcs.DispatchLSF,
703                 ServiceNameDispatchSLURM: svcs.DispatchSLURM,
704                 ServiceNameGitHTTP:       svcs.GitHTTP,
705                 ServiceNameHealth:        svcs.Health,
706                 ServiceNameKeepbalance:   svcs.Keepbalance,
707                 ServiceNameKeepproxy:     svcs.Keepproxy,
708                 ServiceNameKeepstore:     svcs.Keepstore,
709                 ServiceNameKeepweb:       svcs.WebDAV,
710                 ServiceNameRailsAPI:      svcs.RailsAPI,
711                 ServiceNameWebsocket:     svcs.Websocket,
712                 ServiceNameWorkbench1:    svcs.Workbench1,
713                 ServiceNameWorkbench2:    svcs.Workbench2,
714         }
715 }