Merge branch '21126-trash-when-ro'
[arvados.git] / sdk / go / arvados / config.go
1 // Copyright (C) The Arvados Authors. All rights reserved.
2 //
3 // SPDX-License-Identifier: Apache-2.0
4
5 package arvados
6
7 import (
8         "crypto/tls"
9         "encoding/json"
10         "errors"
11         "fmt"
12         "net/url"
13         "os"
14         "time"
15
16         "git.arvados.org/arvados.git/sdk/go/config"
17 )
18
19 var DefaultConfigFile = func() string {
20         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
21                 return path
22         }
23         return "/etc/arvados/config.yml"
24 }()
25
26 type Config struct {
27         Clusters         map[string]Cluster
28         AutoReloadConfig bool
29         SourceTimestamp  time.Time
30         SourceSHA256     string
31 }
32
33 // GetConfig returns the current system config, loading it from
34 // configFile if needed.
35 func GetConfig(configFile string) (*Config, error) {
36         var cfg Config
37         err := config.LoadFile(&cfg, configFile)
38         return &cfg, err
39 }
40
41 // GetCluster returns the cluster ID and config for the given
42 // cluster, or the default/only configured cluster if clusterID is "".
43 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
44         if clusterID == "" {
45                 if len(sc.Clusters) == 0 {
46                         return nil, fmt.Errorf("no clusters configured")
47                 } else if len(sc.Clusters) > 1 {
48                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
49                 } else {
50                         for id, cc := range sc.Clusters {
51                                 cc.ClusterID = id
52                                 return &cc, nil
53                         }
54                 }
55         }
56         cc, ok := sc.Clusters[clusterID]
57         if !ok {
58                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
59         }
60         cc.ClusterID = clusterID
61         return &cc, nil
62 }
63
64 type WebDAVCacheConfig struct {
65         TTL                Duration
66         MaxBlockEntries    int
67         MaxCollectionBytes int64
68         MaxSessions        int
69 }
70
71 type UploadDownloadPermission struct {
72         Upload   bool
73         Download bool
74 }
75
76 type UploadDownloadRolePermissions struct {
77         User  UploadDownloadPermission
78         Admin UploadDownloadPermission
79 }
80
81 type ManagedProperties map[string]struct {
82         Value     interface{}
83         Function  string
84         Protected bool
85 }
86
87 type Cluster struct {
88         ClusterID       string `json:"-"`
89         ManagementToken string
90         SystemRootToken string
91         Services        Services
92         InstanceTypes   InstanceTypeMap
93         Containers      ContainersConfig
94         RemoteClusters  map[string]RemoteCluster
95         PostgreSQL      PostgreSQL
96
97         API struct {
98                 AsyncPermissionsUpdateInterval   Duration
99                 DisabledAPIs                     StringSet
100                 MaxIndexDatabaseRead             int
101                 MaxItemsPerResponse              int
102                 MaxConcurrentRailsRequests       int
103                 MaxConcurrentRequests            int
104                 MaxQueuedRequests                int
105                 MaxQueueTimeForLockRequests      Duration
106                 LogCreateRequestFraction         float64
107                 MaxKeepBlobBuffers               int
108                 MaxRequestAmplification          int
109                 MaxRequestSize                   int
110                 MaxTokenLifetime                 Duration
111                 RequestTimeout                   Duration
112                 SendTimeout                      Duration
113                 WebsocketClientEventQueue        int
114                 WebsocketServerEventQueue        int
115                 KeepServiceRequestTimeout        Duration
116                 VocabularyPath                   string
117                 FreezeProjectRequiresDescription bool
118                 FreezeProjectRequiresProperties  StringSet
119                 UnfreezeProjectRequiresAdmin     bool
120                 LockBeforeUpdate                 bool
121         }
122         AuditLogs struct {
123                 MaxAge             Duration
124                 MaxDeleteBatch     int
125                 UnloggedAttributes StringSet
126         }
127         Collections struct {
128                 BlobSigning                  bool
129                 BlobSigningKey               string
130                 BlobSigningTTL               Duration
131                 BlobTrash                    bool
132                 BlobTrashLifetime            Duration
133                 BlobTrashCheckInterval       Duration
134                 BlobTrashConcurrency         int
135                 BlobDeleteConcurrency        int
136                 BlobReplicateConcurrency     int
137                 CollectionVersioning         bool
138                 DefaultTrashLifetime         Duration
139                 DefaultReplication           int
140                 ManagedProperties            ManagedProperties
141                 PreserveVersionIfIdle        Duration
142                 TrashSweepInterval           Duration
143                 TrustAllContent              bool
144                 ForwardSlashNameSubstitution string
145                 S3FolderObjects              bool
146
147                 BlobMissingReport        string
148                 BalancePeriod            Duration
149                 BalanceCollectionBatch   int
150                 BalanceCollectionBuffers int
151                 BalanceTimeout           Duration
152                 BalanceUpdateLimit       int
153                 BalancePullLimit         int
154                 BalanceTrashLimit        int
155
156                 WebDAVCache WebDAVCacheConfig
157
158                 KeepproxyPermission UploadDownloadRolePermissions
159                 WebDAVPermission    UploadDownloadRolePermissions
160                 WebDAVLogEvents     bool
161         }
162         Git struct {
163                 GitCommand   string
164                 GitoliteHome string
165                 Repositories string
166         }
167         Login struct {
168                 LDAP struct {
169                         Enable             bool
170                         URL                URL
171                         StartTLS           bool
172                         InsecureTLS        bool
173                         MinTLSVersion      TLSVersion
174                         StripDomain        string
175                         AppendDomain       string
176                         SearchAttribute    string
177                         SearchBindUser     string
178                         SearchBindPassword string
179                         SearchBase         string
180                         SearchFilters      string
181                         EmailAttribute     string
182                         UsernameAttribute  string
183                 }
184                 Google struct {
185                         Enable                          bool
186                         ClientID                        string
187                         ClientSecret                    string
188                         AlternateEmailAddresses         bool
189                         AuthenticationRequestParameters map[string]string
190                 }
191                 OpenIDConnect struct {
192                         Enable                          bool
193                         Issuer                          string
194                         ClientID                        string
195                         ClientSecret                    string
196                         EmailClaim                      string
197                         EmailVerifiedClaim              string
198                         UsernameClaim                   string
199                         AcceptAccessToken               bool
200                         AcceptAccessTokenScope          string
201                         AuthenticationRequestParameters map[string]string
202                 }
203                 PAM struct {
204                         Enable             bool
205                         Service            string
206                         DefaultEmailDomain string
207                 }
208                 Test struct {
209                         Enable bool
210                         Users  map[string]TestUser
211                 }
212                 LoginCluster         string
213                 RemoteTokenRefresh   Duration
214                 TokenLifetime        Duration
215                 TrustedClients       map[URL]struct{}
216                 TrustPrivateNetworks bool
217                 IssueTrustedTokens   bool
218         }
219         Mail struct {
220                 MailchimpAPIKey                string
221                 MailchimpListID                string
222                 SendUserSetupNotificationEmail bool
223                 IssueReporterEmailFrom         string
224                 IssueReporterEmailTo           string
225                 SupportEmailAddress            string
226                 EmailFrom                      string
227         }
228         SystemLogs struct {
229                 LogLevel                  string
230                 Format                    string
231                 MaxRequestLogParamsSize   int
232                 RequestQueueDumpDirectory string
233         }
234         TLS struct {
235                 Certificate string
236                 Key         string
237                 Insecure    bool
238                 ACME        struct {
239                         Server string
240                 }
241         }
242         Users struct {
243                 ActivatedUsersAreVisibleToOthers      bool
244                 AnonymousUserToken                    string
245                 AdminNotifierEmailFrom                string
246                 AutoAdminFirstUser                    bool
247                 AutoAdminUserWithEmail                string
248                 AutoSetupNewUsers                     bool
249                 AutoSetupNewUsersWithRepository       bool
250                 AutoSetupNewUsersWithVmUUID           string
251                 AutoSetupUsernameBlacklist            StringSet
252                 EmailSubjectPrefix                    string
253                 NewInactiveUserNotificationRecipients StringSet
254                 NewUserNotificationRecipients         StringSet
255                 NewUsersAreActive                     bool
256                 UserNotifierEmailFrom                 string
257                 UserNotifierEmailBcc                  StringSet
258                 UserProfileNotificationAddress        string
259                 PreferDomainForUsername               string
260                 UserSetupMailText                     string
261                 RoleGroupsVisibleToAll                bool
262                 CanCreateRoleGroups                   bool
263                 ActivityLoggingPeriod                 Duration
264                 SyncIgnoredGroups                     []string
265                 SyncRequiredGroups                    []string
266                 SyncUserAccounts                      bool
267                 SyncUserAPITokens                     bool
268                 SyncUserGroups                        bool
269                 SyncUserSSHKeys                       bool
270         }
271         StorageClasses map[string]StorageClassConfig
272         Volumes        map[string]Volume
273         Workbench      struct {
274                 ActivationContactLink            string
275                 APIClientConnectTimeout          Duration
276                 APIClientReceiveTimeout          Duration
277                 APIResponseCompression           bool
278                 ApplicationMimetypesWithViewIcon StringSet
279                 ArvadosDocsite                   string
280                 ArvadosPublicDataDocURL          string
281                 DefaultOpenIdPrefix              string
282                 DisableSharingURLsUI             bool
283                 EnableGettingStartedPopup        bool
284                 EnablePublicProjectsPage         bool
285                 FileViewersConfigURL             string
286                 LogViewerMaxBytes                ByteSize
287                 MultiSiteSearch                  string
288                 ProfilingEnabled                 bool
289                 Repositories                     bool
290                 RepositoryCache                  string
291                 RunningJobLogRecordsToFetch      int
292                 SecretKeyBase                    string
293                 ShowRecentCollectionsOnDashboard bool
294                 ShowUserAgreementInline          bool
295                 ShowUserNotifications            bool
296                 SiteName                         string
297                 Theme                            string
298                 UserProfileFormFields            map[string]struct {
299                         Type                 string
300                         FormFieldTitle       string
301                         FormFieldDescription string
302                         Required             bool
303                         Position             int
304                         Options              map[string]struct{}
305                 }
306                 UserProfileFormMessage string
307                 WelcomePageHTML        string
308                 InactivePageHTML       string
309                 SSHHelpPageHTML        string
310                 SSHHelpHostSuffix      string
311                 IdleTimeout            Duration
312                 BannerUUID             string
313         }
314 }
315
316 type StorageClassConfig struct {
317         Default  bool
318         Priority int
319 }
320
321 type Volume struct {
322         AccessViaHosts         map[URL]VolumeAccess
323         ReadOnly               bool
324         AllowTrashWhenReadOnly bool
325         Replication            int
326         StorageClasses         map[string]bool
327         Driver                 string
328         DriverParameters       json.RawMessage
329 }
330
331 type S3VolumeDriverParameters struct {
332         IAMRole            string
333         AccessKeyID        string
334         SecretAccessKey    string
335         Endpoint           string
336         Region             string
337         Bucket             string
338         LocationConstraint bool
339         V2Signature        bool
340         IndexPageSize      int
341         ConnectTimeout     Duration
342         ReadTimeout        Duration
343         RaceWindow         Duration
344         UnsafeDelete       bool
345         PrefixLength       int
346 }
347
348 type AzureVolumeDriverParameters struct {
349         StorageAccountName   string
350         StorageAccountKey    string
351         StorageBaseURL       string
352         ContainerName        string
353         RequestTimeout       Duration
354         ListBlobsRetryDelay  Duration
355         ListBlobsMaxAttempts int
356 }
357
358 type DirectoryVolumeDriverParameters struct {
359         Root      string
360         Serialize bool
361 }
362
363 type VolumeAccess struct {
364         ReadOnly bool
365 }
366
367 type Services struct {
368         Composer       Service
369         Controller     Service
370         DispatchCloud  Service
371         DispatchLSF    Service
372         DispatchSLURM  Service
373         GitHTTP        Service
374         GitSSH         Service
375         Health         Service
376         Keepbalance    Service
377         Keepproxy      Service
378         Keepstore      Service
379         RailsAPI       Service
380         WebDAVDownload Service
381         WebDAV         Service
382         WebShell       Service
383         Websocket      Service
384         Workbench1     Service
385         Workbench2     Service
386 }
387
388 type Service struct {
389         InternalURLs map[URL]ServiceInstance
390         ExternalURL  URL
391 }
392
393 type TestUser struct {
394         Email    string
395         Password string
396 }
397
398 // URL is a url.URL that is also usable as a JSON key/value.
399 type URL url.URL
400
401 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
402 // used as a JSON key/value.
403 func (su *URL) UnmarshalText(text []byte) error {
404         u, err := url.Parse(string(text))
405         if err == nil {
406                 *su = URL(*u)
407                 if su.Path == "" && su.Host != "" {
408                         // http://example really means http://example/
409                         su.Path = "/"
410                 }
411         }
412         return err
413 }
414
415 func (su URL) MarshalText() ([]byte, error) {
416         return []byte(su.String()), nil
417 }
418
419 func (su URL) String() string {
420         return (*url.URL)(&su).String()
421 }
422
423 type TLSVersion uint16
424
425 func (v TLSVersion) MarshalText() ([]byte, error) {
426         switch v {
427         case 0:
428                 return []byte{}, nil
429         case tls.VersionTLS10:
430                 return []byte("1.0"), nil
431         case tls.VersionTLS11:
432                 return []byte("1.1"), nil
433         case tls.VersionTLS12:
434                 return []byte("1.2"), nil
435         case tls.VersionTLS13:
436                 return []byte("1.3"), nil
437         default:
438                 return nil, fmt.Errorf("unsupported TLSVersion %x", v)
439         }
440 }
441
442 func (v *TLSVersion) UnmarshalJSON(text []byte) error {
443         if len(text) > 0 && text[0] == '"' {
444                 var s string
445                 err := json.Unmarshal(text, &s)
446                 if err != nil {
447                         return err
448                 }
449                 text = []byte(s)
450         }
451         switch string(text) {
452         case "":
453                 *v = 0
454         case "1.0":
455                 *v = tls.VersionTLS10
456         case "1.1":
457                 *v = tls.VersionTLS11
458         case "1.2":
459                 *v = tls.VersionTLS12
460         case "1.3":
461                 *v = tls.VersionTLS13
462         default:
463                 return fmt.Errorf("unsupported TLSVersion %q", text)
464         }
465         return nil
466 }
467
468 type ServiceInstance struct {
469         ListenURL  URL
470         Rendezvous string `json:",omitempty"`
471 }
472
473 type PostgreSQL struct {
474         Connection     PostgreSQLConnection
475         ConnectionPool int
476 }
477
478 type PostgreSQLConnection map[string]string
479
480 type RemoteCluster struct {
481         Host          string
482         Proxy         bool
483         Scheme        string
484         Insecure      bool
485         ActivateUsers bool
486 }
487
488 type CUDAFeatures struct {
489         DriverVersion      string
490         HardwareCapability string
491         DeviceCount        int
492 }
493
494 type InstanceType struct {
495         Name            string `json:"-"`
496         ProviderType    string
497         VCPUs           int
498         RAM             ByteSize
499         Scratch         ByteSize `json:"-"`
500         IncludedScratch ByteSize
501         AddedScratch    ByteSize
502         Price           float64
503         Preemptible     bool
504         CUDA            CUDAFeatures
505 }
506
507 type ContainersConfig struct {
508         CloudVMs                      CloudVMsConfig
509         CrunchRunCommand              string
510         CrunchRunArgumentsList        []string
511         DefaultKeepCacheRAM           ByteSize
512         DispatchPrivateKey            string
513         LogReuseDecisions             bool
514         MaxDispatchAttempts           int
515         MaxRetryAttempts              int
516         MinRetryPeriod                Duration
517         ReserveExtraRAM               ByteSize
518         StaleLockTimeout              Duration
519         SupportedDockerImageFormats   StringSet
520         AlwaysUsePreemptibleInstances bool
521         PreemptiblePriceFactor        float64
522         MaximumPriceFactor            float64
523         RuntimeEngine                 string
524         LocalKeepBlobBuffersPerVCPU   int
525         LocalKeepLogsToContainerLog   string
526
527         JobsAPI struct {
528                 Enable         string
529                 GitInternalDir string
530         }
531         Logging struct {
532                 MaxAge                       Duration
533                 SweepInterval                Duration
534                 LogBytesPerEvent             int
535                 LogSecondsBetweenEvents      Duration
536                 LogThrottlePeriod            Duration
537                 LogThrottleBytes             int
538                 LogThrottleLines             int
539                 LimitLogBytesPerJob          int
540                 LogPartialLineThrottlePeriod Duration
541                 LogUpdatePeriod              Duration
542                 LogUpdateSize                ByteSize
543         }
544         ShellAccess struct {
545                 Admin bool
546                 User  bool
547         }
548         SLURM struct {
549                 PrioritySpread             int64
550                 SbatchArgumentsList        []string
551                 SbatchEnvironmentVariables map[string]string
552                 Managed                    struct {
553                         DNSServerConfDir       string
554                         DNSServerConfTemplate  string
555                         DNSServerReloadCommand string
556                         DNSServerUpdateCommand string
557                         ComputeNodeDomain      string
558                         ComputeNodeNameservers StringSet
559                         AssignNodeHostname     string
560                 }
561         }
562         LSF struct {
563                 BsubSudoUser      string
564                 BsubArgumentsList []string
565                 BsubCUDAArguments []string
566         }
567 }
568
569 type CloudVMsConfig struct {
570         Enable bool
571
572         BootProbeCommand               string
573         InstanceInitCommand            string
574         DeployRunnerBinary             string
575         DeployPublicKey                bool
576         ImageID                        string
577         MaxCloudOpsPerSecond           int
578         MaxProbesPerSecond             int
579         MaxConcurrentInstanceCreateOps int
580         MaxInstances                   int
581         InitialQuotaEstimate           int
582         SupervisorFraction             float64
583         PollInterval                   Duration
584         ProbeInterval                  Duration
585         SSHPort                        string
586         SyncInterval                   Duration
587         TimeoutBooting                 Duration
588         TimeoutIdle                    Duration
589         TimeoutProbe                   Duration
590         TimeoutShutdown                Duration
591         TimeoutSignal                  Duration
592         TimeoutStaleRunLock            Duration
593         TimeoutTERM                    Duration
594         ResourceTags                   map[string]string
595         TagKeyPrefix                   string
596
597         Driver           string
598         DriverParameters json.RawMessage
599 }
600
601 type InstanceTypeMap map[string]InstanceType
602
603 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
604
605 // UnmarshalJSON does special handling of InstanceTypes:
606 //
607 // - populate computed fields (Name and Scratch)
608 //
609 // - error out if InstancesTypes are populated as an array, which was
610 // deprecated in Arvados 1.2.0
611 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
612         fixup := func(t InstanceType) (InstanceType, error) {
613                 if t.ProviderType == "" {
614                         t.ProviderType = t.Name
615                 }
616                 // If t.Scratch is set in the configuration file, it will be ignored and overwritten.
617                 // It will also generate a "deprecated or unknown config entry" warning.
618                 t.Scratch = t.IncludedScratch + t.AddedScratch
619                 return t, nil
620         }
621
622         if len(data) > 0 && data[0] == '[' {
623                 return fmt.Errorf("InstanceTypes must be specified as a map, not an array, see https://doc.arvados.org/admin/config.html")
624         }
625         var hash map[string]InstanceType
626         err := json.Unmarshal(data, &hash)
627         if err != nil {
628                 return err
629         }
630         // Fill in Name field (and ProviderType field, if not
631         // specified) using hash key.
632         *it = InstanceTypeMap(hash)
633         for name, t := range *it {
634                 t.Name = name
635                 t, err := fixup(t)
636                 if err != nil {
637                         return err
638                 }
639                 (*it)[name] = t
640         }
641         return nil
642 }
643
644 type StringSet map[string]struct{}
645
646 // UnmarshalJSON handles old config files that provide an array of
647 // instance types instead of a hash.
648 func (ss *StringSet) UnmarshalJSON(data []byte) error {
649         if len(data) > 0 && data[0] == '[' {
650                 var arr []string
651                 err := json.Unmarshal(data, &arr)
652                 if err != nil {
653                         return err
654                 }
655                 if len(arr) == 0 {
656                         *ss = nil
657                         return nil
658                 }
659                 *ss = make(map[string]struct{}, len(arr))
660                 for _, t := range arr {
661                         (*ss)[t] = struct{}{}
662                 }
663                 return nil
664         }
665         var hash map[string]struct{}
666         err := json.Unmarshal(data, &hash)
667         if err != nil {
668                 return err
669         }
670         *ss = make(map[string]struct{}, len(hash))
671         for t := range hash {
672                 (*ss)[t] = struct{}{}
673         }
674
675         return nil
676 }
677
678 type ServiceName string
679
680 const (
681         ServiceNameController    ServiceName = "arvados-controller"
682         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
683         ServiceNameDispatchLSF   ServiceName = "arvados-dispatch-lsf"
684         ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm"
685         ServiceNameGitHTTP       ServiceName = "arvados-git-httpd"
686         ServiceNameHealth        ServiceName = "arvados-health"
687         ServiceNameKeepbalance   ServiceName = "keep-balance"
688         ServiceNameKeepproxy     ServiceName = "keepproxy"
689         ServiceNameKeepstore     ServiceName = "keepstore"
690         ServiceNameKeepweb       ServiceName = "keep-web"
691         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
692         ServiceNameWebsocket     ServiceName = "arvados-ws"
693         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
694         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
695 )
696
697 // Map returns all services as a map, suitable for iterating over all
698 // services or looking up a service by name.
699 func (svcs Services) Map() map[ServiceName]Service {
700         return map[ServiceName]Service{
701                 ServiceNameController:    svcs.Controller,
702                 ServiceNameDispatchCloud: svcs.DispatchCloud,
703                 ServiceNameDispatchLSF:   svcs.DispatchLSF,
704                 ServiceNameDispatchSLURM: svcs.DispatchSLURM,
705                 ServiceNameGitHTTP:       svcs.GitHTTP,
706                 ServiceNameHealth:        svcs.Health,
707                 ServiceNameKeepbalance:   svcs.Keepbalance,
708                 ServiceNameKeepproxy:     svcs.Keepproxy,
709                 ServiceNameKeepstore:     svcs.Keepstore,
710                 ServiceNameKeepweb:       svcs.WebDAV,
711                 ServiceNameRailsAPI:      svcs.RailsAPI,
712                 ServiceNameWebsocket:     svcs.Websocket,
713                 ServiceNameWorkbench1:    svcs.Workbench1,
714                 ServiceNameWorkbench2:    svcs.Workbench2,
715         }
716 }