319fa1a38f4a5e1e936315dbacb241f279c04aa6
[arvados.git] / sdk / go / arvados / config.go
1 // Copyright (C) The Arvados Authors. All rights reserved.
2 //
3 // SPDX-License-Identifier: Apache-2.0
4
5 package arvados
6
7 import (
8         "encoding/json"
9         "errors"
10         "fmt"
11         "net/url"
12         "os"
13         "time"
14
15         "git.arvados.org/arvados.git/sdk/go/config"
16 )
17
18 var DefaultConfigFile = func() string {
19         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
20                 return path
21         }
22         return "/etc/arvados/config.yml"
23 }()
24
25 type Config struct {
26         Clusters         map[string]Cluster
27         AutoReloadConfig bool
28         SourceTimestamp  time.Time
29         SourceSHA256     string
30 }
31
32 // GetConfig returns the current system config, loading it from
33 // configFile if needed.
34 func GetConfig(configFile string) (*Config, error) {
35         var cfg Config
36         err := config.LoadFile(&cfg, configFile)
37         return &cfg, err
38 }
39
40 // GetCluster returns the cluster ID and config for the given
41 // cluster, or the default/only configured cluster if clusterID is "".
42 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
43         if clusterID == "" {
44                 if len(sc.Clusters) == 0 {
45                         return nil, fmt.Errorf("no clusters configured")
46                 } else if len(sc.Clusters) > 1 {
47                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
48                 } else {
49                         for id, cc := range sc.Clusters {
50                                 cc.ClusterID = id
51                                 return &cc, nil
52                         }
53                 }
54         }
55         cc, ok := sc.Clusters[clusterID]
56         if !ok {
57                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
58         }
59         cc.ClusterID = clusterID
60         return &cc, nil
61 }
62
63 type WebDAVCacheConfig struct {
64         TTL                  Duration
65         UUIDTTL              Duration
66         MaxBlockEntries      int
67         MaxCollectionEntries int
68         MaxCollectionBytes   int64
69         MaxUUIDEntries       int
70         MaxSessions          int
71 }
72
73 type UploadDownloadPermission struct {
74         Upload   bool
75         Download bool
76 }
77
78 type UploadDownloadRolePermissions struct {
79         User  UploadDownloadPermission
80         Admin UploadDownloadPermission
81 }
82
83 type ManagedProperties map[string]struct {
84         Value     interface{}
85         Function  string
86         Protected bool
87 }
88
89 type Cluster struct {
90         ClusterID       string `json:"-"`
91         ManagementToken string
92         SystemRootToken string
93         Services        Services
94         InstanceTypes   InstanceTypeMap
95         Containers      ContainersConfig
96         RemoteClusters  map[string]RemoteCluster
97         PostgreSQL      PostgreSQL
98
99         API struct {
100                 AsyncPermissionsUpdateInterval   Duration
101                 DisabledAPIs                     StringSet
102                 MaxIndexDatabaseRead             int
103                 MaxItemsPerResponse              int
104                 MaxConcurrentRequests            int
105                 MaxKeepBlobBuffers               int
106                 MaxRequestAmplification          int
107                 MaxRequestSize                   int
108                 MaxTokenLifetime                 Duration
109                 RequestTimeout                   Duration
110                 SendTimeout                      Duration
111                 WebsocketClientEventQueue        int
112                 WebsocketServerEventQueue        int
113                 KeepServiceRequestTimeout        Duration
114                 VocabularyPath                   string
115                 FreezeProjectRequiresDescription bool
116                 FreezeProjectRequiresProperties  StringSet
117                 UnfreezeProjectRequiresAdmin     bool
118         }
119         AuditLogs struct {
120                 MaxAge             Duration
121                 MaxDeleteBatch     int
122                 UnloggedAttributes StringSet
123         }
124         Collections struct {
125                 BlobSigning                  bool
126                 BlobSigningKey               string
127                 BlobSigningTTL               Duration
128                 BlobTrash                    bool
129                 BlobTrashLifetime            Duration
130                 BlobTrashCheckInterval       Duration
131                 BlobTrashConcurrency         int
132                 BlobDeleteConcurrency        int
133                 BlobReplicateConcurrency     int
134                 CollectionVersioning         bool
135                 DefaultTrashLifetime         Duration
136                 DefaultReplication           int
137                 ManagedProperties            ManagedProperties
138                 PreserveVersionIfIdle        Duration
139                 TrashSweepInterval           Duration
140                 TrustAllContent              bool
141                 ForwardSlashNameSubstitution string
142                 S3FolderObjects              bool
143
144                 BlobMissingReport        string
145                 BalancePeriod            Duration
146                 BalanceCollectionBatch   int
147                 BalanceCollectionBuffers int
148                 BalanceTimeout           Duration
149                 BalanceUpdateLimit       int
150
151                 WebDAVCache WebDAVCacheConfig
152
153                 KeepproxyPermission UploadDownloadRolePermissions
154                 WebDAVPermission    UploadDownloadRolePermissions
155                 WebDAVLogEvents     bool
156         }
157         Git struct {
158                 GitCommand   string
159                 GitoliteHome string
160                 Repositories string
161         }
162         Login struct {
163                 LDAP struct {
164                         Enable             bool
165                         URL                URL
166                         StartTLS           bool
167                         InsecureTLS        bool
168                         StripDomain        string
169                         AppendDomain       string
170                         SearchAttribute    string
171                         SearchBindUser     string
172                         SearchBindPassword string
173                         SearchBase         string
174                         SearchFilters      string
175                         EmailAttribute     string
176                         UsernameAttribute  string
177                 }
178                 Google struct {
179                         Enable                          bool
180                         ClientID                        string
181                         ClientSecret                    string
182                         AlternateEmailAddresses         bool
183                         AuthenticationRequestParameters map[string]string
184                 }
185                 OpenIDConnect struct {
186                         Enable                          bool
187                         Issuer                          string
188                         ClientID                        string
189                         ClientSecret                    string
190                         EmailClaim                      string
191                         EmailVerifiedClaim              string
192                         UsernameClaim                   string
193                         AcceptAccessToken               bool
194                         AcceptAccessTokenScope          string
195                         AuthenticationRequestParameters map[string]string
196                 }
197                 PAM struct {
198                         Enable             bool
199                         Service            string
200                         DefaultEmailDomain string
201                 }
202                 Test struct {
203                         Enable bool
204                         Users  map[string]TestUser
205                 }
206                 LoginCluster       string
207                 RemoteTokenRefresh Duration
208                 TokenLifetime      Duration
209                 TrustedClients     map[string]struct{}
210                 IssueTrustedTokens bool
211         }
212         Mail struct {
213                 MailchimpAPIKey                string
214                 MailchimpListID                string
215                 SendUserSetupNotificationEmail bool
216                 IssueReporterEmailFrom         string
217                 IssueReporterEmailTo           string
218                 SupportEmailAddress            string
219                 EmailFrom                      string
220         }
221         SystemLogs struct {
222                 LogLevel                string
223                 Format                  string
224                 MaxRequestLogParamsSize int
225         }
226         TLS struct {
227                 Certificate string
228                 Key         string
229                 Insecure    bool
230         }
231         Users struct {
232                 ActivatedUsersAreVisibleToOthers      bool
233                 AnonymousUserToken                    string
234                 AdminNotifierEmailFrom                string
235                 AutoAdminFirstUser                    bool
236                 AutoAdminUserWithEmail                string
237                 AutoSetupNewUsers                     bool
238                 AutoSetupNewUsersWithRepository       bool
239                 AutoSetupNewUsersWithVmUUID           string
240                 AutoSetupUsernameBlacklist            StringSet
241                 EmailSubjectPrefix                    string
242                 NewInactiveUserNotificationRecipients StringSet
243                 NewUserNotificationRecipients         StringSet
244                 NewUsersAreActive                     bool
245                 UserNotifierEmailFrom                 string
246                 UserNotifierEmailBcc                  StringSet
247                 UserProfileNotificationAddress        string
248                 PreferDomainForUsername               string
249                 UserSetupMailText                     string
250                 RoleGroupsVisibleToAll                bool
251         }
252         StorageClasses map[string]StorageClassConfig
253         Volumes        map[string]Volume
254         Workbench      struct {
255                 ActivationContactLink            string
256                 APIClientConnectTimeout          Duration
257                 APIClientReceiveTimeout          Duration
258                 APIResponseCompression           bool
259                 ApplicationMimetypesWithViewIcon StringSet
260                 ArvadosDocsite                   string
261                 ArvadosPublicDataDocURL          string
262                 DefaultOpenIdPrefix              string
263                 EnableGettingStartedPopup        bool
264                 EnablePublicProjectsPage         bool
265                 FileViewersConfigURL             string
266                 LogViewerMaxBytes                ByteSize
267                 MultiSiteSearch                  string
268                 ProfilingEnabled                 bool
269                 Repositories                     bool
270                 RepositoryCache                  string
271                 RunningJobLogRecordsToFetch      int
272                 SecretKeyBase                    string
273                 ShowRecentCollectionsOnDashboard bool
274                 ShowUserAgreementInline          bool
275                 ShowUserNotifications            bool
276                 SiteName                         string
277                 Theme                            string
278                 UserProfileFormFields            map[string]struct {
279                         Type                 string
280                         FormFieldTitle       string
281                         FormFieldDescription string
282                         Required             bool
283                         Position             int
284                         Options              map[string]struct{}
285                 }
286                 UserProfileFormMessage string
287                 WelcomePageHTML        string
288                 InactivePageHTML       string
289                 SSHHelpPageHTML        string
290                 SSHHelpHostSuffix      string
291                 IdleTimeout            Duration
292         }
293 }
294
295 type StorageClassConfig struct {
296         Default  bool
297         Priority int
298 }
299
300 type Volume struct {
301         AccessViaHosts   map[URL]VolumeAccess
302         ReadOnly         bool
303         Replication      int
304         StorageClasses   map[string]bool
305         Driver           string
306         DriverParameters json.RawMessage
307 }
308
309 type S3VolumeDriverParameters struct {
310         IAMRole            string
311         AccessKeyID        string
312         SecretAccessKey    string
313         Endpoint           string
314         Region             string
315         Bucket             string
316         LocationConstraint bool
317         V2Signature        bool
318         UseAWSS3v2Driver   bool
319         IndexPageSize      int
320         ConnectTimeout     Duration
321         ReadTimeout        Duration
322         RaceWindow         Duration
323         UnsafeDelete       bool
324         PrefixLength       int
325 }
326
327 type AzureVolumeDriverParameters struct {
328         StorageAccountName   string
329         StorageAccountKey    string
330         StorageBaseURL       string
331         ContainerName        string
332         RequestTimeout       Duration
333         ListBlobsRetryDelay  Duration
334         ListBlobsMaxAttempts int
335 }
336
337 type DirectoryVolumeDriverParameters struct {
338         Root      string
339         Serialize bool
340 }
341
342 type VolumeAccess struct {
343         ReadOnly bool
344 }
345
346 type Services struct {
347         Composer       Service
348         Controller     Service
349         DispatchCloud  Service
350         DispatchLSF    Service
351         DispatchSLURM  Service
352         GitHTTP        Service
353         GitSSH         Service
354         Health         Service
355         Keepbalance    Service
356         Keepproxy      Service
357         Keepstore      Service
358         RailsAPI       Service
359         WebDAVDownload Service
360         WebDAV         Service
361         WebShell       Service
362         Websocket      Service
363         Workbench1     Service
364         Workbench2     Service
365 }
366
367 type Service struct {
368         InternalURLs map[URL]ServiceInstance
369         ExternalURL  URL
370 }
371
372 type TestUser struct {
373         Email    string
374         Password string
375 }
376
377 // URL is a url.URL that is also usable as a JSON key/value.
378 type URL url.URL
379
380 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
381 // used as a JSON key/value.
382 func (su *URL) UnmarshalText(text []byte) error {
383         u, err := url.Parse(string(text))
384         if err == nil {
385                 *su = URL(*u)
386                 if su.Path == "" && su.Host != "" {
387                         // http://example really means http://example/
388                         su.Path = "/"
389                 }
390         }
391         return err
392 }
393
394 func (su URL) MarshalText() ([]byte, error) {
395         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
396 }
397
398 func (su URL) String() string {
399         return (*url.URL)(&su).String()
400 }
401
402 type ServiceInstance struct {
403         Rendezvous string `json:",omitempty"`
404 }
405
406 type PostgreSQL struct {
407         Connection     PostgreSQLConnection
408         ConnectionPool int
409 }
410
411 type PostgreSQLConnection map[string]string
412
413 type RemoteCluster struct {
414         Host          string
415         Proxy         bool
416         Scheme        string
417         Insecure      bool
418         ActivateUsers bool
419 }
420
421 type CUDAFeatures struct {
422         DriverVersion      string
423         HardwareCapability string
424         DeviceCount        int
425 }
426
427 type InstanceType struct {
428         Name            string `json:"-"`
429         ProviderType    string
430         VCPUs           int
431         RAM             ByteSize
432         Scratch         ByteSize `json:"-"`
433         IncludedScratch ByteSize
434         AddedScratch    ByteSize
435         Price           float64
436         Preemptible     bool
437         CUDA            CUDAFeatures
438 }
439
440 type ContainersConfig struct {
441         CloudVMs                      CloudVMsConfig
442         CrunchRunCommand              string
443         CrunchRunArgumentsList        []string
444         DefaultKeepCacheRAM           ByteSize
445         DispatchPrivateKey            string
446         LogReuseDecisions             bool
447         MaxComputeVMs                 int
448         MaxDispatchAttempts           int
449         MaxRetryAttempts              int
450         MinRetryPeriod                Duration
451         ReserveExtraRAM               ByteSize
452         StaleLockTimeout              Duration
453         SupportedDockerImageFormats   StringSet
454         AlwaysUsePreemptibleInstances bool
455         PreemptiblePriceFactor        float64
456         RuntimeEngine                 string
457         LocalKeepBlobBuffersPerVCPU   int
458         LocalKeepLogsToContainerLog   string
459
460         JobsAPI struct {
461                 Enable         string
462                 GitInternalDir string
463         }
464         Logging struct {
465                 MaxAge                       Duration
466                 LogBytesPerEvent             int
467                 LogSecondsBetweenEvents      Duration
468                 LogThrottlePeriod            Duration
469                 LogThrottleBytes             int
470                 LogThrottleLines             int
471                 LimitLogBytesPerJob          int
472                 LogPartialLineThrottlePeriod Duration
473                 LogUpdatePeriod              Duration
474                 LogUpdateSize                ByteSize
475         }
476         ShellAccess struct {
477                 Admin bool
478                 User  bool
479         }
480         SLURM struct {
481                 PrioritySpread             int64
482                 SbatchArgumentsList        []string
483                 SbatchEnvironmentVariables map[string]string
484                 Managed                    struct {
485                         DNSServerConfDir       string
486                         DNSServerConfTemplate  string
487                         DNSServerReloadCommand string
488                         DNSServerUpdateCommand string
489                         ComputeNodeDomain      string
490                         ComputeNodeNameservers StringSet
491                         AssignNodeHostname     string
492                 }
493         }
494         LSF struct {
495                 BsubSudoUser      string
496                 BsubArgumentsList []string
497                 BsubCUDAArguments []string
498         }
499 }
500
501 type CloudVMsConfig struct {
502         Enable bool
503
504         BootProbeCommand               string
505         DeployRunnerBinary             string
506         ImageID                        string
507         MaxCloudOpsPerSecond           int
508         MaxProbesPerSecond             int
509         MaxConcurrentInstanceCreateOps int
510         PollInterval                   Duration
511         ProbeInterval                  Duration
512         SSHPort                        string
513         SyncInterval                   Duration
514         TimeoutBooting                 Duration
515         TimeoutIdle                    Duration
516         TimeoutProbe                   Duration
517         TimeoutShutdown                Duration
518         TimeoutSignal                  Duration
519         TimeoutStaleRunLock            Duration
520         TimeoutTERM                    Duration
521         ResourceTags                   map[string]string
522         TagKeyPrefix                   string
523
524         Driver           string
525         DriverParameters json.RawMessage
526 }
527
528 type InstanceTypeMap map[string]InstanceType
529
530 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
531
532 // UnmarshalJSON does special handling of InstanceTypes:
533 // * populate computed fields (Name and Scratch)
534 // * error out if InstancesTypes are populated as an array, which was
535 //   deprecated in Arvados 1.2.0
536 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
537         fixup := func(t InstanceType) (InstanceType, error) {
538                 if t.ProviderType == "" {
539                         t.ProviderType = t.Name
540                 }
541                 // If t.Scratch is set in the configuration file, it will be ignored and overwritten.
542                 // It will also generate a "deprecated or unknown config entry" warning.
543                 t.Scratch = t.IncludedScratch + t.AddedScratch
544                 return t, nil
545         }
546
547         if len(data) > 0 && data[0] == '[' {
548                 return fmt.Errorf("InstanceTypes must be specified as a map, not an array, see https://doc.arvados.org/admin/config.html")
549         }
550         var hash map[string]InstanceType
551         err := json.Unmarshal(data, &hash)
552         if err != nil {
553                 return err
554         }
555         // Fill in Name field (and ProviderType field, if not
556         // specified) using hash key.
557         *it = InstanceTypeMap(hash)
558         for name, t := range *it {
559                 t.Name = name
560                 t, err := fixup(t)
561                 if err != nil {
562                         return err
563                 }
564                 (*it)[name] = t
565         }
566         return nil
567 }
568
569 type StringSet map[string]struct{}
570
571 // UnmarshalJSON handles old config files that provide an array of
572 // instance types instead of a hash.
573 func (ss *StringSet) UnmarshalJSON(data []byte) error {
574         if len(data) > 0 && data[0] == '[' {
575                 var arr []string
576                 err := json.Unmarshal(data, &arr)
577                 if err != nil {
578                         return err
579                 }
580                 if len(arr) == 0 {
581                         *ss = nil
582                         return nil
583                 }
584                 *ss = make(map[string]struct{}, len(arr))
585                 for _, t := range arr {
586                         (*ss)[t] = struct{}{}
587                 }
588                 return nil
589         }
590         var hash map[string]struct{}
591         err := json.Unmarshal(data, &hash)
592         if err != nil {
593                 return err
594         }
595         *ss = make(map[string]struct{}, len(hash))
596         for t := range hash {
597                 (*ss)[t] = struct{}{}
598         }
599
600         return nil
601 }
602
603 type ServiceName string
604
605 const (
606         ServiceNameController    ServiceName = "arvados-controller"
607         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
608         ServiceNameDispatchLSF   ServiceName = "arvados-dispatch-lsf"
609         ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm"
610         ServiceNameGitHTTP       ServiceName = "arvados-git-httpd"
611         ServiceNameHealth        ServiceName = "arvados-health"
612         ServiceNameKeepbalance   ServiceName = "keep-balance"
613         ServiceNameKeepproxy     ServiceName = "keepproxy"
614         ServiceNameKeepstore     ServiceName = "keepstore"
615         ServiceNameKeepweb       ServiceName = "keep-web"
616         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
617         ServiceNameWebsocket     ServiceName = "arvados-ws"
618         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
619         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
620 )
621
622 // Map returns all services as a map, suitable for iterating over all
623 // services or looking up a service by name.
624 func (svcs Services) Map() map[ServiceName]Service {
625         return map[ServiceName]Service{
626                 ServiceNameController:    svcs.Controller,
627                 ServiceNameDispatchCloud: svcs.DispatchCloud,
628                 ServiceNameDispatchLSF:   svcs.DispatchLSF,
629                 ServiceNameDispatchSLURM: svcs.DispatchSLURM,
630                 ServiceNameGitHTTP:       svcs.GitHTTP,
631                 ServiceNameHealth:        svcs.Health,
632                 ServiceNameKeepbalance:   svcs.Keepbalance,
633                 ServiceNameKeepproxy:     svcs.Keepproxy,
634                 ServiceNameKeepstore:     svcs.Keepstore,
635                 ServiceNameKeepweb:       svcs.WebDAV,
636                 ServiceNameRailsAPI:      svcs.RailsAPI,
637                 ServiceNameWebsocket:     svcs.Websocket,
638                 ServiceNameWorkbench1:    svcs.Workbench1,
639                 ServiceNameWorkbench2:    svcs.Workbench2,
640         }
641 }