Merge branch '18323-cwl-gpu' refs #18323
[arvados.git] / sdk / go / arvados / config.go
1 // Copyright (C) The Arvados Authors. All rights reserved.
2 //
3 // SPDX-License-Identifier: Apache-2.0
4
5 package arvados
6
7 import (
8         "encoding/json"
9         "errors"
10         "fmt"
11         "net/url"
12         "os"
13
14         "git.arvados.org/arvados.git/sdk/go/config"
15 )
16
17 var DefaultConfigFile = func() string {
18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
19                 return path
20         }
21         return "/etc/arvados/config.yml"
22 }()
23
24 type Config struct {
25         Clusters         map[string]Cluster
26         AutoReloadConfig bool
27 }
28
29 // GetConfig returns the current system config, loading it from
30 // configFile if needed.
31 func GetConfig(configFile string) (*Config, error) {
32         var cfg Config
33         err := config.LoadFile(&cfg, configFile)
34         return &cfg, err
35 }
36
37 // GetCluster returns the cluster ID and config for the given
38 // cluster, or the default/only configured cluster if clusterID is "".
39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
40         if clusterID == "" {
41                 if len(sc.Clusters) == 0 {
42                         return nil, fmt.Errorf("no clusters configured")
43                 } else if len(sc.Clusters) > 1 {
44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
45                 } else {
46                         for id, cc := range sc.Clusters {
47                                 cc.ClusterID = id
48                                 return &cc, nil
49                         }
50                 }
51         }
52         cc, ok := sc.Clusters[clusterID]
53         if !ok {
54                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
55         }
56         cc.ClusterID = clusterID
57         return &cc, nil
58 }
59
60 type WebDAVCacheConfig struct {
61         TTL                  Duration
62         UUIDTTL              Duration
63         MaxBlockEntries      int
64         MaxCollectionEntries int
65         MaxCollectionBytes   int64
66         MaxUUIDEntries       int
67         MaxSessions          int
68 }
69
70 type UploadDownloadPermission struct {
71         Upload   bool
72         Download bool
73 }
74
75 type UploadDownloadRolePermissions struct {
76         User  UploadDownloadPermission
77         Admin UploadDownloadPermission
78 }
79
80 type ManagedProperties map[string]struct {
81         Value     interface{}
82         Function  string
83         Protected bool
84 }
85
86 type Cluster struct {
87         ClusterID       string `json:"-"`
88         ManagementToken string
89         SystemRootToken string
90         Services        Services
91         InstanceTypes   InstanceTypeMap
92         Containers      ContainersConfig
93         RemoteClusters  map[string]RemoteCluster
94         PostgreSQL      PostgreSQL
95
96         API struct {
97                 AsyncPermissionsUpdateInterval Duration
98                 DisabledAPIs                   StringSet
99                 MaxIndexDatabaseRead           int
100                 MaxItemsPerResponse            int
101                 MaxConcurrentRequests          int
102                 MaxKeepBlobBuffers             int
103                 MaxRequestAmplification        int
104                 MaxRequestSize                 int
105                 MaxTokenLifetime               Duration
106                 RequestTimeout                 Duration
107                 SendTimeout                    Duration
108                 WebsocketClientEventQueue      int
109                 WebsocketServerEventQueue      int
110                 KeepServiceRequestTimeout      Duration
111                 VocabularyPath                 string
112         }
113         AuditLogs struct {
114                 MaxAge             Duration
115                 MaxDeleteBatch     int
116                 UnloggedAttributes StringSet
117         }
118         Collections struct {
119                 BlobSigning                  bool
120                 BlobSigningKey               string
121                 BlobSigningTTL               Duration
122                 BlobTrash                    bool
123                 BlobTrashLifetime            Duration
124                 BlobTrashCheckInterval       Duration
125                 BlobTrashConcurrency         int
126                 BlobDeleteConcurrency        int
127                 BlobReplicateConcurrency     int
128                 CollectionVersioning         bool
129                 DefaultTrashLifetime         Duration
130                 DefaultReplication           int
131                 ManagedProperties            ManagedProperties
132                 PreserveVersionIfIdle        Duration
133                 TrashSweepInterval           Duration
134                 TrustAllContent              bool
135                 ForwardSlashNameSubstitution string
136                 S3FolderObjects              bool
137
138                 BlobMissingReport        string
139                 BalancePeriod            Duration
140                 BalanceCollectionBatch   int
141                 BalanceCollectionBuffers int
142                 BalanceTimeout           Duration
143                 BalanceUpdateLimit       int
144
145                 WebDAVCache WebDAVCacheConfig
146
147                 KeepproxyPermission UploadDownloadRolePermissions
148                 WebDAVPermission    UploadDownloadRolePermissions
149                 WebDAVLogEvents     bool
150         }
151         Git struct {
152                 GitCommand   string
153                 GitoliteHome string
154                 Repositories string
155         }
156         Login struct {
157                 LDAP struct {
158                         Enable             bool
159                         URL                URL
160                         StartTLS           bool
161                         InsecureTLS        bool
162                         StripDomain        string
163                         AppendDomain       string
164                         SearchAttribute    string
165                         SearchBindUser     string
166                         SearchBindPassword string
167                         SearchBase         string
168                         SearchFilters      string
169                         EmailAttribute     string
170                         UsernameAttribute  string
171                 }
172                 Google struct {
173                         Enable                          bool
174                         ClientID                        string
175                         ClientSecret                    string
176                         AlternateEmailAddresses         bool
177                         AuthenticationRequestParameters map[string]string
178                 }
179                 OpenIDConnect struct {
180                         Enable                          bool
181                         Issuer                          string
182                         ClientID                        string
183                         ClientSecret                    string
184                         EmailClaim                      string
185                         EmailVerifiedClaim              string
186                         UsernameClaim                   string
187                         AcceptAccessToken               bool
188                         AcceptAccessTokenScope          string
189                         AuthenticationRequestParameters map[string]string
190                 }
191                 PAM struct {
192                         Enable             bool
193                         Service            string
194                         DefaultEmailDomain string
195                 }
196                 Test struct {
197                         Enable bool
198                         Users  map[string]TestUser
199                 }
200                 LoginCluster       string
201                 RemoteTokenRefresh Duration
202                 TokenLifetime      Duration
203                 TrustedClients     map[string]struct{}
204                 IssueTrustedTokens bool
205         }
206         Mail struct {
207                 MailchimpAPIKey                string
208                 MailchimpListID                string
209                 SendUserSetupNotificationEmail bool
210                 IssueReporterEmailFrom         string
211                 IssueReporterEmailTo           string
212                 SupportEmailAddress            string
213                 EmailFrom                      string
214         }
215         SystemLogs struct {
216                 LogLevel                string
217                 Format                  string
218                 MaxRequestLogParamsSize int
219         }
220         TLS struct {
221                 Certificate string
222                 Key         string
223                 Insecure    bool
224         }
225         Users struct {
226                 ActivatedUsersAreVisibleToOthers      bool
227                 AnonymousUserToken                    string
228                 AdminNotifierEmailFrom                string
229                 AutoAdminFirstUser                    bool
230                 AutoAdminUserWithEmail                string
231                 AutoSetupNewUsers                     bool
232                 AutoSetupNewUsersWithRepository       bool
233                 AutoSetupNewUsersWithVmUUID           string
234                 AutoSetupUsernameBlacklist            StringSet
235                 EmailSubjectPrefix                    string
236                 NewInactiveUserNotificationRecipients StringSet
237                 NewUserNotificationRecipients         StringSet
238                 NewUsersAreActive                     bool
239                 UserNotifierEmailFrom                 string
240                 UserNotifierEmailBcc                  StringSet
241                 UserProfileNotificationAddress        string
242                 PreferDomainForUsername               string
243                 UserSetupMailText                     string
244                 RoleGroupsVisibleToAll                bool
245         }
246         StorageClasses map[string]StorageClassConfig
247         Volumes        map[string]Volume
248         Workbench      struct {
249                 ActivationContactLink            string
250                 APIClientConnectTimeout          Duration
251                 APIClientReceiveTimeout          Duration
252                 APIResponseCompression           bool
253                 ApplicationMimetypesWithViewIcon StringSet
254                 ArvadosDocsite                   string
255                 ArvadosPublicDataDocURL          string
256                 DefaultOpenIdPrefix              string
257                 EnableGettingStartedPopup        bool
258                 EnablePublicProjectsPage         bool
259                 FileViewersConfigURL             string
260                 LogViewerMaxBytes                ByteSize
261                 MultiSiteSearch                  string
262                 ProfilingEnabled                 bool
263                 Repositories                     bool
264                 RepositoryCache                  string
265                 RunningJobLogRecordsToFetch      int
266                 SecretKeyBase                    string
267                 ShowRecentCollectionsOnDashboard bool
268                 ShowUserAgreementInline          bool
269                 ShowUserNotifications            bool
270                 SiteName                         string
271                 Theme                            string
272                 UserProfileFormFields            map[string]struct {
273                         Type                 string
274                         FormFieldTitle       string
275                         FormFieldDescription string
276                         Required             bool
277                         Position             int
278                         Options              map[string]struct{}
279                 }
280                 UserProfileFormMessage string
281                 WelcomePageHTML        string
282                 InactivePageHTML       string
283                 SSHHelpPageHTML        string
284                 SSHHelpHostSuffix      string
285                 IdleTimeout            Duration
286         }
287 }
288
289 type StorageClassConfig struct {
290         Default  bool
291         Priority int
292 }
293
294 type Volume struct {
295         AccessViaHosts   map[URL]VolumeAccess
296         ReadOnly         bool
297         Replication      int
298         StorageClasses   map[string]bool
299         Driver           string
300         DriverParameters json.RawMessage
301 }
302
303 type S3VolumeDriverParameters struct {
304         IAMRole            string
305         AccessKeyID        string
306         SecretAccessKey    string
307         Endpoint           string
308         Region             string
309         Bucket             string
310         LocationConstraint bool
311         V2Signature        bool
312         UseAWSS3v2Driver   bool
313         IndexPageSize      int
314         ConnectTimeout     Duration
315         ReadTimeout        Duration
316         RaceWindow         Duration
317         UnsafeDelete       bool
318         PrefixLength       int
319 }
320
321 type AzureVolumeDriverParameters struct {
322         StorageAccountName   string
323         StorageAccountKey    string
324         StorageBaseURL       string
325         ContainerName        string
326         RequestTimeout       Duration
327         ListBlobsRetryDelay  Duration
328         ListBlobsMaxAttempts int
329 }
330
331 type DirectoryVolumeDriverParameters struct {
332         Root      string
333         Serialize bool
334 }
335
336 type VolumeAccess struct {
337         ReadOnly bool
338 }
339
340 type Services struct {
341         Composer       Service
342         Controller     Service
343         DispatchCloud  Service
344         DispatchLSF    Service
345         GitHTTP        Service
346         GitSSH         Service
347         Health         Service
348         Keepbalance    Service
349         Keepproxy      Service
350         Keepstore      Service
351         RailsAPI       Service
352         WebDAVDownload Service
353         WebDAV         Service
354         WebShell       Service
355         Websocket      Service
356         Workbench1     Service
357         Workbench2     Service
358 }
359
360 type Service struct {
361         InternalURLs map[URL]ServiceInstance
362         ExternalURL  URL
363 }
364
365 type TestUser struct {
366         Email    string
367         Password string
368 }
369
370 // URL is a url.URL that is also usable as a JSON key/value.
371 type URL url.URL
372
373 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
374 // used as a JSON key/value.
375 func (su *URL) UnmarshalText(text []byte) error {
376         u, err := url.Parse(string(text))
377         if err == nil {
378                 *su = URL(*u)
379                 if su.Path == "" && su.Host != "" {
380                         // http://example really means http://example/
381                         su.Path = "/"
382                 }
383         }
384         return err
385 }
386
387 func (su URL) MarshalText() ([]byte, error) {
388         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
389 }
390
391 func (su URL) String() string {
392         return (*url.URL)(&su).String()
393 }
394
395 type ServiceInstance struct {
396         Rendezvous string `json:",omitempty"`
397 }
398
399 type PostgreSQL struct {
400         Connection     PostgreSQLConnection
401         ConnectionPool int
402 }
403
404 type PostgreSQLConnection map[string]string
405
406 type RemoteCluster struct {
407         Host          string
408         Proxy         bool
409         Scheme        string
410         Insecure      bool
411         ActivateUsers bool
412 }
413
414 type CUDAFeatures struct {
415         DriverVersion      string
416         HardwareCapability string
417         DeviceCount        int
418 }
419
420 type InstanceType struct {
421         Name            string
422         ProviderType    string
423         VCPUs           int
424         RAM             ByteSize
425         Scratch         ByteSize
426         IncludedScratch ByteSize
427         AddedScratch    ByteSize
428         Price           float64
429         Preemptible     bool
430         CUDA            CUDAFeatures
431 }
432
433 type ContainersConfig struct {
434         CloudVMs                    CloudVMsConfig
435         CrunchRunCommand            string
436         CrunchRunArgumentsList      []string
437         DefaultKeepCacheRAM         ByteSize
438         DispatchPrivateKey          string
439         LogReuseDecisions           bool
440         MaxComputeVMs               int
441         MaxDispatchAttempts         int
442         MaxRetryAttempts            int
443         MinRetryPeriod              Duration
444         ReserveExtraRAM             ByteSize
445         StaleLockTimeout            Duration
446         SupportedDockerImageFormats StringSet
447         UsePreemptibleInstances     bool
448         RuntimeEngine               string
449         LocalKeepBlobBuffersPerVCPU int
450         LocalKeepLogsToContainerLog string
451
452         JobsAPI struct {
453                 Enable         string
454                 GitInternalDir string
455         }
456         Logging struct {
457                 MaxAge                       Duration
458                 LogBytesPerEvent             int
459                 LogSecondsBetweenEvents      Duration
460                 LogThrottlePeriod            Duration
461                 LogThrottleBytes             int
462                 LogThrottleLines             int
463                 LimitLogBytesPerJob          int
464                 LogPartialLineThrottlePeriod Duration
465                 LogUpdatePeriod              Duration
466                 LogUpdateSize                ByteSize
467         }
468         ShellAccess struct {
469                 Admin bool
470                 User  bool
471         }
472         SLURM struct {
473                 PrioritySpread             int64
474                 SbatchArgumentsList        []string
475                 SbatchEnvironmentVariables map[string]string
476                 Managed                    struct {
477                         DNSServerConfDir       string
478                         DNSServerConfTemplate  string
479                         DNSServerReloadCommand string
480                         DNSServerUpdateCommand string
481                         ComputeNodeDomain      string
482                         ComputeNodeNameservers StringSet
483                         AssignNodeHostname     string
484                 }
485         }
486         LSF struct {
487                 BsubSudoUser      string
488                 BsubArgumentsList []string
489         }
490 }
491
492 type CloudVMsConfig struct {
493         Enable bool
494
495         BootProbeCommand               string
496         DeployRunnerBinary             string
497         ImageID                        string
498         MaxCloudOpsPerSecond           int
499         MaxProbesPerSecond             int
500         MaxConcurrentInstanceCreateOps int
501         PollInterval                   Duration
502         ProbeInterval                  Duration
503         SSHPort                        string
504         SyncInterval                   Duration
505         TimeoutBooting                 Duration
506         TimeoutIdle                    Duration
507         TimeoutProbe                   Duration
508         TimeoutShutdown                Duration
509         TimeoutSignal                  Duration
510         TimeoutStaleRunLock            Duration
511         TimeoutTERM                    Duration
512         ResourceTags                   map[string]string
513         TagKeyPrefix                   string
514
515         Driver           string
516         DriverParameters json.RawMessage
517 }
518
519 type InstanceTypeMap map[string]InstanceType
520
521 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
522
523 // UnmarshalJSON handles old config files that provide an array of
524 // instance types instead of a hash.
525 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
526         fixup := func(t InstanceType) (InstanceType, error) {
527                 if t.ProviderType == "" {
528                         t.ProviderType = t.Name
529                 }
530                 if t.Scratch == 0 {
531                         t.Scratch = t.IncludedScratch + t.AddedScratch
532                 } else if t.AddedScratch == 0 {
533                         t.AddedScratch = t.Scratch - t.IncludedScratch
534                 } else if t.IncludedScratch == 0 {
535                         t.IncludedScratch = t.Scratch - t.AddedScratch
536                 }
537
538                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
539                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
540                 }
541                 return t, nil
542         }
543
544         if len(data) > 0 && data[0] == '[' {
545                 var arr []InstanceType
546                 err := json.Unmarshal(data, &arr)
547                 if err != nil {
548                         return err
549                 }
550                 if len(arr) == 0 {
551                         *it = nil
552                         return nil
553                 }
554                 *it = make(map[string]InstanceType, len(arr))
555                 for _, t := range arr {
556                         if _, ok := (*it)[t.Name]; ok {
557                                 return errDuplicateInstanceTypeName
558                         }
559                         t, err := fixup(t)
560                         if err != nil {
561                                 return err
562                         }
563                         (*it)[t.Name] = t
564                 }
565                 return nil
566         }
567         var hash map[string]InstanceType
568         err := json.Unmarshal(data, &hash)
569         if err != nil {
570                 return err
571         }
572         // Fill in Name field (and ProviderType field, if not
573         // specified) using hash key.
574         *it = InstanceTypeMap(hash)
575         for name, t := range *it {
576                 t.Name = name
577                 t, err := fixup(t)
578                 if err != nil {
579                         return err
580                 }
581                 (*it)[name] = t
582         }
583         return nil
584 }
585
586 type StringSet map[string]struct{}
587
588 // UnmarshalJSON handles old config files that provide an array of
589 // instance types instead of a hash.
590 func (ss *StringSet) UnmarshalJSON(data []byte) error {
591         if len(data) > 0 && data[0] == '[' {
592                 var arr []string
593                 err := json.Unmarshal(data, &arr)
594                 if err != nil {
595                         return err
596                 }
597                 if len(arr) == 0 {
598                         *ss = nil
599                         return nil
600                 }
601                 *ss = make(map[string]struct{}, len(arr))
602                 for _, t := range arr {
603                         (*ss)[t] = struct{}{}
604                 }
605                 return nil
606         }
607         var hash map[string]struct{}
608         err := json.Unmarshal(data, &hash)
609         if err != nil {
610                 return err
611         }
612         *ss = make(map[string]struct{}, len(hash))
613         for t := range hash {
614                 (*ss)[t] = struct{}{}
615         }
616
617         return nil
618 }
619
620 type ServiceName string
621
622 const (
623         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
624         ServiceNameController    ServiceName = "arvados-controller"
625         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
626         ServiceNameDispatchLSF   ServiceName = "arvados-dispatch-lsf"
627         ServiceNameHealth        ServiceName = "arvados-health"
628         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
629         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
630         ServiceNameWebsocket     ServiceName = "arvados-ws"
631         ServiceNameKeepbalance   ServiceName = "keep-balance"
632         ServiceNameKeepweb       ServiceName = "keep-web"
633         ServiceNameKeepproxy     ServiceName = "keepproxy"
634         ServiceNameKeepstore     ServiceName = "keepstore"
635 )
636
637 // Map returns all services as a map, suitable for iterating over all
638 // services or looking up a service by name.
639 func (svcs Services) Map() map[ServiceName]Service {
640         return map[ServiceName]Service{
641                 ServiceNameRailsAPI:      svcs.RailsAPI,
642                 ServiceNameController:    svcs.Controller,
643                 ServiceNameDispatchCloud: svcs.DispatchCloud,
644                 ServiceNameDispatchLSF:   svcs.DispatchLSF,
645                 ServiceNameHealth:        svcs.Health,
646                 ServiceNameWorkbench1:    svcs.Workbench1,
647                 ServiceNameWorkbench2:    svcs.Workbench2,
648                 ServiceNameWebsocket:     svcs.Websocket,
649                 ServiceNameKeepbalance:   svcs.Keepbalance,
650                 ServiceNameKeepweb:       svcs.WebDAV,
651                 ServiceNameKeepproxy:     svcs.Keepproxy,
652                 ServiceNameKeepstore:     svcs.Keepstore,
653         }
654 }