sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.arvados.org/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         }
  21         return "/etc/arvados/config.yml"
  22 }()
  23
  24 type Config struct {
  25         Clusters         map[string]Cluster
  26         AutoReloadConfig bool
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         cc, ok := sc.Clusters[clusterID]
  53         if !ok {
  54                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  55         }
  56         cc.ClusterID = clusterID
  57         return &cc, nil
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxBlockEntries      int
  64         MaxCollectionEntries int
  65         MaxCollectionBytes   int64
  66         MaxPermissionEntries int
  67         MaxUUIDEntries       int
  68         MaxSessions          int
  69 }
  70
  71 type Cluster struct {
  72         ClusterID       string `json:"-"`
  73         ManagementToken string
  74         SystemRootToken string
  75         Services        Services
  76         InstanceTypes   InstanceTypeMap
  77         Containers      ContainersConfig
  78         RemoteClusters  map[string]RemoteCluster
  79         PostgreSQL      PostgreSQL
  80
  81         API struct {
  82                 AsyncPermissionsUpdateInterval Duration
  83                 DisabledAPIs                   StringSet
  84                 MaxIndexDatabaseRead           int
  85                 MaxItemsPerResponse            int
  86                 MaxConcurrentRequests          int
  87                 MaxKeepBlobBuffers             int
  88                 MaxRequestAmplification        int
  89                 MaxRequestSize                 int
  90                 MaxTokenLifetime               Duration
  91                 RequestTimeout                 Duration
  92                 SendTimeout                    Duration
  93                 WebsocketClientEventQueue      int
  94                 WebsocketServerEventQueue      int
  95                 KeepServiceRequestTimeout      Duration
  96         }
  97         AuditLogs struct {
  98                 MaxAge             Duration
  99                 MaxDeleteBatch     int
 100                 UnloggedAttributes StringSet
 101         }
 102         Collections struct {
 103                 BlobSigning              bool
 104                 BlobSigningKey           string
 105                 BlobSigningTTL           Duration
 106                 BlobTrash                bool
 107                 BlobTrashLifetime        Duration
 108                 BlobTrashCheckInterval   Duration
 109                 BlobTrashConcurrency     int
 110                 BlobDeleteConcurrency    int
 111                 BlobReplicateConcurrency int
 112                 CollectionVersioning     bool
 113                 DefaultTrashLifetime     Duration
 114                 DefaultReplication       int
 115                 ManagedProperties        map[string]struct {
 116                         Value     interface{}
 117                         Function  string
 118                         Protected bool
 119                 }
 120                 PreserveVersionIfIdle        Duration
 121                 TrashSweepInterval           Duration
 122                 TrustAllContent              bool
 123                 ForwardSlashNameSubstitution string
 124                 S3FolderObjects              bool
 125
 126                 BlobMissingReport        string
 127                 BalancePeriod            Duration
 128                 BalanceCollectionBatch   int
 129                 BalanceCollectionBuffers int
 130                 BalanceTimeout           Duration
 131
 132                 WebDAVCache WebDAVCacheConfig
 133         }
 134         Git struct {
 135                 GitCommand   string
 136                 GitoliteHome string
 137                 Repositories string
 138         }
 139         Login struct {
 140                 LDAP struct {
 141                         Enable             bool
 142                         URL                URL
 143                         StartTLS           bool
 144                         InsecureTLS        bool
 145                         StripDomain        string
 146                         AppendDomain       string
 147                         SearchAttribute    string
 148                         SearchBindUser     string
 149                         SearchBindPassword string
 150                         SearchBase         string
 151                         SearchFilters      string
 152                         EmailAttribute     string
 153                         UsernameAttribute  string
 154                 }
 155                 Google struct {
 156                         Enable                          bool
 157                         ClientID                        string
 158                         ClientSecret                    string
 159                         AlternateEmailAddresses         bool
 160                         AuthenticationRequestParameters map[string]string
 161                 }
 162                 OpenIDConnect struct {
 163                         Enable                          bool
 164                         Issuer                          string
 165                         ClientID                        string
 166                         ClientSecret                    string
 167                         EmailClaim                      string
 168                         EmailVerifiedClaim              string
 169                         UsernameClaim                   string
 170                         AcceptAccessToken               bool
 171                         AcceptAccessTokenScope          string
 172                         AuthenticationRequestParameters map[string]string
 173                 }
 174                 PAM struct {
 175                         Enable             bool
 176                         Service            string
 177                         DefaultEmailDomain string
 178                 }
 179                 SSO struct {
 180                         Enable            bool
 181                         ProviderAppID     string
 182                         ProviderAppSecret string
 183                 }
 184                 Test struct {
 185                         Enable bool
 186                         Users  map[string]TestUser
 187                 }
 188                 LoginCluster       string
 189                 RemoteTokenRefresh Duration
 190                 TokenLifetime      Duration
 191                 TrustedClients     map[string]struct{}
 192         }
 193         Mail struct {
 194                 MailchimpAPIKey                string
 195                 MailchimpListID                string
 196                 SendUserSetupNotificationEmail bool
 197                 IssueReporterEmailFrom         string
 198                 IssueReporterEmailTo           string
 199                 SupportEmailAddress            string
 200                 EmailFrom                      string
 201         }
 202         SystemLogs struct {
 203                 LogLevel                string
 204                 Format                  string
 205                 MaxRequestLogParamsSize int
 206         }
 207         TLS struct {
 208                 Certificate string
 209                 Key         string
 210                 Insecure    bool
 211         }
 212         Users struct {
 213                 AnonymousUserToken                    string
 214                 AdminNotifierEmailFrom                string
 215                 AutoAdminFirstUser                    bool
 216                 AutoAdminUserWithEmail                string
 217                 AutoSetupNewUsers                     bool
 218                 AutoSetupNewUsersWithRepository       bool
 219                 AutoSetupNewUsersWithVmUUID           string
 220                 AutoSetupUsernameBlacklist            StringSet
 221                 EmailSubjectPrefix                    string
 222                 NewInactiveUserNotificationRecipients StringSet
 223                 NewUserNotificationRecipients         StringSet
 224                 NewUsersAreActive                     bool
 225                 UserNotifierEmailFrom                 string
 226                 UserProfileNotificationAddress        string
 227                 PreferDomainForUsername               string
 228                 UserSetupMailText                     string
 229         }
 230         Volumes   map[string]Volume
 231         Workbench struct {
 232                 ActivationContactLink            string
 233                 APIClientConnectTimeout          Duration
 234                 APIClientReceiveTimeout          Duration
 235                 APIResponseCompression           bool
 236                 ApplicationMimetypesWithViewIcon StringSet
 237                 ArvadosDocsite                   string
 238                 ArvadosPublicDataDocURL          string
 239                 DefaultOpenIdPrefix              string
 240                 EnableGettingStartedPopup        bool
 241                 EnablePublicProjectsPage         bool
 242                 FileViewersConfigURL             string
 243                 LogViewerMaxBytes                ByteSize
 244                 MultiSiteSearch                  string
 245                 ProfilingEnabled                 bool
 246                 Repositories                     bool
 247                 RepositoryCache                  string
 248                 RunningJobLogRecordsToFetch      int
 249                 SecretKeyBase                    string
 250                 ShowRecentCollectionsOnDashboard bool
 251                 ShowUserAgreementInline          bool
 252                 ShowUserNotifications            bool
 253                 SiteName                         string
 254                 Theme                            string
 255                 UserProfileFormFields            map[string]struct {
 256                         Type                 string
 257                         FormFieldTitle       string
 258                         FormFieldDescription string
 259                         Required             bool
 260                         Position             int
 261                         Options              map[string]struct{}
 262                 }
 263                 UserProfileFormMessage string
 264                 VocabularyURL          string
 265                 WelcomePageHTML        string
 266                 InactivePageHTML       string
 267                 SSHHelpPageHTML        string
 268                 SSHHelpHostSuffix      string
 269                 IdleTimeout            Duration
 270         }
 271 }
 272
 273 type Volume struct {
 274         AccessViaHosts   map[URL]VolumeAccess
 275         ReadOnly         bool
 276         Replication      int
 277         StorageClasses   map[string]bool
 278         Driver           string
 279         DriverParameters json.RawMessage
 280 }
 281
 282 type S3VolumeDriverParameters struct {
 283         IAMRole            string
 284         AccessKeyID        string
 285         SecretAccessKey    string
 286         Endpoint           string
 287         Region             string
 288         Bucket             string
 289         LocationConstraint bool
 290         V2Signature        bool
 291         UseAWSS3v2Driver   bool
 292         IndexPageSize      int
 293         ConnectTimeout     Duration
 294         ReadTimeout        Duration
 295         RaceWindow         Duration
 296         UnsafeDelete       bool
 297 }
 298
 299 type AzureVolumeDriverParameters struct {
 300         StorageAccountName   string
 301         StorageAccountKey    string
 302         StorageBaseURL       string
 303         ContainerName        string
 304         RequestTimeout       Duration
 305         ListBlobsRetryDelay  Duration
 306         ListBlobsMaxAttempts int
 307 }
 308
 309 type DirectoryVolumeDriverParameters struct {
 310         Root      string
 311         Serialize bool
 312 }
 313
 314 type VolumeAccess struct {
 315         ReadOnly bool
 316 }
 317
 318 type Services struct {
 319         Composer       Service
 320         Controller     Service
 321         DispatchCloud  Service
 322         GitHTTP        Service
 323         GitSSH         Service
 324         Health         Service
 325         Keepbalance    Service
 326         Keepproxy      Service
 327         Keepstore      Service
 328         RailsAPI       Service
 329         SSO            Service
 330         WebDAVDownload Service
 331         WebDAV         Service
 332         WebShell       Service
 333         Websocket      Service
 334         Workbench1     Service
 335         Workbench2     Service
 336 }
 337
 338 type Service struct {
 339         InternalURLs map[URL]ServiceInstance
 340         ExternalURL  URL
 341 }
 342
 343 type TestUser struct {
 344         Email    string
 345         Password string
 346 }
 347
 348 // URL is a url.URL that is also usable as a JSON key/value.
 349 type URL url.URL
 350
 351 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 352 // used as a JSON key/value.
 353 func (su *URL) UnmarshalText(text []byte) error {
 354         u, err := url.Parse(string(text))
 355         if err == nil {
 356                 *su = URL(*u)
 357                 if su.Path == "" && su.Host != "" {
 358                         // http://example really means http://example/
 359                         su.Path = "/"
 360                 }
 361         }
 362         return err
 363 }
 364
 365 func (su URL) MarshalText() ([]byte, error) {
 366         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 367 }
 368
 369 func (su URL) String() string {
 370         return (*url.URL)(&su).String()
 371 }
 372
 373 type ServiceInstance struct {
 374         Rendezvous string `json:",omitempty"`
 375 }
 376
 377 type PostgreSQL struct {
 378         Connection     PostgreSQLConnection
 379         ConnectionPool int
 380 }
 381
 382 type PostgreSQLConnection map[string]string
 383
 384 type RemoteCluster struct {
 385         Host          string
 386         Proxy         bool
 387         Scheme        string
 388         Insecure      bool
 389         ActivateUsers bool
 390 }
 391
 392 type InstanceType struct {
 393         Name            string
 394         ProviderType    string
 395         VCPUs           int
 396         RAM             ByteSize
 397         Scratch         ByteSize
 398         IncludedScratch ByteSize
 399         AddedScratch    ByteSize
 400         Price           float64
 401         Preemptible     bool
 402 }
 403
 404 type ContainersConfig struct {
 405         CloudVMs                    CloudVMsConfig
 406         CrunchRunCommand            string
 407         CrunchRunArgumentsList      []string
 408         DefaultKeepCacheRAM         ByteSize
 409         DispatchPrivateKey          string
 410         LogReuseDecisions           bool
 411         MaxComputeVMs               int
 412         MaxDispatchAttempts         int
 413         MaxRetryAttempts            int
 414         MinRetryPeriod              Duration
 415         ReserveExtraRAM             ByteSize
 416         StaleLockTimeout            Duration
 417         SupportedDockerImageFormats StringSet
 418         UsePreemptibleInstances     bool
 419
 420         JobsAPI struct {
 421                 Enable         string
 422                 GitInternalDir string
 423         }
 424         Logging struct {
 425                 MaxAge                       Duration
 426                 LogBytesPerEvent             int
 427                 LogSecondsBetweenEvents      Duration
 428                 LogThrottlePeriod            Duration
 429                 LogThrottleBytes             int
 430                 LogThrottleLines             int
 431                 LimitLogBytesPerJob          int
 432                 LogPartialLineThrottlePeriod Duration
 433                 LogUpdatePeriod              Duration
 434                 LogUpdateSize                ByteSize
 435         }
 436         ShellAccess struct {
 437                 Admin bool
 438                 User  bool
 439         }
 440         SLURM struct {
 441                 PrioritySpread             int64
 442                 SbatchArgumentsList        []string
 443                 SbatchEnvironmentVariables map[string]string
 444                 Managed                    struct {
 445                         DNSServerConfDir       string
 446                         DNSServerConfTemplate  string
 447                         DNSServerReloadCommand string
 448                         DNSServerUpdateCommand string
 449                         ComputeNodeDomain      string
 450                         ComputeNodeNameservers StringSet
 451                         AssignNodeHostname     string
 452                 }
 453         }
 454 }
 455
 456 type CloudVMsConfig struct {
 457         Enable bool
 458
 459         BootProbeCommand               string
 460         DeployRunnerBinary             string
 461         ImageID                        string
 462         MaxCloudOpsPerSecond           int
 463         MaxProbesPerSecond             int
 464         MaxConcurrentInstanceCreateOps int
 465         PollInterval                   Duration
 466         ProbeInterval                  Duration
 467         SSHPort                        string
 468         SyncInterval                   Duration
 469         TimeoutBooting                 Duration
 470         TimeoutIdle                    Duration
 471         TimeoutProbe                   Duration
 472         TimeoutShutdown                Duration
 473         TimeoutSignal                  Duration
 474         TimeoutStaleRunLock            Duration
 475         TimeoutTERM                    Duration
 476         ResourceTags                   map[string]string
 477         TagKeyPrefix                   string
 478
 479         Driver           string
 480         DriverParameters json.RawMessage
 481 }
 482
 483 type InstanceTypeMap map[string]InstanceType
 484
 485 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 486
 487 // UnmarshalJSON handles old config files that provide an array of
 488 // instance types instead of a hash.
 489 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 490         fixup := func(t InstanceType) (InstanceType, error) {
 491                 if t.ProviderType == "" {
 492                         t.ProviderType = t.Name
 493                 }
 494                 if t.Scratch == 0 {
 495                         t.Scratch = t.IncludedScratch + t.AddedScratch
 496                 } else if t.AddedScratch == 0 {
 497                         t.AddedScratch = t.Scratch - t.IncludedScratch
 498                 } else if t.IncludedScratch == 0 {
 499                         t.IncludedScratch = t.Scratch - t.AddedScratch
 500                 }
 501
 502                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 503                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 504                 }
 505                 return t, nil
 506         }
 507
 508         if len(data) > 0 && data[0] == '[' {
 509                 var arr []InstanceType
 510                 err := json.Unmarshal(data, &arr)
 511                 if err != nil {
 512                         return err
 513                 }
 514                 if len(arr) == 0 {
 515                         *it = nil
 516                         return nil
 517                 }
 518                 *it = make(map[string]InstanceType, len(arr))
 519                 for _, t := range arr {
 520                         if _, ok := (*it)[t.Name]; ok {
 521                                 return errDuplicateInstanceTypeName
 522                         }
 523                         t, err := fixup(t)
 524                         if err != nil {
 525                                 return err
 526                         }
 527                         (*it)[t.Name] = t
 528                 }
 529                 return nil
 530         }
 531         var hash map[string]InstanceType
 532         err := json.Unmarshal(data, &hash)
 533         if err != nil {
 534                 return err
 535         }
 536         // Fill in Name field (and ProviderType field, if not
 537         // specified) using hash key.
 538         *it = InstanceTypeMap(hash)
 539         for name, t := range *it {
 540                 t.Name = name
 541                 t, err := fixup(t)
 542                 if err != nil {
 543                         return err
 544                 }
 545                 (*it)[name] = t
 546         }
 547         return nil
 548 }
 549
 550 type StringSet map[string]struct{}
 551
 552 // UnmarshalJSON handles old config files that provide an array of
 553 // instance types instead of a hash.
 554 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 555         if len(data) > 0 && data[0] == '[' {
 556                 var arr []string
 557                 err := json.Unmarshal(data, &arr)
 558                 if err != nil {
 559                         return err
 560                 }
 561                 if len(arr) == 0 {
 562                         *ss = nil
 563                         return nil
 564                 }
 565                 *ss = make(map[string]struct{}, len(arr))
 566                 for _, t := range arr {
 567                         (*ss)[t] = struct{}{}
 568                 }
 569                 return nil
 570         }
 571         var hash map[string]struct{}
 572         err := json.Unmarshal(data, &hash)
 573         if err != nil {
 574                 return err
 575         }
 576         *ss = make(map[string]struct{}, len(hash))
 577         for t := range hash {
 578                 (*ss)[t] = struct{}{}
 579         }
 580
 581         return nil
 582 }
 583
 584 type ServiceName string
 585
 586 const (
 587         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 588         ServiceNameController    ServiceName = "arvados-controller"
 589         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 590         ServiceNameHealth        ServiceName = "arvados-health"
 591         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 592         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 593         ServiceNameWebsocket     ServiceName = "arvados-ws"
 594         ServiceNameKeepbalance   ServiceName = "keep-balance"
 595         ServiceNameKeepweb       ServiceName = "keep-web"
 596         ServiceNameKeepproxy     ServiceName = "keepproxy"
 597         ServiceNameKeepstore     ServiceName = "keepstore"
 598 )
 599
 600 // Map returns all services as a map, suitable for iterating over all
 601 // services or looking up a service by name.
 602 func (svcs Services) Map() map[ServiceName]Service {
 603         return map[ServiceName]Service{
 604                 ServiceNameRailsAPI:      svcs.RailsAPI,
 605                 ServiceNameController:    svcs.Controller,
 606                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 607                 ServiceNameHealth:        svcs.Health,
 608                 ServiceNameWorkbench1:    svcs.Workbench1,
 609                 ServiceNameWorkbench2:    svcs.Workbench2,
 610                 ServiceNameWebsocket:     svcs.Websocket,
 611                 ServiceNameKeepbalance:   svcs.Keepbalance,
 612                 ServiceNameKeepweb:       svcs.WebDAV,
 613                 ServiceNameKeepproxy:     svcs.Keepproxy,
 614                 ServiceNameKeepstore:     svcs.Keepstore,
 615         }
 616 }