sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.arvados.org/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         }
  21         return "/etc/arvados/config.yml"
  22 }()
  23
  24 type Config struct {
  25         Clusters         map[string]Cluster
  26         AutoReloadConfig bool
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         cc, ok := sc.Clusters[clusterID]
  53         if !ok {
  54                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  55         }
  56         cc.ClusterID = clusterID
  57         return &cc, nil
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxBlockEntries      int
  64         MaxCollectionEntries int
  65         MaxCollectionBytes   int64
  66         MaxPermissionEntries int
  67         MaxUUIDEntries       int
  68         MaxSessions          int
  69 }
  70
  71 type Cluster struct {
  72         ClusterID       string `json:"-"`
  73         ManagementToken string
  74         SystemRootToken string
  75         Services        Services
  76         InstanceTypes   InstanceTypeMap
  77         Containers      ContainersConfig
  78         RemoteClusters  map[string]RemoteCluster
  79         PostgreSQL      PostgreSQL
  80
  81         API struct {
  82                 AsyncPermissionsUpdateInterval Duration
  83                 DisabledAPIs                   StringSet
  84                 MaxIndexDatabaseRead           int
  85                 MaxItemsPerResponse            int
  86                 MaxConcurrentRequests          int
  87                 MaxKeepBlobBuffers             int
  88                 MaxRequestAmplification        int
  89                 MaxRequestSize                 int
  90                 MaxTokenLifetime               Duration
  91                 RequestTimeout                 Duration
  92                 SendTimeout                    Duration
  93                 WebsocketClientEventQueue      int
  94                 WebsocketServerEventQueue      int
  95                 KeepServiceRequestTimeout      Duration
  96         }
  97         AuditLogs struct {
  98                 MaxAge             Duration
  99                 MaxDeleteBatch     int
 100                 UnloggedAttributes StringSet
 101         }
 102         Collections struct {
 103                 BlobSigning              bool
 104                 BlobSigningKey           string
 105                 BlobSigningTTL           Duration
 106                 BlobTrash                bool
 107                 BlobTrashLifetime        Duration
 108                 BlobTrashCheckInterval   Duration
 109                 BlobTrashConcurrency     int
 110                 BlobDeleteConcurrency    int
 111                 BlobReplicateConcurrency int
 112                 CollectionVersioning     bool
 113                 DefaultTrashLifetime     Duration
 114                 DefaultReplication       int
 115                 ManagedProperties        map[string]struct {
 116                         Value     interface{}
 117                         Function  string
 118                         Protected bool
 119                 }
 120                 PreserveVersionIfIdle        Duration
 121                 TrashSweepInterval           Duration
 122                 TrustAllContent              bool
 123                 ForwardSlashNameSubstitution string
 124                 S3FolderObjects              bool
 125
 126                 BlobMissingReport        string
 127                 BalancePeriod            Duration
 128                 BalanceCollectionBatch   int
 129                 BalanceCollectionBuffers int
 130                 BalanceTimeout           Duration
 131
 132                 WebDAVCache WebDAVCacheConfig
 133         }
 134         Git struct {
 135                 GitCommand   string
 136                 GitoliteHome string
 137                 Repositories string
 138         }
 139         Login struct {
 140                 LDAP struct {
 141                         Enable             bool
 142                         URL                URL
 143                         StartTLS           bool
 144                         InsecureTLS        bool
 145                         StripDomain        string
 146                         AppendDomain       string
 147                         SearchAttribute    string
 148                         SearchBindUser     string
 149                         SearchBindPassword string
 150                         SearchBase         string
 151                         SearchFilters      string
 152                         EmailAttribute     string
 153                         UsernameAttribute  string
 154                 }
 155                 Google struct {
 156                         Enable                          bool
 157                         ClientID                        string
 158                         ClientSecret                    string
 159                         AlternateEmailAddresses         bool
 160                         AuthenticationRequestParameters map[string]string
 161                 }
 162                 OpenIDConnect struct {
 163                         Enable                          bool
 164                         Issuer                          string
 165                         ClientID                        string
 166                         ClientSecret                    string
 167                         EmailClaim                      string
 168                         EmailVerifiedClaim              string
 169                         UsernameClaim                   string
 170                         AcceptAccessTokenScope          string
 171                         AuthenticationRequestParameters map[string]string
 172                 }
 173                 PAM struct {
 174                         Enable             bool
 175                         Service            string
 176                         DefaultEmailDomain string
 177                 }
 178                 SSO struct {
 179                         Enable            bool
 180                         ProviderAppID     string
 181                         ProviderAppSecret string
 182                 }
 183                 Test struct {
 184                         Enable bool
 185                         Users  map[string]TestUser
 186                 }
 187                 LoginCluster       string
 188                 RemoteTokenRefresh Duration
 189                 TokenLifetime      Duration
 190                 TrustedClients     map[string]struct{}
 191         }
 192         Mail struct {
 193                 MailchimpAPIKey                string
 194                 MailchimpListID                string
 195                 SendUserSetupNotificationEmail bool
 196                 IssueReporterEmailFrom         string
 197                 IssueReporterEmailTo           string
 198                 SupportEmailAddress            string
 199                 EmailFrom                      string
 200         }
 201         SystemLogs struct {
 202                 LogLevel                string
 203                 Format                  string
 204                 MaxRequestLogParamsSize int
 205         }
 206         TLS struct {
 207                 Certificate string
 208                 Key         string
 209                 Insecure    bool
 210         }
 211         Users struct {
 212                 AnonymousUserToken                    string
 213                 AdminNotifierEmailFrom                string
 214                 AutoAdminFirstUser                    bool
 215                 AutoAdminUserWithEmail                string
 216                 AutoSetupNewUsers                     bool
 217                 AutoSetupNewUsersWithRepository       bool
 218                 AutoSetupNewUsersWithVmUUID           string
 219                 AutoSetupUsernameBlacklist            StringSet
 220                 EmailSubjectPrefix                    string
 221                 NewInactiveUserNotificationRecipients StringSet
 222                 NewUserNotificationRecipients         StringSet
 223                 NewUsersAreActive                     bool
 224                 UserNotifierEmailFrom                 string
 225                 UserProfileNotificationAddress        string
 226                 PreferDomainForUsername               string
 227                 UserSetupMailText                     string
 228         }
 229         Volumes   map[string]Volume
 230         Workbench struct {
 231                 ActivationContactLink            string
 232                 APIClientConnectTimeout          Duration
 233                 APIClientReceiveTimeout          Duration
 234                 APIResponseCompression           bool
 235                 ApplicationMimetypesWithViewIcon StringSet
 236                 ArvadosDocsite                   string
 237                 ArvadosPublicDataDocURL          string
 238                 DefaultOpenIdPrefix              string
 239                 EnableGettingStartedPopup        bool
 240                 EnablePublicProjectsPage         bool
 241                 FileViewersConfigURL             string
 242                 LogViewerMaxBytes                ByteSize
 243                 MultiSiteSearch                  string
 244                 ProfilingEnabled                 bool
 245                 Repositories                     bool
 246                 RepositoryCache                  string
 247                 RunningJobLogRecordsToFetch      int
 248                 SecretKeyBase                    string
 249                 ShowRecentCollectionsOnDashboard bool
 250                 ShowUserAgreementInline          bool
 251                 ShowUserNotifications            bool
 252                 SiteName                         string
 253                 Theme                            string
 254                 UserProfileFormFields            map[string]struct {
 255                         Type                 string
 256                         FormFieldTitle       string
 257                         FormFieldDescription string
 258                         Required             bool
 259                         Position             int
 260                         Options              map[string]struct{}
 261                 }
 262                 UserProfileFormMessage string
 263                 VocabularyURL          string
 264                 WelcomePageHTML        string
 265                 InactivePageHTML       string
 266                 SSHHelpPageHTML        string
 267                 SSHHelpHostSuffix      string
 268                 IdleTimeout            Duration
 269         }
 270 }
 271
 272 type Volume struct {
 273         AccessViaHosts   map[URL]VolumeAccess
 274         ReadOnly         bool
 275         Replication      int
 276         StorageClasses   map[string]bool
 277         Driver           string
 278         DriverParameters json.RawMessage
 279 }
 280
 281 type S3VolumeDriverParameters struct {
 282         IAMRole            string
 283         AccessKeyID        string
 284         SecretAccessKey    string
 285         Endpoint           string
 286         Region             string
 287         Bucket             string
 288         LocationConstraint bool
 289         V2Signature        bool
 290         UseAWSS3v2Driver   bool
 291         IndexPageSize      int
 292         ConnectTimeout     Duration
 293         ReadTimeout        Duration
 294         RaceWindow         Duration
 295         UnsafeDelete       bool
 296 }
 297
 298 type AzureVolumeDriverParameters struct {
 299         StorageAccountName   string
 300         StorageAccountKey    string
 301         StorageBaseURL       string
 302         ContainerName        string
 303         RequestTimeout       Duration
 304         ListBlobsRetryDelay  Duration
 305         ListBlobsMaxAttempts int
 306 }
 307
 308 type DirectoryVolumeDriverParameters struct {
 309         Root      string
 310         Serialize bool
 311 }
 312
 313 type VolumeAccess struct {
 314         ReadOnly bool
 315 }
 316
 317 type Services struct {
 318         Composer       Service
 319         Controller     Service
 320         DispatchCloud  Service
 321         GitHTTP        Service
 322         GitSSH         Service
 323         Health         Service
 324         Keepbalance    Service
 325         Keepproxy      Service
 326         Keepstore      Service
 327         RailsAPI       Service
 328         SSO            Service
 329         WebDAVDownload Service
 330         WebDAV         Service
 331         WebShell       Service
 332         Websocket      Service
 333         Workbench1     Service
 334         Workbench2     Service
 335 }
 336
 337 type Service struct {
 338         InternalURLs map[URL]ServiceInstance
 339         ExternalURL  URL
 340 }
 341
 342 type TestUser struct {
 343         Email    string
 344         Password string
 345 }
 346
 347 // URL is a url.URL that is also usable as a JSON key/value.
 348 type URL url.URL
 349
 350 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 351 // used as a JSON key/value.
 352 func (su *URL) UnmarshalText(text []byte) error {
 353         u, err := url.Parse(string(text))
 354         if err == nil {
 355                 *su = URL(*u)
 356                 if su.Path == "" && su.Host != "" {
 357                         // http://example really means http://example/
 358                         su.Path = "/"
 359                 }
 360         }
 361         return err
 362 }
 363
 364 func (su URL) MarshalText() ([]byte, error) {
 365         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 366 }
 367
 368 func (su URL) String() string {
 369         return (*url.URL)(&su).String()
 370 }
 371
 372 type ServiceInstance struct {
 373         Rendezvous string `json:",omitempty"`
 374 }
 375
 376 type PostgreSQL struct {
 377         Connection     PostgreSQLConnection
 378         ConnectionPool int
 379 }
 380
 381 type PostgreSQLConnection map[string]string
 382
 383 type RemoteCluster struct {
 384         Host          string
 385         Proxy         bool
 386         Scheme        string
 387         Insecure      bool
 388         ActivateUsers bool
 389 }
 390
 391 type InstanceType struct {
 392         Name            string
 393         ProviderType    string
 394         VCPUs           int
 395         RAM             ByteSize
 396         Scratch         ByteSize
 397         IncludedScratch ByteSize
 398         AddedScratch    ByteSize
 399         Price           float64
 400         Preemptible     bool
 401 }
 402
 403 type ContainersConfig struct {
 404         CloudVMs                    CloudVMsConfig
 405         CrunchRunCommand            string
 406         CrunchRunArgumentsList      []string
 407         DefaultKeepCacheRAM         ByteSize
 408         DispatchPrivateKey          string
 409         LogReuseDecisions           bool
 410         MaxComputeVMs               int
 411         MaxDispatchAttempts         int
 412         MaxRetryAttempts            int
 413         MinRetryPeriod              Duration
 414         ReserveExtraRAM             ByteSize
 415         StaleLockTimeout            Duration
 416         SupportedDockerImageFormats StringSet
 417         UsePreemptibleInstances     bool
 418
 419         JobsAPI struct {
 420                 Enable         string
 421                 GitInternalDir string
 422         }
 423         Logging struct {
 424                 MaxAge                       Duration
 425                 LogBytesPerEvent             int
 426                 LogSecondsBetweenEvents      Duration
 427                 LogThrottlePeriod            Duration
 428                 LogThrottleBytes             int
 429                 LogThrottleLines             int
 430                 LimitLogBytesPerJob          int
 431                 LogPartialLineThrottlePeriod Duration
 432                 LogUpdatePeriod              Duration
 433                 LogUpdateSize                ByteSize
 434         }
 435         ShellAccess struct {
 436                 Admin bool
 437                 User  bool
 438         }
 439         SLURM struct {
 440                 PrioritySpread             int64
 441                 SbatchArgumentsList        []string
 442                 SbatchEnvironmentVariables map[string]string
 443                 Managed                    struct {
 444                         DNSServerConfDir       string
 445                         DNSServerConfTemplate  string
 446                         DNSServerReloadCommand string
 447                         DNSServerUpdateCommand string
 448                         ComputeNodeDomain      string
 449                         ComputeNodeNameservers StringSet
 450                         AssignNodeHostname     string
 451                 }
 452         }
 453 }
 454
 455 type CloudVMsConfig struct {
 456         Enable bool
 457
 458         BootProbeCommand               string
 459         DeployRunnerBinary             string
 460         ImageID                        string
 461         MaxCloudOpsPerSecond           int
 462         MaxProbesPerSecond             int
 463         MaxConcurrentInstanceCreateOps int
 464         PollInterval                   Duration
 465         ProbeInterval                  Duration
 466         SSHPort                        string
 467         SyncInterval                   Duration
 468         TimeoutBooting                 Duration
 469         TimeoutIdle                    Duration
 470         TimeoutProbe                   Duration
 471         TimeoutShutdown                Duration
 472         TimeoutSignal                  Duration
 473         TimeoutStaleRunLock            Duration
 474         TimeoutTERM                    Duration
 475         ResourceTags                   map[string]string
 476         TagKeyPrefix                   string
 477
 478         Driver           string
 479         DriverParameters json.RawMessage
 480 }
 481
 482 type InstanceTypeMap map[string]InstanceType
 483
 484 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 485
 486 // UnmarshalJSON handles old config files that provide an array of
 487 // instance types instead of a hash.
 488 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 489         fixup := func(t InstanceType) (InstanceType, error) {
 490                 if t.ProviderType == "" {
 491                         t.ProviderType = t.Name
 492                 }
 493                 if t.Scratch == 0 {
 494                         t.Scratch = t.IncludedScratch + t.AddedScratch
 495                 } else if t.AddedScratch == 0 {
 496                         t.AddedScratch = t.Scratch - t.IncludedScratch
 497                 } else if t.IncludedScratch == 0 {
 498                         t.IncludedScratch = t.Scratch - t.AddedScratch
 499                 }
 500
 501                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 502                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 503                 }
 504                 return t, nil
 505         }
 506
 507         if len(data) > 0 && data[0] == '[' {
 508                 var arr []InstanceType
 509                 err := json.Unmarshal(data, &arr)
 510                 if err != nil {
 511                         return err
 512                 }
 513                 if len(arr) == 0 {
 514                         *it = nil
 515                         return nil
 516                 }
 517                 *it = make(map[string]InstanceType, len(arr))
 518                 for _, t := range arr {
 519                         if _, ok := (*it)[t.Name]; ok {
 520                                 return errDuplicateInstanceTypeName
 521                         }
 522                         t, err := fixup(t)
 523                         if err != nil {
 524                                 return err
 525                         }
 526                         (*it)[t.Name] = t
 527                 }
 528                 return nil
 529         }
 530         var hash map[string]InstanceType
 531         err := json.Unmarshal(data, &hash)
 532         if err != nil {
 533                 return err
 534         }
 535         // Fill in Name field (and ProviderType field, if not
 536         // specified) using hash key.
 537         *it = InstanceTypeMap(hash)
 538         for name, t := range *it {
 539                 t.Name = name
 540                 t, err := fixup(t)
 541                 if err != nil {
 542                         return err
 543                 }
 544                 (*it)[name] = t
 545         }
 546         return nil
 547 }
 548
 549 type StringSet map[string]struct{}
 550
 551 // UnmarshalJSON handles old config files that provide an array of
 552 // instance types instead of a hash.
 553 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 554         if len(data) > 0 && data[0] == '[' {
 555                 var arr []string
 556                 err := json.Unmarshal(data, &arr)
 557                 if err != nil {
 558                         return err
 559                 }
 560                 if len(arr) == 0 {
 561                         *ss = nil
 562                         return nil
 563                 }
 564                 *ss = make(map[string]struct{}, len(arr))
 565                 for _, t := range arr {
 566                         (*ss)[t] = struct{}{}
 567                 }
 568                 return nil
 569         }
 570         var hash map[string]struct{}
 571         err := json.Unmarshal(data, &hash)
 572         if err != nil {
 573                 return err
 574         }
 575         *ss = make(map[string]struct{}, len(hash))
 576         for t := range hash {
 577                 (*ss)[t] = struct{}{}
 578         }
 579
 580         return nil
 581 }
 582
 583 type ServiceName string
 584
 585 const (
 586         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 587         ServiceNameController    ServiceName = "arvados-controller"
 588         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 589         ServiceNameHealth        ServiceName = "arvados-health"
 590         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 591         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 592         ServiceNameWebsocket     ServiceName = "arvados-ws"
 593         ServiceNameKeepbalance   ServiceName = "keep-balance"
 594         ServiceNameKeepweb       ServiceName = "keep-web"
 595         ServiceNameKeepproxy     ServiceName = "keepproxy"
 596         ServiceNameKeepstore     ServiceName = "keepstore"
 597 )
 598
 599 // Map returns all services as a map, suitable for iterating over all
 600 // services or looking up a service by name.
 601 func (svcs Services) Map() map[ServiceName]Service {
 602         return map[ServiceName]Service{
 603                 ServiceNameRailsAPI:      svcs.RailsAPI,
 604                 ServiceNameController:    svcs.Controller,
 605                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 606                 ServiceNameHealth:        svcs.Health,
 607                 ServiceNameWorkbench1:    svcs.Workbench1,
 608                 ServiceNameWorkbench2:    svcs.Workbench2,
 609                 ServiceNameWebsocket:     svcs.Websocket,
 610                 ServiceNameKeepbalance:   svcs.Keepbalance,
 611                 ServiceNameKeepweb:       svcs.WebDAV,
 612                 ServiceNameKeepproxy:     svcs.Keepproxy,
 613                 ServiceNameKeepstore:     svcs.Keepstore,
 614         }
 615 }