sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.arvados.org/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         }
  21         return "/etc/arvados/config.yml"
  22 }()
  23
  24 type Config struct {
  25         Clusters         map[string]Cluster
  26         AutoReloadConfig bool
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         cc, ok := sc.Clusters[clusterID]
  53         if !ok {
  54                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  55         }
  56         cc.ClusterID = clusterID
  57         return &cc, nil
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxBlockEntries      int
  64         MaxCollectionEntries int
  65         MaxCollectionBytes   int64
  66         MaxPermissionEntries int
  67         MaxUUIDEntries       int
  68         MaxSessions          int
  69 }
  70
  71 type Cluster struct {
  72         ClusterID       string `json:"-"`
  73         ManagementToken string
  74         SystemRootToken string
  75         Services        Services
  76         InstanceTypes   InstanceTypeMap
  77         Containers      ContainersConfig
  78         RemoteClusters  map[string]RemoteCluster
  79         PostgreSQL      PostgreSQL
  80
  81         API struct {
  82                 AsyncPermissionsUpdateInterval Duration
  83                 DisabledAPIs                   StringSet
  84                 MaxIndexDatabaseRead           int
  85                 MaxItemsPerResponse            int
  86                 MaxConcurrentRequests          int
  87                 MaxKeepBlobBuffers             int
  88                 MaxRequestAmplification        int
  89                 MaxRequestSize                 int
  90                 MaxTokenLifetime               Duration
  91                 RequestTimeout                 Duration
  92                 SendTimeout                    Duration
  93                 WebsocketClientEventQueue      int
  94                 WebsocketServerEventQueue      int
  95                 KeepServiceRequestTimeout      Duration
  96         }
  97         AuditLogs struct {
  98                 MaxAge             Duration
  99                 MaxDeleteBatch     int
 100                 UnloggedAttributes StringSet
 101         }
 102         Collections struct {
 103                 BlobSigning              bool
 104                 BlobSigningKey           string
 105                 BlobSigningTTL           Duration
 106                 BlobTrash                bool
 107                 BlobTrashLifetime        Duration
 108                 BlobTrashCheckInterval   Duration
 109                 BlobTrashConcurrency     int
 110                 BlobDeleteConcurrency    int
 111                 BlobReplicateConcurrency int
 112                 CollectionVersioning     bool
 113                 DefaultTrashLifetime     Duration
 114                 DefaultReplication       int
 115                 ManagedProperties        map[string]struct {
 116                         Value     interface{}
 117                         Function  string
 118                         Protected bool
 119                 }
 120                 PreserveVersionIfIdle        Duration
 121                 TrashSweepInterval           Duration
 122                 TrustAllContent              bool
 123                 ForwardSlashNameSubstitution string
 124                 S3FolderObjects              bool
 125
 126                 BlobMissingReport        string
 127                 BalancePeriod            Duration
 128                 BalanceCollectionBatch   int
 129                 BalanceCollectionBuffers int
 130                 BalanceTimeout           Duration
 131
 132                 WebDAVCache WebDAVCacheConfig
 133         }
 134         Git struct {
 135                 GitCommand   string
 136                 GitoliteHome string
 137                 Repositories string
 138         }
 139         Login struct {
 140                 LDAP struct {
 141                         Enable             bool
 142                         URL                URL
 143                         StartTLS           bool
 144                         InsecureTLS        bool
 145                         StripDomain        string
 146                         AppendDomain       string
 147                         SearchAttribute    string
 148                         SearchBindUser     string
 149                         SearchBindPassword string
 150                         SearchBase         string
 151                         SearchFilters      string
 152                         EmailAttribute     string
 153                         UsernameAttribute  string
 154                 }
 155                 Google struct {
 156                         Enable                          bool
 157                         ClientID                        string
 158                         ClientSecret                    string
 159                         AlternateEmailAddresses         bool
 160                         AuthenticationRequestParameters map[string]string
 161                 }
 162                 OpenIDConnect struct {
 163                         Enable                          bool
 164                         Issuer                          string
 165                         ClientID                        string
 166                         ClientSecret                    string
 167                         EmailClaim                      string
 168                         EmailVerifiedClaim              string
 169                         UsernameClaim                   string
 170                         AcceptAccessToken               bool
 171                         AcceptAccessTokenScope          string
 172                         AuthenticationRequestParameters map[string]string
 173                 }
 174                 PAM struct {
 175                         Enable             bool
 176                         Service            string
 177                         DefaultEmailDomain string
 178                 }
 179                 SSO struct {
 180                         Enable            bool
 181                         ProviderAppID     string
 182                         ProviderAppSecret string
 183                 }
 184                 Test struct {
 185                         Enable bool
 186                         Users  map[string]TestUser
 187                 }
 188                 LoginCluster       string
 189                 RemoteTokenRefresh Duration
 190                 TokenLifetime      Duration
 191                 TrustedClients     map[string]struct{}
 192                 IssueTrustedTokens bool
 193         }
 194         Mail struct {
 195                 MailchimpAPIKey                string
 196                 MailchimpListID                string
 197                 SendUserSetupNotificationEmail bool
 198                 IssueReporterEmailFrom         string
 199                 IssueReporterEmailTo           string
 200                 SupportEmailAddress            string
 201                 EmailFrom                      string
 202         }
 203         SystemLogs struct {
 204                 LogLevel                string
 205                 Format                  string
 206                 MaxRequestLogParamsSize int
 207         }
 208         TLS struct {
 209                 Certificate string
 210                 Key         string
 211                 Insecure    bool
 212         }
 213         Users struct {
 214                 AnonymousUserToken                    string
 215                 AdminNotifierEmailFrom                string
 216                 AutoAdminFirstUser                    bool
 217                 AutoAdminUserWithEmail                string
 218                 AutoSetupNewUsers                     bool
 219                 AutoSetupNewUsersWithRepository       bool
 220                 AutoSetupNewUsersWithVmUUID           string
 221                 AutoSetupUsernameBlacklist            StringSet
 222                 EmailSubjectPrefix                    string
 223                 NewInactiveUserNotificationRecipients StringSet
 224                 NewUserNotificationRecipients         StringSet
 225                 NewUsersAreActive                     bool
 226                 UserNotifierEmailFrom                 string
 227                 UserProfileNotificationAddress        string
 228                 PreferDomainForUsername               string
 229                 UserSetupMailText                     string
 230         }
 231         Volumes   map[string]Volume
 232         Workbench struct {
 233                 ActivationContactLink            string
 234                 APIClientConnectTimeout          Duration
 235                 APIClientReceiveTimeout          Duration
 236                 APIResponseCompression           bool
 237                 ApplicationMimetypesWithViewIcon StringSet
 238                 ArvadosDocsite                   string
 239                 ArvadosPublicDataDocURL          string
 240                 DefaultOpenIdPrefix              string
 241                 EnableGettingStartedPopup        bool
 242                 EnablePublicProjectsPage         bool
 243                 FileViewersConfigURL             string
 244                 LogViewerMaxBytes                ByteSize
 245                 MultiSiteSearch                  string
 246                 ProfilingEnabled                 bool
 247                 Repositories                     bool
 248                 RepositoryCache                  string
 249                 RunningJobLogRecordsToFetch      int
 250                 SecretKeyBase                    string
 251                 ShowRecentCollectionsOnDashboard bool
 252                 ShowUserAgreementInline          bool
 253                 ShowUserNotifications            bool
 254                 SiteName                         string
 255                 Theme                            string
 256                 UserProfileFormFields            map[string]struct {
 257                         Type                 string
 258                         FormFieldTitle       string
 259                         FormFieldDescription string
 260                         Required             bool
 261                         Position             int
 262                         Options              map[string]struct{}
 263                 }
 264                 UserProfileFormMessage string
 265                 VocabularyURL          string
 266                 WelcomePageHTML        string
 267                 InactivePageHTML       string
 268                 SSHHelpPageHTML        string
 269                 SSHHelpHostSuffix      string
 270                 IdleTimeout            Duration
 271         }
 272 }
 273
 274 type Volume struct {
 275         AccessViaHosts   map[URL]VolumeAccess
 276         ReadOnly         bool
 277         Replication      int
 278         StorageClasses   map[string]bool
 279         Driver           string
 280         DriverParameters json.RawMessage
 281 }
 282
 283 type S3VolumeDriverParameters struct {
 284         IAMRole            string
 285         AccessKeyID        string
 286         SecretAccessKey    string
 287         Endpoint           string
 288         Region             string
 289         Bucket             string
 290         LocationConstraint bool
 291         V2Signature        bool
 292         UseAWSS3v2Driver   bool
 293         IndexPageSize      int
 294         ConnectTimeout     Duration
 295         ReadTimeout        Duration
 296         RaceWindow         Duration
 297         UnsafeDelete       bool
 298 }
 299
 300 type AzureVolumeDriverParameters struct {
 301         StorageAccountName   string
 302         StorageAccountKey    string
 303         StorageBaseURL       string
 304         ContainerName        string
 305         RequestTimeout       Duration
 306         ListBlobsRetryDelay  Duration
 307         ListBlobsMaxAttempts int
 308 }
 309
 310 type DirectoryVolumeDriverParameters struct {
 311         Root      string
 312         Serialize bool
 313 }
 314
 315 type VolumeAccess struct {
 316         ReadOnly bool
 317 }
 318
 319 type Services struct {
 320         Composer       Service
 321         Controller     Service
 322         DispatchCloud  Service
 323         GitHTTP        Service
 324         GitSSH         Service
 325         Health         Service
 326         Keepbalance    Service
 327         Keepproxy      Service
 328         Keepstore      Service
 329         RailsAPI       Service
 330         SSO            Service
 331         WebDAVDownload Service
 332         WebDAV         Service
 333         WebShell       Service
 334         Websocket      Service
 335         Workbench1     Service
 336         Workbench2     Service
 337 }
 338
 339 type Service struct {
 340         InternalURLs map[URL]ServiceInstance
 341         ExternalURL  URL
 342 }
 343
 344 type TestUser struct {
 345         Email    string
 346         Password string
 347 }
 348
 349 // URL is a url.URL that is also usable as a JSON key/value.
 350 type URL url.URL
 351
 352 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 353 // used as a JSON key/value.
 354 func (su *URL) UnmarshalText(text []byte) error {
 355         u, err := url.Parse(string(text))
 356         if err == nil {
 357                 *su = URL(*u)
 358                 if su.Path == "" && su.Host != "" {
 359                         // http://example really means http://example/
 360                         su.Path = "/"
 361                 }
 362         }
 363         return err
 364 }
 365
 366 func (su URL) MarshalText() ([]byte, error) {
 367         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 368 }
 369
 370 func (su URL) String() string {
 371         return (*url.URL)(&su).String()
 372 }
 373
 374 type ServiceInstance struct {
 375         Rendezvous string `json:",omitempty"`
 376 }
 377
 378 type PostgreSQL struct {
 379         Connection     PostgreSQLConnection
 380         ConnectionPool int
 381 }
 382
 383 type PostgreSQLConnection map[string]string
 384
 385 type RemoteCluster struct {
 386         Host          string
 387         Proxy         bool
 388         Scheme        string
 389         Insecure      bool
 390         ActivateUsers bool
 391 }
 392
 393 type InstanceType struct {
 394         Name            string
 395         ProviderType    string
 396         VCPUs           int
 397         RAM             ByteSize
 398         Scratch         ByteSize
 399         IncludedScratch ByteSize
 400         AddedScratch    ByteSize
 401         Price           float64
 402         Preemptible     bool
 403 }
 404
 405 type ContainersConfig struct {
 406         CloudVMs                    CloudVMsConfig
 407         CrunchRunCommand            string
 408         CrunchRunArgumentsList      []string
 409         DefaultKeepCacheRAM         ByteSize
 410         DispatchPrivateKey          string
 411         LogReuseDecisions           bool
 412         MaxComputeVMs               int
 413         MaxDispatchAttempts         int
 414         MaxRetryAttempts            int
 415         MinRetryPeriod              Duration
 416         ReserveExtraRAM             ByteSize
 417         StaleLockTimeout            Duration
 418         SupportedDockerImageFormats StringSet
 419         UsePreemptibleInstances     bool
 420         RuntimeEngine               string
 421
 422         JobsAPI struct {
 423                 Enable         string
 424                 GitInternalDir string
 425         }
 426         Logging struct {
 427                 MaxAge                       Duration
 428                 LogBytesPerEvent             int
 429                 LogSecondsBetweenEvents      Duration
 430                 LogThrottlePeriod            Duration
 431                 LogThrottleBytes             int
 432                 LogThrottleLines             int
 433                 LimitLogBytesPerJob          int
 434                 LogPartialLineThrottlePeriod Duration
 435                 LogUpdatePeriod              Duration
 436                 LogUpdateSize                ByteSize
 437         }
 438         ShellAccess struct {
 439                 Admin bool
 440                 User  bool
 441         }
 442         SLURM struct {
 443                 PrioritySpread             int64
 444                 SbatchArgumentsList        []string
 445                 SbatchEnvironmentVariables map[string]string
 446                 Managed                    struct {
 447                         DNSServerConfDir       string
 448                         DNSServerConfTemplate  string
 449                         DNSServerReloadCommand string
 450                         DNSServerUpdateCommand string
 451                         ComputeNodeDomain      string
 452                         ComputeNodeNameservers StringSet
 453                         AssignNodeHostname     string
 454                 }
 455         }
 456 }
 457
 458 type CloudVMsConfig struct {
 459         Enable bool
 460
 461         BootProbeCommand               string
 462         DeployRunnerBinary             string
 463         ImageID                        string
 464         MaxCloudOpsPerSecond           int
 465         MaxProbesPerSecond             int
 466         MaxConcurrentInstanceCreateOps int
 467         PollInterval                   Duration
 468         ProbeInterval                  Duration
 469         SSHPort                        string
 470         SyncInterval                   Duration
 471         TimeoutBooting                 Duration
 472         TimeoutIdle                    Duration
 473         TimeoutProbe                   Duration
 474         TimeoutShutdown                Duration
 475         TimeoutSignal                  Duration
 476         TimeoutStaleRunLock            Duration
 477         TimeoutTERM                    Duration
 478         ResourceTags                   map[string]string
 479         TagKeyPrefix                   string
 480
 481         Driver           string
 482         DriverParameters json.RawMessage
 483 }
 484
 485 type InstanceTypeMap map[string]InstanceType
 486
 487 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 488
 489 // UnmarshalJSON handles old config files that provide an array of
 490 // instance types instead of a hash.
 491 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 492         fixup := func(t InstanceType) (InstanceType, error) {
 493                 if t.ProviderType == "" {
 494                         t.ProviderType = t.Name
 495                 }
 496                 if t.Scratch == 0 {
 497                         t.Scratch = t.IncludedScratch + t.AddedScratch
 498                 } else if t.AddedScratch == 0 {
 499                         t.AddedScratch = t.Scratch - t.IncludedScratch
 500                 } else if t.IncludedScratch == 0 {
 501                         t.IncludedScratch = t.Scratch - t.AddedScratch
 502                 }
 503
 504                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 505                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 506                 }
 507                 return t, nil
 508         }
 509
 510         if len(data) > 0 && data[0] == '[' {
 511                 var arr []InstanceType
 512                 err := json.Unmarshal(data, &arr)
 513                 if err != nil {
 514                         return err
 515                 }
 516                 if len(arr) == 0 {
 517                         *it = nil
 518                         return nil
 519                 }
 520                 *it = make(map[string]InstanceType, len(arr))
 521                 for _, t := range arr {
 522                         if _, ok := (*it)[t.Name]; ok {
 523                                 return errDuplicateInstanceTypeName
 524                         }
 525                         t, err := fixup(t)
 526                         if err != nil {
 527                                 return err
 528                         }
 529                         (*it)[t.Name] = t
 530                 }
 531                 return nil
 532         }
 533         var hash map[string]InstanceType
 534         err := json.Unmarshal(data, &hash)
 535         if err != nil {
 536                 return err
 537         }
 538         // Fill in Name field (and ProviderType field, if not
 539         // specified) using hash key.
 540         *it = InstanceTypeMap(hash)
 541         for name, t := range *it {
 542                 t.Name = name
 543                 t, err := fixup(t)
 544                 if err != nil {
 545                         return err
 546                 }
 547                 (*it)[name] = t
 548         }
 549         return nil
 550 }
 551
 552 type StringSet map[string]struct{}
 553
 554 // UnmarshalJSON handles old config files that provide an array of
 555 // instance types instead of a hash.
 556 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 557         if len(data) > 0 && data[0] == '[' {
 558                 var arr []string
 559                 err := json.Unmarshal(data, &arr)
 560                 if err != nil {
 561                         return err
 562                 }
 563                 if len(arr) == 0 {
 564                         *ss = nil
 565                         return nil
 566                 }
 567                 *ss = make(map[string]struct{}, len(arr))
 568                 for _, t := range arr {
 569                         (*ss)[t] = struct{}{}
 570                 }
 571                 return nil
 572         }
 573         var hash map[string]struct{}
 574         err := json.Unmarshal(data, &hash)
 575         if err != nil {
 576                 return err
 577         }
 578         *ss = make(map[string]struct{}, len(hash))
 579         for t := range hash {
 580                 (*ss)[t] = struct{}{}
 581         }
 582
 583         return nil
 584 }
 585
 586 type ServiceName string
 587
 588 const (
 589         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 590         ServiceNameController    ServiceName = "arvados-controller"
 591         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 592         ServiceNameHealth        ServiceName = "arvados-health"
 593         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 594         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 595         ServiceNameWebsocket     ServiceName = "arvados-ws"
 596         ServiceNameKeepbalance   ServiceName = "keep-balance"
 597         ServiceNameKeepweb       ServiceName = "keep-web"
 598         ServiceNameKeepproxy     ServiceName = "keepproxy"
 599         ServiceNameKeepstore     ServiceName = "keepstore"
 600 )
 601
 602 // Map returns all services as a map, suitable for iterating over all
 603 // services or looking up a service by name.
 604 func (svcs Services) Map() map[ServiceName]Service {
 605         return map[ServiceName]Service{
 606                 ServiceNameRailsAPI:      svcs.RailsAPI,
 607                 ServiceNameController:    svcs.Controller,
 608                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 609                 ServiceNameHealth:        svcs.Health,
 610                 ServiceNameWorkbench1:    svcs.Workbench1,
 611                 ServiceNameWorkbench2:    svcs.Workbench2,
 612                 ServiceNameWebsocket:     svcs.Websocket,
 613                 ServiceNameKeepbalance:   svcs.Keepbalance,
 614                 ServiceNameKeepweb:       svcs.WebDAV,
 615                 ServiceNameKeepproxy:     svcs.Keepproxy,
 616                 ServiceNameKeepstore:     svcs.Keepstore,
 617         }
 618 }