sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.arvados.org/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         }
  21         return "/etc/arvados/config.yml"
  22 }()
  23
  24 type Config struct {
  25         Clusters         map[string]Cluster
  26         AutoReloadConfig bool
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         cc, ok := sc.Clusters[clusterID]
  53         if !ok {
  54                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  55         }
  56         cc.ClusterID = clusterID
  57         return &cc, nil
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxBlockEntries      int
  64         MaxCollectionEntries int
  65         MaxCollectionBytes   int64
  66         MaxPermissionEntries int
  67         MaxUUIDEntries       int
  68         MaxSessions          int
  69 }
  70
  71 type Cluster struct {
  72         ClusterID       string `json:"-"`
  73         ManagementToken string
  74         SystemRootToken string
  75         Services        Services
  76         InstanceTypes   InstanceTypeMap
  77         Containers      ContainersConfig
  78         RemoteClusters  map[string]RemoteCluster
  79         PostgreSQL      PostgreSQL
  80
  81         API struct {
  82                 AsyncPermissionsUpdateInterval Duration
  83                 DisabledAPIs                   StringSet
  84                 MaxIndexDatabaseRead           int
  85                 MaxItemsPerResponse            int
  86                 MaxConcurrentRequests          int
  87                 MaxKeepBlobBuffers             int
  88                 MaxRequestAmplification        int
  89                 MaxRequestSize                 int
  90                 MaxTokenLifetime               Duration
  91                 RequestTimeout                 Duration
  92                 SendTimeout                    Duration
  93                 WebsocketClientEventQueue      int
  94                 WebsocketServerEventQueue      int
  95                 KeepServiceRequestTimeout      Duration
  96         }
  97         AuditLogs struct {
  98                 MaxAge             Duration
  99                 MaxDeleteBatch     int
 100                 UnloggedAttributes StringSet
 101         }
 102         Collections struct {
 103                 BlobSigning              bool
 104                 BlobSigningKey           string
 105                 BlobSigningTTL           Duration
 106                 BlobTrash                bool
 107                 BlobTrashLifetime        Duration
 108                 BlobTrashCheckInterval   Duration
 109                 BlobTrashConcurrency     int
 110                 BlobDeleteConcurrency    int
 111                 BlobReplicateConcurrency int
 112                 CollectionVersioning     bool
 113                 DefaultTrashLifetime     Duration
 114                 DefaultReplication       int
 115                 ManagedProperties        map[string]struct {
 116                         Value     interface{}
 117                         Function  string
 118                         Protected bool
 119                 }
 120                 PreserveVersionIfIdle        Duration
 121                 TrashSweepInterval           Duration
 122                 TrustAllContent              bool
 123                 ForwardSlashNameSubstitution string
 124                 S3FolderObjects              bool
 125
 126                 BlobMissingReport        string
 127                 BalancePeriod            Duration
 128                 BalanceCollectionBatch   int
 129                 BalanceCollectionBuffers int
 130                 BalanceTimeout           Duration
 131
 132                 WebDAVCache WebDAVCacheConfig
 133         }
 134         Git struct {
 135                 GitCommand   string
 136                 GitoliteHome string
 137                 Repositories string
 138         }
 139         Login struct {
 140                 LDAP struct {
 141                         Enable             bool
 142                         URL                URL
 143                         StartTLS           bool
 144                         InsecureTLS        bool
 145                         StripDomain        string
 146                         AppendDomain       string
 147                         SearchAttribute    string
 148                         SearchBindUser     string
 149                         SearchBindPassword string
 150                         SearchBase         string
 151                         SearchFilters      string
 152                         EmailAttribute     string
 153                         UsernameAttribute  string
 154                 }
 155                 Google struct {
 156                         Enable                          bool
 157                         ClientID                        string
 158                         ClientSecret                    string
 159                         AlternateEmailAddresses         bool
 160                         AuthenticationRequestParameters map[string]string
 161                 }
 162                 OpenIDConnect struct {
 163                         Enable                          bool
 164                         Issuer                          string
 165                         ClientID                        string
 166                         ClientSecret                    string
 167                         EmailClaim                      string
 168                         EmailVerifiedClaim              string
 169                         UsernameClaim                   string
 170                         AuthenticationRequestParameters map[string]string
 171                 }
 172                 PAM struct {
 173                         Enable             bool
 174                         Service            string
 175                         DefaultEmailDomain string
 176                 }
 177                 SSO struct {
 178                         Enable            bool
 179                         ProviderAppID     string
 180                         ProviderAppSecret string
 181                 }
 182                 Test struct {
 183                         Enable bool
 184                         Users  map[string]TestUser
 185                 }
 186                 LoginCluster       string
 187                 RemoteTokenRefresh Duration
 188                 TokenLifetime      Duration
 189                 TrustedClients     map[string]struct{}
 190         }
 191         Mail struct {
 192                 MailchimpAPIKey                string
 193                 MailchimpListID                string
 194                 SendUserSetupNotificationEmail bool
 195                 IssueReporterEmailFrom         string
 196                 IssueReporterEmailTo           string
 197                 SupportEmailAddress            string
 198                 EmailFrom                      string
 199         }
 200         SystemLogs struct {
 201                 LogLevel                string
 202                 Format                  string
 203                 MaxRequestLogParamsSize int
 204         }
 205         TLS struct {
 206                 Certificate string
 207                 Key         string
 208                 Insecure    bool
 209         }
 210         Users struct {
 211                 AnonymousUserToken                    string
 212                 AdminNotifierEmailFrom                string
 213                 AutoAdminFirstUser                    bool
 214                 AutoAdminUserWithEmail                string
 215                 AutoSetupNewUsers                     bool
 216                 AutoSetupNewUsersWithRepository       bool
 217                 AutoSetupNewUsersWithVmUUID           string
 218                 AutoSetupUsernameBlacklist            StringSet
 219                 EmailSubjectPrefix                    string
 220                 NewInactiveUserNotificationRecipients StringSet
 221                 NewUserNotificationRecipients         StringSet
 222                 NewUsersAreActive                     bool
 223                 UserNotifierEmailFrom                 string
 224                 UserProfileNotificationAddress        string
 225                 PreferDomainForUsername               string
 226                 UserSetupMailText                     string
 227         }
 228         Volumes   map[string]Volume
 229         Workbench struct {
 230                 ActivationContactLink            string
 231                 APIClientConnectTimeout          Duration
 232                 APIClientReceiveTimeout          Duration
 233                 APIResponseCompression           bool
 234                 ApplicationMimetypesWithViewIcon StringSet
 235                 ArvadosDocsite                   string
 236                 ArvadosPublicDataDocURL          string
 237                 DefaultOpenIdPrefix              string
 238                 EnableGettingStartedPopup        bool
 239                 EnablePublicProjectsPage         bool
 240                 FileViewersConfigURL             string
 241                 LogViewerMaxBytes                ByteSize
 242                 MultiSiteSearch                  string
 243                 ProfilingEnabled                 bool
 244                 Repositories                     bool
 245                 RepositoryCache                  string
 246                 RunningJobLogRecordsToFetch      int
 247                 SecretKeyBase                    string
 248                 ShowRecentCollectionsOnDashboard bool
 249                 ShowUserAgreementInline          bool
 250                 ShowUserNotifications            bool
 251                 SiteName                         string
 252                 Theme                            string
 253                 UserProfileFormFields            map[string]struct {
 254                         Type                 string
 255                         FormFieldTitle       string
 256                         FormFieldDescription string
 257                         Required             bool
 258                         Position             int
 259                         Options              map[string]struct{}
 260                 }
 261                 UserProfileFormMessage string
 262                 VocabularyURL          string
 263                 WelcomePageHTML        string
 264                 InactivePageHTML       string
 265                 SSHHelpPageHTML        string
 266                 SSHHelpHostSuffix      string
 267                 IdleTimeout            Duration
 268         }
 269 }
 270
 271 type Volume struct {
 272         AccessViaHosts   map[URL]VolumeAccess
 273         ReadOnly         bool
 274         Replication      int
 275         StorageClasses   map[string]bool
 276         Driver           string
 277         DriverParameters json.RawMessage
 278 }
 279
 280 type S3VolumeDriverParameters struct {
 281         IAMRole            string
 282         AccessKeyID        string
 283         SecretAccessKey    string
 284         Endpoint           string
 285         Region             string
 286         Bucket             string
 287         LocationConstraint bool
 288         V2Signature        bool
 289         UseAWSS3v2Driver   bool
 290         IndexPageSize      int
 291         ConnectTimeout     Duration
 292         ReadTimeout        Duration
 293         RaceWindow         Duration
 294         UnsafeDelete       bool
 295 }
 296
 297 type AzureVolumeDriverParameters struct {
 298         StorageAccountName   string
 299         StorageAccountKey    string
 300         StorageBaseURL       string
 301         ContainerName        string
 302         RequestTimeout       Duration
 303         ListBlobsRetryDelay  Duration
 304         ListBlobsMaxAttempts int
 305 }
 306
 307 type DirectoryVolumeDriverParameters struct {
 308         Root      string
 309         Serialize bool
 310 }
 311
 312 type VolumeAccess struct {
 313         ReadOnly bool
 314 }
 315
 316 type Services struct {
 317         Composer       Service
 318         Controller     Service
 319         DispatchCloud  Service
 320         GitHTTP        Service
 321         GitSSH         Service
 322         Health         Service
 323         Keepbalance    Service
 324         Keepproxy      Service
 325         Keepstore      Service
 326         RailsAPI       Service
 327         SSO            Service
 328         WebDAVDownload Service
 329         WebDAV         Service
 330         WebShell       Service
 331         Websocket      Service
 332         Workbench1     Service
 333         Workbench2     Service
 334 }
 335
 336 type Service struct {
 337         InternalURLs map[URL]ServiceInstance
 338         ExternalURL  URL
 339 }
 340
 341 type TestUser struct {
 342         Email    string
 343         Password string
 344 }
 345
 346 // URL is a url.URL that is also usable as a JSON key/value.
 347 type URL url.URL
 348
 349 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 350 // used as a JSON key/value.
 351 func (su *URL) UnmarshalText(text []byte) error {
 352         u, err := url.Parse(string(text))
 353         if err == nil {
 354                 *su = URL(*u)
 355                 if su.Path == "" && su.Host != "" {
 356                         // http://example really means http://example/
 357                         su.Path = "/"
 358                 }
 359         }
 360         return err
 361 }
 362
 363 func (su URL) MarshalText() ([]byte, error) {
 364         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 365 }
 366
 367 func (su URL) String() string {
 368         return (*url.URL)(&su).String()
 369 }
 370
 371 type ServiceInstance struct {
 372         Rendezvous string `json:",omitempty"`
 373 }
 374
 375 type PostgreSQL struct {
 376         Connection     PostgreSQLConnection
 377         ConnectionPool int
 378 }
 379
 380 type PostgreSQLConnection map[string]string
 381
 382 type RemoteCluster struct {
 383         Host          string
 384         Proxy         bool
 385         Scheme        string
 386         Insecure      bool
 387         ActivateUsers bool
 388 }
 389
 390 type InstanceType struct {
 391         Name            string
 392         ProviderType    string
 393         VCPUs           int
 394         RAM             ByteSize
 395         Scratch         ByteSize
 396         IncludedScratch ByteSize
 397         AddedScratch    ByteSize
 398         Price           float64
 399         Preemptible     bool
 400 }
 401
 402 type ContainersConfig struct {
 403         CloudVMs                    CloudVMsConfig
 404         CrunchRunCommand            string
 405         CrunchRunArgumentsList      []string
 406         DefaultKeepCacheRAM         ByteSize
 407         DispatchPrivateKey          string
 408         LogReuseDecisions           bool
 409         MaxComputeVMs               int
 410         MaxDispatchAttempts         int
 411         MaxRetryAttempts            int
 412         MinRetryPeriod              Duration
 413         ReserveExtraRAM             ByteSize
 414         StaleLockTimeout            Duration
 415         SupportedDockerImageFormats StringSet
 416         UsePreemptibleInstances     bool
 417
 418         JobsAPI struct {
 419                 Enable         string
 420                 GitInternalDir string
 421         }
 422         Logging struct {
 423                 MaxAge                       Duration
 424                 LogBytesPerEvent             int
 425                 LogSecondsBetweenEvents      Duration
 426                 LogThrottlePeriod            Duration
 427                 LogThrottleBytes             int
 428                 LogThrottleLines             int
 429                 LimitLogBytesPerJob          int
 430                 LogPartialLineThrottlePeriod Duration
 431                 LogUpdatePeriod              Duration
 432                 LogUpdateSize                ByteSize
 433         }
 434         ShellAccess struct {
 435                 Admin bool
 436                 User  bool
 437         }
 438         SLURM struct {
 439                 PrioritySpread             int64
 440                 SbatchArgumentsList        []string
 441                 SbatchEnvironmentVariables map[string]string
 442                 Managed                    struct {
 443                         DNSServerConfDir       string
 444                         DNSServerConfTemplate  string
 445                         DNSServerReloadCommand string
 446                         DNSServerUpdateCommand string
 447                         ComputeNodeDomain      string
 448                         ComputeNodeNameservers StringSet
 449                         AssignNodeHostname     string
 450                 }
 451         }
 452 }
 453
 454 type CloudVMsConfig struct {
 455         Enable bool
 456
 457         BootProbeCommand               string
 458         DeployRunnerBinary             string
 459         ImageID                        string
 460         MaxCloudOpsPerSecond           int
 461         MaxProbesPerSecond             int
 462         MaxConcurrentInstanceCreateOps int
 463         PollInterval                   Duration
 464         ProbeInterval                  Duration
 465         SSHPort                        string
 466         SyncInterval                   Duration
 467         TimeoutBooting                 Duration
 468         TimeoutIdle                    Duration
 469         TimeoutProbe                   Duration
 470         TimeoutShutdown                Duration
 471         TimeoutSignal                  Duration
 472         TimeoutStaleRunLock            Duration
 473         TimeoutTERM                    Duration
 474         ResourceTags                   map[string]string
 475         TagKeyPrefix                   string
 476
 477         Driver           string
 478         DriverParameters json.RawMessage
 479 }
 480
 481 type InstanceTypeMap map[string]InstanceType
 482
 483 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 484
 485 // UnmarshalJSON handles old config files that provide an array of
 486 // instance types instead of a hash.
 487 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 488         fixup := func(t InstanceType) (InstanceType, error) {
 489                 if t.ProviderType == "" {
 490                         t.ProviderType = t.Name
 491                 }
 492                 if t.Scratch == 0 {
 493                         t.Scratch = t.IncludedScratch + t.AddedScratch
 494                 } else if t.AddedScratch == 0 {
 495                         t.AddedScratch = t.Scratch - t.IncludedScratch
 496                 } else if t.IncludedScratch == 0 {
 497                         t.IncludedScratch = t.Scratch - t.AddedScratch
 498                 }
 499
 500                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 501                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 502                 }
 503                 return t, nil
 504         }
 505
 506         if len(data) > 0 && data[0] == '[' {
 507                 var arr []InstanceType
 508                 err := json.Unmarshal(data, &arr)
 509                 if err != nil {
 510                         return err
 511                 }
 512                 if len(arr) == 0 {
 513                         *it = nil
 514                         return nil
 515                 }
 516                 *it = make(map[string]InstanceType, len(arr))
 517                 for _, t := range arr {
 518                         if _, ok := (*it)[t.Name]; ok {
 519                                 return errDuplicateInstanceTypeName
 520                         }
 521                         t, err := fixup(t)
 522                         if err != nil {
 523                                 return err
 524                         }
 525                         (*it)[t.Name] = t
 526                 }
 527                 return nil
 528         }
 529         var hash map[string]InstanceType
 530         err := json.Unmarshal(data, &hash)
 531         if err != nil {
 532                 return err
 533         }
 534         // Fill in Name field (and ProviderType field, if not
 535         // specified) using hash key.
 536         *it = InstanceTypeMap(hash)
 537         for name, t := range *it {
 538                 t.Name = name
 539                 t, err := fixup(t)
 540                 if err != nil {
 541                         return err
 542                 }
 543                 (*it)[name] = t
 544         }
 545         return nil
 546 }
 547
 548 type StringSet map[string]struct{}
 549
 550 // UnmarshalJSON handles old config files that provide an array of
 551 // instance types instead of a hash.
 552 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 553         if len(data) > 0 && data[0] == '[' {
 554                 var arr []string
 555                 err := json.Unmarshal(data, &arr)
 556                 if err != nil {
 557                         return err
 558                 }
 559                 if len(arr) == 0 {
 560                         *ss = nil
 561                         return nil
 562                 }
 563                 *ss = make(map[string]struct{}, len(arr))
 564                 for _, t := range arr {
 565                         (*ss)[t] = struct{}{}
 566                 }
 567                 return nil
 568         }
 569         var hash map[string]struct{}
 570         err := json.Unmarshal(data, &hash)
 571         if err != nil {
 572                 return err
 573         }
 574         *ss = make(map[string]struct{}, len(hash))
 575         for t := range hash {
 576                 (*ss)[t] = struct{}{}
 577         }
 578
 579         return nil
 580 }
 581
 582 type ServiceName string
 583
 584 const (
 585         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 586         ServiceNameController    ServiceName = "arvados-controller"
 587         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 588         ServiceNameHealth        ServiceName = "arvados-health"
 589         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 590         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 591         ServiceNameWebsocket     ServiceName = "arvados-ws"
 592         ServiceNameKeepbalance   ServiceName = "keep-balance"
 593         ServiceNameKeepweb       ServiceName = "keep-web"
 594         ServiceNameKeepproxy     ServiceName = "keepproxy"
 595         ServiceNameKeepstore     ServiceName = "keepstore"
 596 )
 597
 598 // Map returns all services as a map, suitable for iterating over all
 599 // services or looking up a service by name.
 600 func (svcs Services) Map() map[ServiceName]Service {
 601         return map[ServiceName]Service{
 602                 ServiceNameRailsAPI:      svcs.RailsAPI,
 603                 ServiceNameController:    svcs.Controller,
 604                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 605                 ServiceNameHealth:        svcs.Health,
 606                 ServiceNameWorkbench1:    svcs.Workbench1,
 607                 ServiceNameWorkbench2:    svcs.Workbench2,
 608                 ServiceNameWebsocket:     svcs.Websocket,
 609                 ServiceNameKeepbalance:   svcs.Keepbalance,
 610                 ServiceNameKeepweb:       svcs.WebDAV,
 611                 ServiceNameKeepproxy:     svcs.Keepproxy,
 612                 ServiceNameKeepstore:     svcs.Keepstore,
 613         }
 614 }