sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.arvados.org/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         }
  21         return "/etc/arvados/config.yml"
  22 }()
  23
  24 type Config struct {
  25         Clusters         map[string]Cluster
  26         AutoReloadConfig bool
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         cc, ok := sc.Clusters[clusterID]
  53         if !ok {
  54                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  55         }
  56         cc.ClusterID = clusterID
  57         return &cc, nil
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxBlockEntries      int
  64         MaxCollectionEntries int
  65         MaxCollectionBytes   int64
  66         MaxPermissionEntries int
  67         MaxUUIDEntries       int
  68         MaxSessions          int
  69 }
  70
  71 type Cluster struct {
  72         ClusterID       string `json:"-"`
  73         ManagementToken string
  74         SystemRootToken string
  75         Services        Services
  76         InstanceTypes   InstanceTypeMap
  77         Containers      ContainersConfig
  78         RemoteClusters  map[string]RemoteCluster
  79         PostgreSQL      PostgreSQL
  80
  81         API struct {
  82                 AsyncPermissionsUpdateInterval Duration
  83                 DisabledAPIs                   StringSet
  84                 MaxIndexDatabaseRead           int
  85                 MaxItemsPerResponse            int
  86                 MaxConcurrentRequests          int
  87                 MaxKeepBlobBuffers             int
  88                 MaxRequestAmplification        int
  89                 MaxRequestSize                 int
  90                 MaxTokenLifetime               Duration
  91                 RequestTimeout                 Duration
  92                 SendTimeout                    Duration
  93                 WebsocketClientEventQueue      int
  94                 WebsocketServerEventQueue      int
  95                 KeepServiceRequestTimeout      Duration
  96         }
  97         AuditLogs struct {
  98                 MaxAge             Duration
  99                 MaxDeleteBatch     int
 100                 UnloggedAttributes StringSet
 101         }
 102         Collections struct {
 103                 BlobSigning              bool
 104                 BlobSigningKey           string
 105                 BlobSigningTTL           Duration
 106                 BlobTrash                bool
 107                 BlobTrashLifetime        Duration
 108                 BlobTrashCheckInterval   Duration
 109                 BlobTrashConcurrency     int
 110                 BlobDeleteConcurrency    int
 111                 BlobReplicateConcurrency int
 112                 CollectionVersioning     bool
 113                 DefaultTrashLifetime     Duration
 114                 DefaultReplication       int
 115                 ManagedProperties        map[string]struct {
 116                         Value     interface{}
 117                         Function  string
 118                         Protected bool
 119                 }
 120                 PreserveVersionIfIdle        Duration
 121                 TrashSweepInterval           Duration
 122                 TrustAllContent              bool
 123                 ForwardSlashNameSubstitution string
 124                 S3FolderObjects              bool
 125
 126                 BlobMissingReport        string
 127                 BalancePeriod            Duration
 128                 BalanceCollectionBatch   int
 129                 BalanceCollectionBuffers int
 130                 BalanceTimeout           Duration
 131
 132                 WebDAVCache WebDAVCacheConfig
 133         }
 134         Git struct {
 135                 GitCommand   string
 136                 GitoliteHome string
 137                 Repositories string
 138         }
 139         Login struct {
 140                 LDAP struct {
 141                         Enable             bool
 142                         URL                URL
 143                         StartTLS           bool
 144                         InsecureTLS        bool
 145                         StripDomain        string
 146                         AppendDomain       string
 147                         SearchAttribute    string
 148                         SearchBindUser     string
 149                         SearchBindPassword string
 150                         SearchBase         string
 151                         SearchFilters      string
 152                         EmailAttribute     string
 153                         UsernameAttribute  string
 154                 }
 155                 Google struct {
 156                         Enable                          bool
 157                         ClientID                        string
 158                         ClientSecret                    string
 159                         AlternateEmailAddresses         bool
 160                         AuthenticationRequestParameters map[string]string
 161                 }
 162                 OpenIDConnect struct {
 163                         Enable                          bool
 164                         Issuer                          string
 165                         ClientID                        string
 166                         ClientSecret                    string
 167                         EmailClaim                      string
 168                         EmailVerifiedClaim              string
 169                         UsernameClaim                   string
 170                         AcceptAccessToken               bool
 171                         AcceptAccessTokenScope          string
 172                         AuthenticationRequestParameters map[string]string
 173                 }
 174                 PAM struct {
 175                         Enable             bool
 176                         Service            string
 177                         DefaultEmailDomain string
 178                 }
 179                 Test struct {
 180                         Enable bool
 181                         Users  map[string]TestUser
 182                 }
 183                 LoginCluster       string
 184                 RemoteTokenRefresh Duration
 185                 TokenLifetime      Duration
 186                 TrustedClients     map[string]struct{}
 187                 IssueTrustedTokens bool
 188         }
 189         Mail struct {
 190                 MailchimpAPIKey                string
 191                 MailchimpListID                string
 192                 SendUserSetupNotificationEmail bool
 193                 IssueReporterEmailFrom         string
 194                 IssueReporterEmailTo           string
 195                 SupportEmailAddress            string
 196                 EmailFrom                      string
 197         }
 198         SystemLogs struct {
 199                 LogLevel                string
 200                 Format                  string
 201                 MaxRequestLogParamsSize int
 202         }
 203         TLS struct {
 204                 Certificate string
 205                 Key         string
 206                 Insecure    bool
 207         }
 208         Users struct {
 209                 AnonymousUserToken                    string
 210                 AdminNotifierEmailFrom                string
 211                 AutoAdminFirstUser                    bool
 212                 AutoAdminUserWithEmail                string
 213                 AutoSetupNewUsers                     bool
 214                 AutoSetupNewUsersWithRepository       bool
 215                 AutoSetupNewUsersWithVmUUID           string
 216                 AutoSetupUsernameBlacklist            StringSet
 217                 EmailSubjectPrefix                    string
 218                 NewInactiveUserNotificationRecipients StringSet
 219                 NewUserNotificationRecipients         StringSet
 220                 NewUsersAreActive                     bool
 221                 UserNotifierEmailFrom                 string
 222                 UserProfileNotificationAddress        string
 223                 PreferDomainForUsername               string
 224                 UserSetupMailText                     string
 225         }
 226         Volumes   map[string]Volume
 227         Workbench struct {
 228                 ActivationContactLink            string
 229                 APIClientConnectTimeout          Duration
 230                 APIClientReceiveTimeout          Duration
 231                 APIResponseCompression           bool
 232                 ApplicationMimetypesWithViewIcon StringSet
 233                 ArvadosDocsite                   string
 234                 ArvadosPublicDataDocURL          string
 235                 DefaultOpenIdPrefix              string
 236                 EnableGettingStartedPopup        bool
 237                 EnablePublicProjectsPage         bool
 238                 FileViewersConfigURL             string
 239                 LogViewerMaxBytes                ByteSize
 240                 MultiSiteSearch                  string
 241                 ProfilingEnabled                 bool
 242                 Repositories                     bool
 243                 RepositoryCache                  string
 244                 RunningJobLogRecordsToFetch      int
 245                 SecretKeyBase                    string
 246                 ShowRecentCollectionsOnDashboard bool
 247                 ShowUserAgreementInline          bool
 248                 ShowUserNotifications            bool
 249                 SiteName                         string
 250                 Theme                            string
 251                 UserProfileFormFields            map[string]struct {
 252                         Type                 string
 253                         FormFieldTitle       string
 254                         FormFieldDescription string
 255                         Required             bool
 256                         Position             int
 257                         Options              map[string]struct{}
 258                 }
 259                 UserProfileFormMessage string
 260                 VocabularyURL          string
 261                 WelcomePageHTML        string
 262                 InactivePageHTML       string
 263                 SSHHelpPageHTML        string
 264                 SSHHelpHostSuffix      string
 265                 IdleTimeout            Duration
 266         }
 267 }
 268
 269 type Volume struct {
 270         AccessViaHosts   map[URL]VolumeAccess
 271         ReadOnly         bool
 272         Replication      int
 273         StorageClasses   map[string]bool
 274         Driver           string
 275         DriverParameters json.RawMessage
 276 }
 277
 278 type S3VolumeDriverParameters struct {
 279         IAMRole            string
 280         AccessKeyID        string
 281         SecretAccessKey    string
 282         Endpoint           string
 283         Region             string
 284         Bucket             string
 285         LocationConstraint bool
 286         V2Signature        bool
 287         UseAWSS3v2Driver   bool
 288         IndexPageSize      int
 289         ConnectTimeout     Duration
 290         ReadTimeout        Duration
 291         RaceWindow         Duration
 292         UnsafeDelete       bool
 293 }
 294
 295 type AzureVolumeDriverParameters struct {
 296         StorageAccountName   string
 297         StorageAccountKey    string
 298         StorageBaseURL       string
 299         ContainerName        string
 300         RequestTimeout       Duration
 301         ListBlobsRetryDelay  Duration
 302         ListBlobsMaxAttempts int
 303 }
 304
 305 type DirectoryVolumeDriverParameters struct {
 306         Root      string
 307         Serialize bool
 308 }
 309
 310 type VolumeAccess struct {
 311         ReadOnly bool
 312 }
 313
 314 type Services struct {
 315         Composer       Service
 316         Controller     Service
 317         DispatchCloud  Service
 318         GitHTTP        Service
 319         GitSSH         Service
 320         Health         Service
 321         Keepbalance    Service
 322         Keepproxy      Service
 323         Keepstore      Service
 324         RailsAPI       Service
 325         SSO            Service
 326         WebDAVDownload Service
 327         WebDAV         Service
 328         WebShell       Service
 329         Websocket      Service
 330         Workbench1     Service
 331         Workbench2     Service
 332 }
 333
 334 type Service struct {
 335         InternalURLs map[URL]ServiceInstance
 336         ExternalURL  URL
 337 }
 338
 339 type TestUser struct {
 340         Email    string
 341         Password string
 342 }
 343
 344 // URL is a url.URL that is also usable as a JSON key/value.
 345 type URL url.URL
 346
 347 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 348 // used as a JSON key/value.
 349 func (su *URL) UnmarshalText(text []byte) error {
 350         u, err := url.Parse(string(text))
 351         if err == nil {
 352                 *su = URL(*u)
 353                 if su.Path == "" && su.Host != "" {
 354                         // http://example really means http://example/
 355                         su.Path = "/"
 356                 }
 357         }
 358         return err
 359 }
 360
 361 func (su URL) MarshalText() ([]byte, error) {
 362         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 363 }
 364
 365 func (su URL) String() string {
 366         return (*url.URL)(&su).String()
 367 }
 368
 369 type ServiceInstance struct {
 370         Rendezvous string `json:",omitempty"`
 371 }
 372
 373 type PostgreSQL struct {
 374         Connection     PostgreSQLConnection
 375         ConnectionPool int
 376 }
 377
 378 type PostgreSQLConnection map[string]string
 379
 380 type RemoteCluster struct {
 381         Host          string
 382         Proxy         bool
 383         Scheme        string
 384         Insecure      bool
 385         ActivateUsers bool
 386 }
 387
 388 type InstanceType struct {
 389         Name            string
 390         ProviderType    string
 391         VCPUs           int
 392         RAM             ByteSize
 393         Scratch         ByteSize
 394         IncludedScratch ByteSize
 395         AddedScratch    ByteSize
 396         Price           float64
 397         Preemptible     bool
 398 }
 399
 400 type ContainersConfig struct {
 401         CloudVMs                    CloudVMsConfig
 402         CrunchRunCommand            string
 403         CrunchRunArgumentsList      []string
 404         DefaultKeepCacheRAM         ByteSize
 405         DispatchPrivateKey          string
 406         LogReuseDecisions           bool
 407         MaxComputeVMs               int
 408         MaxDispatchAttempts         int
 409         MaxRetryAttempts            int
 410         MinRetryPeriod              Duration
 411         ReserveExtraRAM             ByteSize
 412         StaleLockTimeout            Duration
 413         SupportedDockerImageFormats StringSet
 414         UsePreemptibleInstances     bool
 415         RuntimeEngine               string
 416
 417         JobsAPI struct {
 418                 Enable         string
 419                 GitInternalDir string
 420         }
 421         Logging struct {
 422                 MaxAge                       Duration
 423                 LogBytesPerEvent             int
 424                 LogSecondsBetweenEvents      Duration
 425                 LogThrottlePeriod            Duration
 426                 LogThrottleBytes             int
 427                 LogThrottleLines             int
 428                 LimitLogBytesPerJob          int
 429                 LogPartialLineThrottlePeriod Duration
 430                 LogUpdatePeriod              Duration
 431                 LogUpdateSize                ByteSize
 432         }
 433         ShellAccess struct {
 434                 Admin bool
 435                 User  bool
 436         }
 437         SLURM struct {
 438                 PrioritySpread             int64
 439                 SbatchArgumentsList        []string
 440                 SbatchEnvironmentVariables map[string]string
 441                 Managed                    struct {
 442                         DNSServerConfDir       string
 443                         DNSServerConfTemplate  string
 444                         DNSServerReloadCommand string
 445                         DNSServerUpdateCommand string
 446                         ComputeNodeDomain      string
 447                         ComputeNodeNameservers StringSet
 448                         AssignNodeHostname     string
 449                 }
 450         }
 451 }
 452
 453 type CloudVMsConfig struct {
 454         Enable bool
 455
 456         BootProbeCommand               string
 457         DeployRunnerBinary             string
 458         ImageID                        string
 459         MaxCloudOpsPerSecond           int
 460         MaxProbesPerSecond             int
 461         MaxConcurrentInstanceCreateOps int
 462         PollInterval                   Duration
 463         ProbeInterval                  Duration
 464         SSHPort                        string
 465         SyncInterval                   Duration
 466         TimeoutBooting                 Duration
 467         TimeoutIdle                    Duration
 468         TimeoutProbe                   Duration
 469         TimeoutShutdown                Duration
 470         TimeoutSignal                  Duration
 471         TimeoutStaleRunLock            Duration
 472         TimeoutTERM                    Duration
 473         ResourceTags                   map[string]string
 474         TagKeyPrefix                   string
 475
 476         Driver           string
 477         DriverParameters json.RawMessage
 478 }
 479
 480 type InstanceTypeMap map[string]InstanceType
 481
 482 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 483
 484 // UnmarshalJSON handles old config files that provide an array of
 485 // instance types instead of a hash.
 486 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 487         fixup := func(t InstanceType) (InstanceType, error) {
 488                 if t.ProviderType == "" {
 489                         t.ProviderType = t.Name
 490                 }
 491                 if t.Scratch == 0 {
 492                         t.Scratch = t.IncludedScratch + t.AddedScratch
 493                 } else if t.AddedScratch == 0 {
 494                         t.AddedScratch = t.Scratch - t.IncludedScratch
 495                 } else if t.IncludedScratch == 0 {
 496                         t.IncludedScratch = t.Scratch - t.AddedScratch
 497                 }
 498
 499                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 500                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 501                 }
 502                 return t, nil
 503         }
 504
 505         if len(data) > 0 && data[0] == '[' {
 506                 var arr []InstanceType
 507                 err := json.Unmarshal(data, &arr)
 508                 if err != nil {
 509                         return err
 510                 }
 511                 if len(arr) == 0 {
 512                         *it = nil
 513                         return nil
 514                 }
 515                 *it = make(map[string]InstanceType, len(arr))
 516                 for _, t := range arr {
 517                         if _, ok := (*it)[t.Name]; ok {
 518                                 return errDuplicateInstanceTypeName
 519                         }
 520                         t, err := fixup(t)
 521                         if err != nil {
 522                                 return err
 523                         }
 524                         (*it)[t.Name] = t
 525                 }
 526                 return nil
 527         }
 528         var hash map[string]InstanceType
 529         err := json.Unmarshal(data, &hash)
 530         if err != nil {
 531                 return err
 532         }
 533         // Fill in Name field (and ProviderType field, if not
 534         // specified) using hash key.
 535         *it = InstanceTypeMap(hash)
 536         for name, t := range *it {
 537                 t.Name = name
 538                 t, err := fixup(t)
 539                 if err != nil {
 540                         return err
 541                 }
 542                 (*it)[name] = t
 543         }
 544         return nil
 545 }
 546
 547 type StringSet map[string]struct{}
 548
 549 // UnmarshalJSON handles old config files that provide an array of
 550 // instance types instead of a hash.
 551 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 552         if len(data) > 0 && data[0] == '[' {
 553                 var arr []string
 554                 err := json.Unmarshal(data, &arr)
 555                 if err != nil {
 556                         return err
 557                 }
 558                 if len(arr) == 0 {
 559                         *ss = nil
 560                         return nil
 561                 }
 562                 *ss = make(map[string]struct{}, len(arr))
 563                 for _, t := range arr {
 564                         (*ss)[t] = struct{}{}
 565                 }
 566                 return nil
 567         }
 568         var hash map[string]struct{}
 569         err := json.Unmarshal(data, &hash)
 570         if err != nil {
 571                 return err
 572         }
 573         *ss = make(map[string]struct{}, len(hash))
 574         for t := range hash {
 575                 (*ss)[t] = struct{}{}
 576         }
 577
 578         return nil
 579 }
 580
 581 type ServiceName string
 582
 583 const (
 584         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 585         ServiceNameController    ServiceName = "arvados-controller"
 586         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 587         ServiceNameHealth        ServiceName = "arvados-health"
 588         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 589         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 590         ServiceNameWebsocket     ServiceName = "arvados-ws"
 591         ServiceNameKeepbalance   ServiceName = "keep-balance"
 592         ServiceNameKeepweb       ServiceName = "keep-web"
 593         ServiceNameKeepproxy     ServiceName = "keepproxy"
 594         ServiceNameKeepstore     ServiceName = "keepstore"
 595 )
 596
 597 // Map returns all services as a map, suitable for iterating over all
 598 // services or looking up a service by name.
 599 func (svcs Services) Map() map[ServiceName]Service {
 600         return map[ServiceName]Service{
 601                 ServiceNameRailsAPI:      svcs.RailsAPI,
 602                 ServiceNameController:    svcs.Controller,
 603                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 604                 ServiceNameHealth:        svcs.Health,
 605                 ServiceNameWorkbench1:    svcs.Workbench1,
 606                 ServiceNameWorkbench2:    svcs.Workbench2,
 607                 ServiceNameWebsocket:     svcs.Websocket,
 608                 ServiceNameKeepbalance:   svcs.Keepbalance,
 609                 ServiceNameKeepweb:       svcs.WebDAV,
 610                 ServiceNameKeepproxy:     svcs.Keepproxy,
 611                 ServiceNameKeepstore:     svcs.Keepstore,
 612         }
 613 }