sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.arvados.org/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         }
  21         return "/etc/arvados/config.yml"
  22 }()
  23
  24 type Config struct {
  25         Clusters         map[string]Cluster
  26         AutoReloadConfig bool
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         if cc, ok := sc.Clusters[clusterID]; !ok {
  53                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  54         } else {
  55                 cc.ClusterID = clusterID
  56                 return &cc, nil
  57         }
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxBlockEntries      int
  64         MaxCollectionEntries int
  65         MaxCollectionBytes   int64
  66         MaxPermissionEntries int
  67         MaxUUIDEntries       int
  68 }
  69
  70 type Cluster struct {
  71         ClusterID       string `json:"-"`
  72         ManagementToken string
  73         SystemRootToken string
  74         Services        Services
  75         InstanceTypes   InstanceTypeMap
  76         Containers      ContainersConfig
  77         RemoteClusters  map[string]RemoteCluster
  78         PostgreSQL      PostgreSQL
  79
  80         API struct {
  81                 AsyncPermissionsUpdateInterval Duration
  82                 DisabledAPIs                   StringSet
  83                 MaxIndexDatabaseRead           int
  84                 MaxItemsPerResponse            int
  85                 MaxConcurrentRequests          int
  86                 MaxKeepBlobBuffers             int
  87                 MaxRequestAmplification        int
  88                 MaxRequestSize                 int
  89                 RailsSessionSecretToken        string
  90                 RequestTimeout                 Duration
  91                 SendTimeout                    Duration
  92                 WebsocketClientEventQueue      int
  93                 WebsocketServerEventQueue      int
  94                 KeepServiceRequestTimeout      Duration
  95         }
  96         AuditLogs struct {
  97                 MaxAge             Duration
  98                 MaxDeleteBatch     int
  99                 UnloggedAttributes StringSet
 100         }
 101         Collections struct {
 102                 BlobSigning              bool
 103                 BlobSigningKey           string
 104                 BlobSigningTTL           Duration
 105                 BlobTrash                bool
 106                 BlobTrashLifetime        Duration
 107                 BlobTrashCheckInterval   Duration
 108                 BlobTrashConcurrency     int
 109                 BlobDeleteConcurrency    int
 110                 BlobReplicateConcurrency int
 111                 CollectionVersioning     bool
 112                 DefaultTrashLifetime     Duration
 113                 DefaultReplication       int
 114                 ManagedProperties        map[string]struct {
 115                         Value     interface{}
 116                         Function  string
 117                         Protected bool
 118                 }
 119                 PreserveVersionIfIdle        Duration
 120                 TrashSweepInterval           Duration
 121                 TrustAllContent              bool
 122                 ForwardSlashNameSubstitution string
 123                 S3FolderObjects              bool
 124
 125                 BlobMissingReport        string
 126                 BalancePeriod            Duration
 127                 BalanceCollectionBatch   int
 128                 BalanceCollectionBuffers int
 129                 BalanceTimeout           Duration
 130
 131                 WebDAVCache WebDAVCacheConfig
 132         }
 133         Git struct {
 134                 GitCommand   string
 135                 GitoliteHome string
 136                 Repositories string
 137         }
 138         Login struct {
 139                 LDAP struct {
 140                         Enable             bool
 141                         URL                URL
 142                         StartTLS           bool
 143                         InsecureTLS        bool
 144                         StripDomain        string
 145                         AppendDomain       string
 146                         SearchAttribute    string
 147                         SearchBindUser     string
 148                         SearchBindPassword string
 149                         SearchBase         string
 150                         SearchFilters      string
 151                         EmailAttribute     string
 152                         UsernameAttribute  string
 153                 }
 154                 Google struct {
 155                         Enable                  bool
 156                         ClientID                string
 157                         ClientSecret            string
 158                         AlternateEmailAddresses bool
 159                 }
 160                 OpenIDConnect struct {
 161                         Enable             bool
 162                         Issuer             string
 163                         ClientID           string
 164                         ClientSecret       string
 165                         EmailClaim         string
 166                         EmailVerifiedClaim string
 167                         UsernameClaim      string
 168                 }
 169                 PAM struct {
 170                         Enable             bool
 171                         Service            string
 172                         DefaultEmailDomain string
 173                 }
 174                 SSO struct {
 175                         Enable            bool
 176                         ProviderAppID     string
 177                         ProviderAppSecret string
 178                 }
 179                 Test struct {
 180                         Enable bool
 181                         Users  map[string]TestUser
 182                 }
 183                 LoginCluster       string
 184                 RemoteTokenRefresh Duration
 185                 TokenLifetime      Duration
 186         }
 187         Mail struct {
 188                 MailchimpAPIKey                string
 189                 MailchimpListID                string
 190                 SendUserSetupNotificationEmail bool
 191                 IssueReporterEmailFrom         string
 192                 IssueReporterEmailTo           string
 193                 SupportEmailAddress            string
 194                 EmailFrom                      string
 195         }
 196         SystemLogs struct {
 197                 LogLevel                string
 198                 Format                  string
 199                 MaxRequestLogParamsSize int
 200         }
 201         TLS struct {
 202                 Certificate string
 203                 Key         string
 204                 Insecure    bool
 205         }
 206         Users struct {
 207                 AnonymousUserToken                    string
 208                 AdminNotifierEmailFrom                string
 209                 AutoAdminFirstUser                    bool
 210                 AutoAdminUserWithEmail                string
 211                 AutoSetupNewUsers                     bool
 212                 AutoSetupNewUsersWithRepository       bool
 213                 AutoSetupNewUsersWithVmUUID           string
 214                 AutoSetupUsernameBlacklist            StringSet
 215                 EmailSubjectPrefix                    string
 216                 NewInactiveUserNotificationRecipients StringSet
 217                 NewUserNotificationRecipients         StringSet
 218                 NewUsersAreActive                     bool
 219                 UserNotifierEmailFrom                 string
 220                 UserProfileNotificationAddress        string
 221                 PreferDomainForUsername               string
 222         }
 223         Volumes   map[string]Volume
 224         Workbench struct {
 225                 ActivationContactLink            string
 226                 APIClientConnectTimeout          Duration
 227                 APIClientReceiveTimeout          Duration
 228                 APIResponseCompression           bool
 229                 ApplicationMimetypesWithViewIcon StringSet
 230                 ArvadosDocsite                   string
 231                 ArvadosPublicDataDocURL          string
 232                 DefaultOpenIdPrefix              string
 233                 EnableGettingStartedPopup        bool
 234                 EnablePublicProjectsPage         bool
 235                 FileViewersConfigURL             string
 236                 LogViewerMaxBytes                ByteSize
 237                 MultiSiteSearch                  string
 238                 ProfilingEnabled                 bool
 239                 Repositories                     bool
 240                 RepositoryCache                  string
 241                 RunningJobLogRecordsToFetch      int
 242                 SecretKeyBase                    string
 243                 ShowRecentCollectionsOnDashboard bool
 244                 ShowUserAgreementInline          bool
 245                 ShowUserNotifications            bool
 246                 SiteName                         string
 247                 Theme                            string
 248                 UserProfileFormFields            map[string]struct {
 249                         Type                 string
 250                         FormFieldTitle       string
 251                         FormFieldDescription string
 252                         Required             bool
 253                         Position             int
 254                         Options              map[string]struct{}
 255                 }
 256                 UserProfileFormMessage string
 257                 VocabularyURL          string
 258                 WelcomePageHTML        string
 259                 InactivePageHTML       string
 260                 SSHHelpPageHTML        string
 261                 SSHHelpHostSuffix      string
 262         }
 263
 264         ForceLegacyAPI14 bool
 265 }
 266
 267 type Volume struct {
 268         AccessViaHosts   map[URL]VolumeAccess
 269         ReadOnly         bool
 270         Replication      int
 271         StorageClasses   map[string]bool
 272         Driver           string
 273         DriverParameters json.RawMessage
 274 }
 275
 276 type S3VolumeDriverParameters struct {
 277         IAMRole            string
 278         AccessKey          string
 279         SecretKey          string
 280         Endpoint           string
 281         Region             string
 282         Bucket             string
 283         LocationConstraint bool
 284         V2Signature        bool
 285         UseAWSS3v2Driver   bool
 286         IndexPageSize      int
 287         ConnectTimeout     Duration
 288         ReadTimeout        Duration
 289         RaceWindow         Duration
 290         UnsafeDelete       bool
 291 }
 292
 293 type AzureVolumeDriverParameters struct {
 294         StorageAccountName   string
 295         StorageAccountKey    string
 296         StorageBaseURL       string
 297         ContainerName        string
 298         RequestTimeout       Duration
 299         ListBlobsRetryDelay  Duration
 300         ListBlobsMaxAttempts int
 301 }
 302
 303 type DirectoryVolumeDriverParameters struct {
 304         Root      string
 305         Serialize bool
 306 }
 307
 308 type VolumeAccess struct {
 309         ReadOnly bool
 310 }
 311
 312 type Services struct {
 313         Composer       Service
 314         Controller     Service
 315         DispatchCloud  Service
 316         GitHTTP        Service
 317         GitSSH         Service
 318         Health         Service
 319         Keepbalance    Service
 320         Keepproxy      Service
 321         Keepstore      Service
 322         RailsAPI       Service
 323         SSO            Service
 324         WebDAVDownload Service
 325         WebDAV         Service
 326         WebShell       Service
 327         Websocket      Service
 328         Workbench1     Service
 329         Workbench2     Service
 330 }
 331
 332 type Service struct {
 333         InternalURLs map[URL]ServiceInstance
 334         ExternalURL  URL
 335 }
 336
 337 type TestUser struct {
 338         Email    string
 339         Password string
 340 }
 341
 342 // URL is a url.URL that is also usable as a JSON key/value.
 343 type URL url.URL
 344
 345 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 346 // used as a JSON key/value.
 347 func (su *URL) UnmarshalText(text []byte) error {
 348         u, err := url.Parse(string(text))
 349         if err == nil {
 350                 *su = URL(*u)
 351                 if su.Path == "" && su.Host != "" {
 352                         // http://example really means http://example/
 353                         su.Path = "/"
 354                 }
 355         }
 356         return err
 357 }
 358
 359 func (su URL) MarshalText() ([]byte, error) {
 360         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 361 }
 362
 363 func (su URL) String() string {
 364         return (*url.URL)(&su).String()
 365 }
 366
 367 type ServiceInstance struct {
 368         Rendezvous string `json:",omitempty"`
 369 }
 370
 371 type PostgreSQL struct {
 372         Connection     PostgreSQLConnection
 373         ConnectionPool int
 374 }
 375
 376 type PostgreSQLConnection map[string]string
 377
 378 type RemoteCluster struct {
 379         Host          string
 380         Proxy         bool
 381         Scheme        string
 382         Insecure      bool
 383         ActivateUsers bool
 384 }
 385
 386 type InstanceType struct {
 387         Name            string
 388         ProviderType    string
 389         VCPUs           int
 390         RAM             ByteSize
 391         Scratch         ByteSize
 392         IncludedScratch ByteSize
 393         AddedScratch    ByteSize
 394         Price           float64
 395         Preemptible     bool
 396 }
 397
 398 type ContainersConfig struct {
 399         CloudVMs                    CloudVMsConfig
 400         CrunchRunCommand            string
 401         CrunchRunArgumentsList      []string
 402         DefaultKeepCacheRAM         ByteSize
 403         DispatchPrivateKey          string
 404         LogReuseDecisions           bool
 405         MaxComputeVMs               int
 406         MaxDispatchAttempts         int
 407         MaxRetryAttempts            int
 408         MinRetryPeriod              Duration
 409         ReserveExtraRAM             ByteSize
 410         StaleLockTimeout            Duration
 411         SupportedDockerImageFormats StringSet
 412         UsePreemptibleInstances     bool
 413
 414         JobsAPI struct {
 415                 Enable         string
 416                 GitInternalDir string
 417         }
 418         Logging struct {
 419                 MaxAge                       Duration
 420                 LogBytesPerEvent             int
 421                 LogSecondsBetweenEvents      Duration
 422                 LogThrottlePeriod            Duration
 423                 LogThrottleBytes             int
 424                 LogThrottleLines             int
 425                 LimitLogBytesPerJob          int
 426                 LogPartialLineThrottlePeriod Duration
 427                 LogUpdatePeriod              Duration
 428                 LogUpdateSize                ByteSize
 429         }
 430         SLURM struct {
 431                 PrioritySpread             int64
 432                 SbatchArgumentsList        []string
 433                 SbatchEnvironmentVariables map[string]string
 434                 Managed                    struct {
 435                         DNSServerConfDir       string
 436                         DNSServerConfTemplate  string
 437                         DNSServerReloadCommand string
 438                         DNSServerUpdateCommand string
 439                         ComputeNodeDomain      string
 440                         ComputeNodeNameservers StringSet
 441                         AssignNodeHostname     string
 442                 }
 443         }
 444 }
 445
 446 type CloudVMsConfig struct {
 447         Enable bool
 448
 449         BootProbeCommand               string
 450         DeployRunnerBinary             string
 451         ImageID                        string
 452         MaxCloudOpsPerSecond           int
 453         MaxProbesPerSecond             int
 454         MaxConcurrentInstanceCreateOps int
 455         PollInterval                   Duration
 456         ProbeInterval                  Duration
 457         SSHPort                        string
 458         SyncInterval                   Duration
 459         TimeoutBooting                 Duration
 460         TimeoutIdle                    Duration
 461         TimeoutProbe                   Duration
 462         TimeoutShutdown                Duration
 463         TimeoutSignal                  Duration
 464         TimeoutTERM                    Duration
 465         ResourceTags                   map[string]string
 466         TagKeyPrefix                   string
 467
 468         Driver           string
 469         DriverParameters json.RawMessage
 470 }
 471
 472 type InstanceTypeMap map[string]InstanceType
 473
 474 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 475
 476 // UnmarshalJSON handles old config files that provide an array of
 477 // instance types instead of a hash.
 478 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 479         fixup := func(t InstanceType) (InstanceType, error) {
 480                 if t.ProviderType == "" {
 481                         t.ProviderType = t.Name
 482                 }
 483                 if t.Scratch == 0 {
 484                         t.Scratch = t.IncludedScratch + t.AddedScratch
 485                 } else if t.AddedScratch == 0 {
 486                         t.AddedScratch = t.Scratch - t.IncludedScratch
 487                 } else if t.IncludedScratch == 0 {
 488                         t.IncludedScratch = t.Scratch - t.AddedScratch
 489                 }
 490
 491                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 492                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 493                 }
 494                 return t, nil
 495         }
 496
 497         if len(data) > 0 && data[0] == '[' {
 498                 var arr []InstanceType
 499                 err := json.Unmarshal(data, &arr)
 500                 if err != nil {
 501                         return err
 502                 }
 503                 if len(arr) == 0 {
 504                         *it = nil
 505                         return nil
 506                 }
 507                 *it = make(map[string]InstanceType, len(arr))
 508                 for _, t := range arr {
 509                         if _, ok := (*it)[t.Name]; ok {
 510                                 return errDuplicateInstanceTypeName
 511                         }
 512                         t, err := fixup(t)
 513                         if err != nil {
 514                                 return err
 515                         }
 516                         (*it)[t.Name] = t
 517                 }
 518                 return nil
 519         }
 520         var hash map[string]InstanceType
 521         err := json.Unmarshal(data, &hash)
 522         if err != nil {
 523                 return err
 524         }
 525         // Fill in Name field (and ProviderType field, if not
 526         // specified) using hash key.
 527         *it = InstanceTypeMap(hash)
 528         for name, t := range *it {
 529                 t.Name = name
 530                 t, err := fixup(t)
 531                 if err != nil {
 532                         return err
 533                 }
 534                 (*it)[name] = t
 535         }
 536         return nil
 537 }
 538
 539 type StringSet map[string]struct{}
 540
 541 // UnmarshalJSON handles old config files that provide an array of
 542 // instance types instead of a hash.
 543 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 544         if len(data) > 0 && data[0] == '[' {
 545                 var arr []string
 546                 err := json.Unmarshal(data, &arr)
 547                 if err != nil {
 548                         return err
 549                 }
 550                 if len(arr) == 0 {
 551                         *ss = nil
 552                         return nil
 553                 }
 554                 *ss = make(map[string]struct{}, len(arr))
 555                 for _, t := range arr {
 556                         (*ss)[t] = struct{}{}
 557                 }
 558                 return nil
 559         }
 560         var hash map[string]struct{}
 561         err := json.Unmarshal(data, &hash)
 562         if err != nil {
 563                 return err
 564         }
 565         *ss = make(map[string]struct{}, len(hash))
 566         for t := range hash {
 567                 (*ss)[t] = struct{}{}
 568         }
 569
 570         return nil
 571 }
 572
 573 type ServiceName string
 574
 575 const (
 576         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 577         ServiceNameController    ServiceName = "arvados-controller"
 578         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 579         ServiceNameHealth        ServiceName = "arvados-health"
 580         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 581         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 582         ServiceNameWebsocket     ServiceName = "arvados-ws"
 583         ServiceNameKeepbalance   ServiceName = "keep-balance"
 584         ServiceNameKeepweb       ServiceName = "keep-web"
 585         ServiceNameKeepproxy     ServiceName = "keepproxy"
 586         ServiceNameKeepstore     ServiceName = "keepstore"
 587 )
 588
 589 // Map returns all services as a map, suitable for iterating over all
 590 // services or looking up a service by name.
 591 func (svcs Services) Map() map[ServiceName]Service {
 592         return map[ServiceName]Service{
 593                 ServiceNameRailsAPI:      svcs.RailsAPI,
 594                 ServiceNameController:    svcs.Controller,
 595                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 596                 ServiceNameHealth:        svcs.Health,
 597                 ServiceNameWorkbench1:    svcs.Workbench1,
 598                 ServiceNameWorkbench2:    svcs.Workbench2,
 599                 ServiceNameWebsocket:     svcs.Websocket,
 600                 ServiceNameKeepbalance:   svcs.Keepbalance,
 601                 ServiceNameKeepweb:       svcs.WebDAV,
 602                 ServiceNameKeepproxy:     svcs.Keepproxy,
 603                 ServiceNameKeepstore:     svcs.Keepstore,
 604         }
 605 }