sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.arvados.org/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         }
  21         return "/etc/arvados/config.yml"
  22 }()
  23
  24 type Config struct {
  25         Clusters         map[string]Cluster
  26         AutoReloadConfig bool
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         cc, ok := sc.Clusters[clusterID]
  53         if !ok {
  54                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  55         }
  56         cc.ClusterID = clusterID
  57         return &cc, nil
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxBlockEntries      int
  64         MaxCollectionEntries int
  65         MaxCollectionBytes   int64
  66         MaxPermissionEntries int
  67         MaxUUIDEntries       int
  68 }
  69
  70 type Cluster struct {
  71         ClusterID       string `json:"-"`
  72         ManagementToken string
  73         SystemRootToken string
  74         Services        Services
  75         InstanceTypes   InstanceTypeMap
  76         Containers      ContainersConfig
  77         RemoteClusters  map[string]RemoteCluster
  78         PostgreSQL      PostgreSQL
  79
  80         API struct {
  81                 AsyncPermissionsUpdateInterval Duration
  82                 DisabledAPIs                   StringSet
  83                 MaxIndexDatabaseRead           int
  84                 MaxItemsPerResponse            int
  85                 MaxConcurrentRequests          int
  86                 MaxKeepBlobBuffers             int
  87                 MaxRequestAmplification        int
  88                 MaxRequestSize                 int
  89                 RailsSessionSecretToken        string
  90                 RequestTimeout                 Duration
  91                 SendTimeout                    Duration
  92                 WebsocketClientEventQueue      int
  93                 WebsocketServerEventQueue      int
  94                 KeepServiceRequestTimeout      Duration
  95         }
  96         AuditLogs struct {
  97                 MaxAge             Duration
  98                 MaxDeleteBatch     int
  99                 UnloggedAttributes StringSet
 100         }
 101         Collections struct {
 102                 BlobSigning              bool
 103                 BlobSigningKey           string
 104                 BlobSigningTTL           Duration
 105                 BlobTrash                bool
 106                 BlobTrashLifetime        Duration
 107                 BlobTrashCheckInterval   Duration
 108                 BlobTrashConcurrency     int
 109                 BlobDeleteConcurrency    int
 110                 BlobReplicateConcurrency int
 111                 CollectionVersioning     bool
 112                 DefaultTrashLifetime     Duration
 113                 DefaultReplication       int
 114                 ManagedProperties        map[string]struct {
 115                         Value     interface{}
 116                         Function  string
 117                         Protected bool
 118                 }
 119                 PreserveVersionIfIdle        Duration
 120                 TrashSweepInterval           Duration
 121                 TrustAllContent              bool
 122                 ForwardSlashNameSubstitution string
 123                 S3FolderObjects              bool
 124
 125                 BlobMissingReport        string
 126                 BalancePeriod            Duration
 127                 BalanceCollectionBatch   int
 128                 BalanceCollectionBuffers int
 129                 BalanceTimeout           Duration
 130
 131                 WebDAVCache WebDAVCacheConfig
 132         }
 133         Git struct {
 134                 GitCommand   string
 135                 GitoliteHome string
 136                 Repositories string
 137         }
 138         Login struct {
 139                 LDAP struct {
 140                         Enable             bool
 141                         URL                URL
 142                         StartTLS           bool
 143                         InsecureTLS        bool
 144                         StripDomain        string
 145                         AppendDomain       string
 146                         SearchAttribute    string
 147                         SearchBindUser     string
 148                         SearchBindPassword string
 149                         SearchBase         string
 150                         SearchFilters      string
 151                         EmailAttribute     string
 152                         UsernameAttribute  string
 153                 }
 154                 Google struct {
 155                         Enable                  bool
 156                         ClientID                string
 157                         ClientSecret            string
 158                         AlternateEmailAddresses bool
 159                 }
 160                 OpenIDConnect struct {
 161                         Enable             bool
 162                         Issuer             string
 163                         ClientID           string
 164                         ClientSecret       string
 165                         EmailClaim         string
 166                         EmailVerifiedClaim string
 167                         UsernameClaim      string
 168                 }
 169                 PAM struct {
 170                         Enable             bool
 171                         Service            string
 172                         DefaultEmailDomain string
 173                 }
 174                 SSO struct {
 175                         Enable            bool
 176                         ProviderAppID     string
 177                         ProviderAppSecret string
 178                 }
 179                 Test struct {
 180                         Enable bool
 181                         Users  map[string]TestUser
 182                 }
 183                 LoginCluster       string
 184                 RemoteTokenRefresh Duration
 185                 TokenLifetime      Duration
 186                 TrustedClients     map[string]struct{}
 187         }
 188         Mail struct {
 189                 MailchimpAPIKey                string
 190                 MailchimpListID                string
 191                 SendUserSetupNotificationEmail bool
 192                 IssueReporterEmailFrom         string
 193                 IssueReporterEmailTo           string
 194                 SupportEmailAddress            string
 195                 EmailFrom                      string
 196         }
 197         SystemLogs struct {
 198                 LogLevel                string
 199                 Format                  string
 200                 MaxRequestLogParamsSize int
 201         }
 202         TLS struct {
 203                 Certificate string
 204                 Key         string
 205                 Insecure    bool
 206         }
 207         Users struct {
 208                 AnonymousUserToken                    string
 209                 AdminNotifierEmailFrom                string
 210                 AutoAdminFirstUser                    bool
 211                 AutoAdminUserWithEmail                string
 212                 AutoSetupNewUsers                     bool
 213                 AutoSetupNewUsersWithRepository       bool
 214                 AutoSetupNewUsersWithVmUUID           string
 215                 AutoSetupUsernameBlacklist            StringSet
 216                 EmailSubjectPrefix                    string
 217                 NewInactiveUserNotificationRecipients StringSet
 218                 NewUserNotificationRecipients         StringSet
 219                 NewUsersAreActive                     bool
 220                 UserNotifierEmailFrom                 string
 221                 UserProfileNotificationAddress        string
 222                 PreferDomainForUsername               string
 223                 UserSetupMailText                     string
 224         }
 225         Volumes   map[string]Volume
 226         Workbench struct {
 227                 ActivationContactLink            string
 228                 APIClientConnectTimeout          Duration
 229                 APIClientReceiveTimeout          Duration
 230                 APIResponseCompression           bool
 231                 ApplicationMimetypesWithViewIcon StringSet
 232                 ArvadosDocsite                   string
 233                 ArvadosPublicDataDocURL          string
 234                 DefaultOpenIdPrefix              string
 235                 EnableGettingStartedPopup        bool
 236                 EnablePublicProjectsPage         bool
 237                 FileViewersConfigURL             string
 238                 LogViewerMaxBytes                ByteSize
 239                 MultiSiteSearch                  string
 240                 ProfilingEnabled                 bool
 241                 Repositories                     bool
 242                 RepositoryCache                  string
 243                 RunningJobLogRecordsToFetch      int
 244                 SecretKeyBase                    string
 245                 ShowRecentCollectionsOnDashboard bool
 246                 ShowUserAgreementInline          bool
 247                 ShowUserNotifications            bool
 248                 SiteName                         string
 249                 Theme                            string
 250                 UserProfileFormFields            map[string]struct {
 251                         Type                 string
 252                         FormFieldTitle       string
 253                         FormFieldDescription string
 254                         Required             bool
 255                         Position             int
 256                         Options              map[string]struct{}
 257                 }
 258                 UserProfileFormMessage string
 259                 VocabularyURL          string
 260                 WelcomePageHTML        string
 261                 InactivePageHTML       string
 262                 SSHHelpPageHTML        string
 263                 SSHHelpHostSuffix      string
 264                 IdleTimeout            Duration
 265         }
 266
 267         ForceLegacyAPI14 bool
 268 }
 269
 270 type Volume struct {
 271         AccessViaHosts   map[URL]VolumeAccess
 272         ReadOnly         bool
 273         Replication      int
 274         StorageClasses   map[string]bool
 275         Driver           string
 276         DriverParameters json.RawMessage
 277 }
 278
 279 type S3VolumeDriverParameters struct {
 280         IAMRole            string
 281         AccessKey          string
 282         SecretKey          string
 283         Endpoint           string
 284         Region             string
 285         Bucket             string
 286         LocationConstraint bool
 287         V2Signature        bool
 288         UseAWSS3v2Driver   bool
 289         IndexPageSize      int
 290         ConnectTimeout     Duration
 291         ReadTimeout        Duration
 292         RaceWindow         Duration
 293         UnsafeDelete       bool
 294 }
 295
 296 type AzureVolumeDriverParameters struct {
 297         StorageAccountName   string
 298         StorageAccountKey    string
 299         StorageBaseURL       string
 300         ContainerName        string
 301         RequestTimeout       Duration
 302         ListBlobsRetryDelay  Duration
 303         ListBlobsMaxAttempts int
 304 }
 305
 306 type DirectoryVolumeDriverParameters struct {
 307         Root      string
 308         Serialize bool
 309 }
 310
 311 type VolumeAccess struct {
 312         ReadOnly bool
 313 }
 314
 315 type Services struct {
 316         Composer       Service
 317         Controller     Service
 318         DispatchCloud  Service
 319         GitHTTP        Service
 320         GitSSH         Service
 321         Health         Service
 322         Keepbalance    Service
 323         Keepproxy      Service
 324         Keepstore      Service
 325         RailsAPI       Service
 326         SSO            Service
 327         WebDAVDownload Service
 328         WebDAV         Service
 329         WebShell       Service
 330         Websocket      Service
 331         Workbench1     Service
 332         Workbench2     Service
 333 }
 334
 335 type Service struct {
 336         InternalURLs map[URL]ServiceInstance
 337         ExternalURL  URL
 338 }
 339
 340 type TestUser struct {
 341         Email    string
 342         Password string
 343 }
 344
 345 // URL is a url.URL that is also usable as a JSON key/value.
 346 type URL url.URL
 347
 348 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 349 // used as a JSON key/value.
 350 func (su *URL) UnmarshalText(text []byte) error {
 351         u, err := url.Parse(string(text))
 352         if err == nil {
 353                 *su = URL(*u)
 354                 if su.Path == "" && su.Host != "" {
 355                         // http://example really means http://example/
 356                         su.Path = "/"
 357                 }
 358         }
 359         return err
 360 }
 361
 362 func (su URL) MarshalText() ([]byte, error) {
 363         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 364 }
 365
 366 func (su URL) String() string {
 367         return (*url.URL)(&su).String()
 368 }
 369
 370 type ServiceInstance struct {
 371         Rendezvous string `json:",omitempty"`
 372 }
 373
 374 type PostgreSQL struct {
 375         Connection     PostgreSQLConnection
 376         ConnectionPool int
 377 }
 378
 379 type PostgreSQLConnection map[string]string
 380
 381 type RemoteCluster struct {
 382         Host          string
 383         Proxy         bool
 384         Scheme        string
 385         Insecure      bool
 386         ActivateUsers bool
 387 }
 388
 389 type InstanceType struct {
 390         Name            string
 391         ProviderType    string
 392         VCPUs           int
 393         RAM             ByteSize
 394         Scratch         ByteSize
 395         IncludedScratch ByteSize
 396         AddedScratch    ByteSize
 397         Price           float64
 398         Preemptible     bool
 399 }
 400
 401 type ContainersConfig struct {
 402         CloudVMs                    CloudVMsConfig
 403         CrunchRunCommand            string
 404         CrunchRunArgumentsList      []string
 405         DefaultKeepCacheRAM         ByteSize
 406         DispatchPrivateKey          string
 407         LogReuseDecisions           bool
 408         MaxComputeVMs               int
 409         MaxDispatchAttempts         int
 410         MaxRetryAttempts            int
 411         MinRetryPeriod              Duration
 412         ReserveExtraRAM             ByteSize
 413         StaleLockTimeout            Duration
 414         SupportedDockerImageFormats StringSet
 415         UsePreemptibleInstances     bool
 416
 417         JobsAPI struct {
 418                 Enable         string
 419                 GitInternalDir string
 420         }
 421         Logging struct {
 422                 MaxAge                       Duration
 423                 LogBytesPerEvent             int
 424                 LogSecondsBetweenEvents      Duration
 425                 LogThrottlePeriod            Duration
 426                 LogThrottleBytes             int
 427                 LogThrottleLines             int
 428                 LimitLogBytesPerJob          int
 429                 LogPartialLineThrottlePeriod Duration
 430                 LogUpdatePeriod              Duration
 431                 LogUpdateSize                ByteSize
 432         }
 433         SLURM struct {
 434                 PrioritySpread             int64
 435                 SbatchArgumentsList        []string
 436                 SbatchEnvironmentVariables map[string]string
 437                 Managed                    struct {
 438                         DNSServerConfDir       string
 439                         DNSServerConfTemplate  string
 440                         DNSServerReloadCommand string
 441                         DNSServerUpdateCommand string
 442                         ComputeNodeDomain      string
 443                         ComputeNodeNameservers StringSet
 444                         AssignNodeHostname     string
 445                 }
 446         }
 447 }
 448
 449 type CloudVMsConfig struct {
 450         Enable bool
 451
 452         BootProbeCommand               string
 453         DeployRunnerBinary             string
 454         ImageID                        string
 455         MaxCloudOpsPerSecond           int
 456         MaxProbesPerSecond             int
 457         MaxConcurrentInstanceCreateOps int
 458         PollInterval                   Duration
 459         ProbeInterval                  Duration
 460         SSHPort                        string
 461         SyncInterval                   Duration
 462         TimeoutBooting                 Duration
 463         TimeoutIdle                    Duration
 464         TimeoutProbe                   Duration
 465         TimeoutShutdown                Duration
 466         TimeoutSignal                  Duration
 467         TimeoutStaleRunLock            Duration
 468         TimeoutTERM                    Duration
 469         ResourceTags                   map[string]string
 470         TagKeyPrefix                   string
 471
 472         Driver           string
 473         DriverParameters json.RawMessage
 474 }
 475
 476 type InstanceTypeMap map[string]InstanceType
 477
 478 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 479
 480 // UnmarshalJSON handles old config files that provide an array of
 481 // instance types instead of a hash.
 482 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 483         fixup := func(t InstanceType) (InstanceType, error) {
 484                 if t.ProviderType == "" {
 485                         t.ProviderType = t.Name
 486                 }
 487                 if t.Scratch == 0 {
 488                         t.Scratch = t.IncludedScratch + t.AddedScratch
 489                 } else if t.AddedScratch == 0 {
 490                         t.AddedScratch = t.Scratch - t.IncludedScratch
 491                 } else if t.IncludedScratch == 0 {
 492                         t.IncludedScratch = t.Scratch - t.AddedScratch
 493                 }
 494
 495                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 496                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 497                 }
 498                 return t, nil
 499         }
 500
 501         if len(data) > 0 && data[0] == '[' {
 502                 var arr []InstanceType
 503                 err := json.Unmarshal(data, &arr)
 504                 if err != nil {
 505                         return err
 506                 }
 507                 if len(arr) == 0 {
 508                         *it = nil
 509                         return nil
 510                 }
 511                 *it = make(map[string]InstanceType, len(arr))
 512                 for _, t := range arr {
 513                         if _, ok := (*it)[t.Name]; ok {
 514                                 return errDuplicateInstanceTypeName
 515                         }
 516                         t, err := fixup(t)
 517                         if err != nil {
 518                                 return err
 519                         }
 520                         (*it)[t.Name] = t
 521                 }
 522                 return nil
 523         }
 524         var hash map[string]InstanceType
 525         err := json.Unmarshal(data, &hash)
 526         if err != nil {
 527                 return err
 528         }
 529         // Fill in Name field (and ProviderType field, if not
 530         // specified) using hash key.
 531         *it = InstanceTypeMap(hash)
 532         for name, t := range *it {
 533                 t.Name = name
 534                 t, err := fixup(t)
 535                 if err != nil {
 536                         return err
 537                 }
 538                 (*it)[name] = t
 539         }
 540         return nil
 541 }
 542
 543 type StringSet map[string]struct{}
 544
 545 // UnmarshalJSON handles old config files that provide an array of
 546 // instance types instead of a hash.
 547 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 548         if len(data) > 0 && data[0] == '[' {
 549                 var arr []string
 550                 err := json.Unmarshal(data, &arr)
 551                 if err != nil {
 552                         return err
 553                 }
 554                 if len(arr) == 0 {
 555                         *ss = nil
 556                         return nil
 557                 }
 558                 *ss = make(map[string]struct{}, len(arr))
 559                 for _, t := range arr {
 560                         (*ss)[t] = struct{}{}
 561                 }
 562                 return nil
 563         }
 564         var hash map[string]struct{}
 565         err := json.Unmarshal(data, &hash)
 566         if err != nil {
 567                 return err
 568         }
 569         *ss = make(map[string]struct{}, len(hash))
 570         for t := range hash {
 571                 (*ss)[t] = struct{}{}
 572         }
 573
 574         return nil
 575 }
 576
 577 type ServiceName string
 578
 579 const (
 580         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 581         ServiceNameController    ServiceName = "arvados-controller"
 582         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 583         ServiceNameHealth        ServiceName = "arvados-health"
 584         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 585         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 586         ServiceNameWebsocket     ServiceName = "arvados-ws"
 587         ServiceNameKeepbalance   ServiceName = "keep-balance"
 588         ServiceNameKeepweb       ServiceName = "keep-web"
 589         ServiceNameKeepproxy     ServiceName = "keepproxy"
 590         ServiceNameKeepstore     ServiceName = "keepstore"
 591 )
 592
 593 // Map returns all services as a map, suitable for iterating over all
 594 // services or looking up a service by name.
 595 func (svcs Services) Map() map[ServiceName]Service {
 596         return map[ServiceName]Service{
 597                 ServiceNameRailsAPI:      svcs.RailsAPI,
 598                 ServiceNameController:    svcs.Controller,
 599                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 600                 ServiceNameHealth:        svcs.Health,
 601                 ServiceNameWorkbench1:    svcs.Workbench1,
 602                 ServiceNameWorkbench2:    svcs.Workbench2,
 603                 ServiceNameWebsocket:     svcs.Websocket,
 604                 ServiceNameKeepbalance:   svcs.Keepbalance,
 605                 ServiceNameKeepweb:       svcs.WebDAV,
 606                 ServiceNameKeepproxy:     svcs.Keepproxy,
 607                 ServiceNameKeepstore:     svcs.Keepstore,
 608         }
 609 }