sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.arvados.org/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters         map[string]Cluster
  27         AutoReloadConfig bool
  28 }
  29
  30 // GetConfig returns the current system config, loading it from
  31 // configFile if needed.
  32 func GetConfig(configFile string) (*Config, error) {
  33         var cfg Config
  34         err := config.LoadFile(&cfg, configFile)
  35         return &cfg, err
  36 }
  37
  38 // GetCluster returns the cluster ID and config for the given
  39 // cluster, or the default/only configured cluster if clusterID is "".
  40 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  41         if clusterID == "" {
  42                 if len(sc.Clusters) == 0 {
  43                         return nil, fmt.Errorf("no clusters configured")
  44                 } else if len(sc.Clusters) > 1 {
  45                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  46                 } else {
  47                         for id, cc := range sc.Clusters {
  48                                 cc.ClusterID = id
  49                                 return &cc, nil
  50                         }
  51                 }
  52         }
  53         if cc, ok := sc.Clusters[clusterID]; !ok {
  54                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  55         } else {
  56                 cc.ClusterID = clusterID
  57                 return &cc, nil
  58         }
  59 }
  60
  61 type WebDAVCacheConfig struct {
  62         TTL                  Duration
  63         UUIDTTL              Duration
  64         MaxBlockEntries      int
  65         MaxCollectionEntries int
  66         MaxCollectionBytes   int64
  67         MaxPermissionEntries int
  68         MaxUUIDEntries       int
  69 }
  70
  71 type Cluster struct {
  72         ClusterID       string `json:"-"`
  73         ManagementToken string
  74         SystemRootToken string
  75         Services        Services
  76         InstanceTypes   InstanceTypeMap
  77         Containers      ContainersConfig
  78         RemoteClusters  map[string]RemoteCluster
  79         PostgreSQL      PostgreSQL
  80
  81         API struct {
  82                 AsyncPermissionsUpdateInterval Duration
  83                 DisabledAPIs                   StringSet
  84                 MaxIndexDatabaseRead           int
  85                 MaxItemsPerResponse            int
  86                 MaxConcurrentRequests          int
  87                 MaxKeepBlobBuffers             int
  88                 MaxRequestAmplification        int
  89                 MaxRequestSize                 int
  90                 RailsSessionSecretToken        string
  91                 RequestTimeout                 Duration
  92                 SendTimeout                    Duration
  93                 WebsocketClientEventQueue      int
  94                 WebsocketServerEventQueue      int
  95                 KeepServiceRequestTimeout      Duration
  96         }
  97         AuditLogs struct {
  98                 MaxAge             Duration
  99                 MaxDeleteBatch     int
 100                 UnloggedAttributes StringSet
 101         }
 102         Collections struct {
 103                 BlobSigning              bool
 104                 BlobSigningKey           string
 105                 BlobSigningTTL           Duration
 106                 BlobTrash                bool
 107                 BlobTrashLifetime        Duration
 108                 BlobTrashCheckInterval   Duration
 109                 BlobTrashConcurrency     int
 110                 BlobDeleteConcurrency    int
 111                 BlobReplicateConcurrency int
 112                 CollectionVersioning     bool
 113                 DefaultTrashLifetime     Duration
 114                 DefaultReplication       int
 115                 ManagedProperties        map[string]struct {
 116                         Value     interface{}
 117                         Function  string
 118                         Protected bool
 119                 }
 120                 PreserveVersionIfIdle        Duration
 121                 TrashSweepInterval           Duration
 122                 TrustAllContent              bool
 123                 ForwardSlashNameSubstitution string
 124
 125                 BlobMissingReport        string
 126                 BalancePeriod            Duration
 127                 BalanceCollectionBatch   int
 128                 BalanceCollectionBuffers int
 129                 BalanceTimeout           Duration
 130
 131                 WebDAVCache WebDAVCacheConfig
 132         }
 133         Git struct {
 134                 GitCommand   string
 135                 GitoliteHome string
 136                 Repositories string
 137         }
 138         Login struct {
 139                 LDAP struct {
 140                         Enable             bool
 141                         URL                URL
 142                         StartTLS           bool
 143                         InsecureTLS        bool
 144                         StripDomain        string
 145                         AppendDomain       string
 146                         SearchAttribute    string
 147                         SearchBindUser     string
 148                         SearchBindPassword string
 149                         SearchBase         string
 150                         SearchFilters      string
 151                         EmailAttribute     string
 152                         UsernameAttribute  string
 153                 }
 154                 Google struct {
 155                         Enable                  bool
 156                         ClientID                string
 157                         ClientSecret            string
 158                         AlternateEmailAddresses bool
 159                 }
 160                 OpenIDConnect struct {
 161                         Enable             bool
 162                         Issuer             string
 163                         ClientID           string
 164                         ClientSecret       string
 165                         EmailClaim         string
 166                         EmailVerifiedClaim string
 167                         UsernameClaim      string
 168                 }
 169                 PAM struct {
 170                         Enable             bool
 171                         Service            string
 172                         DefaultEmailDomain string
 173                 }
 174                 SSO struct {
 175                         Enable            bool
 176                         ProviderAppID     string
 177                         ProviderAppSecret string
 178                 }
 179                 Test struct {
 180                         Enable bool
 181                         Users  map[string]TestUser
 182                 }
 183                 LoginCluster       string
 184                 RemoteTokenRefresh Duration
 185         }
 186         Mail struct {
 187                 MailchimpAPIKey                string
 188                 MailchimpListID                string
 189                 SendUserSetupNotificationEmail bool
 190                 IssueReporterEmailFrom         string
 191                 IssueReporterEmailTo           string
 192                 SupportEmailAddress            string
 193                 EmailFrom                      string
 194         }
 195         SystemLogs struct {
 196                 LogLevel                string
 197                 Format                  string
 198                 MaxRequestLogParamsSize int
 199         }
 200         TLS struct {
 201                 Certificate string
 202                 Key         string
 203                 Insecure    bool
 204         }
 205         Users struct {
 206                 AnonymousUserToken                    string
 207                 AdminNotifierEmailFrom                string
 208                 AutoAdminFirstUser                    bool
 209                 AutoAdminUserWithEmail                string
 210                 AutoSetupNewUsers                     bool
 211                 AutoSetupNewUsersWithRepository       bool
 212                 AutoSetupNewUsersWithVmUUID           string
 213                 AutoSetupUsernameBlacklist            StringSet
 214                 EmailSubjectPrefix                    string
 215                 NewInactiveUserNotificationRecipients StringSet
 216                 NewUserNotificationRecipients         StringSet
 217                 NewUsersAreActive                     bool
 218                 UserNotifierEmailFrom                 string
 219                 UserProfileNotificationAddress        string
 220                 PreferDomainForUsername               string
 221         }
 222         Volumes   map[string]Volume
 223         Workbench struct {
 224                 ActivationContactLink            string
 225                 APIClientConnectTimeout          Duration
 226                 APIClientReceiveTimeout          Duration
 227                 APIResponseCompression           bool
 228                 ApplicationMimetypesWithViewIcon StringSet
 229                 ArvadosDocsite                   string
 230                 ArvadosPublicDataDocURL          string
 231                 DefaultOpenIdPrefix              string
 232                 EnableGettingStartedPopup        bool
 233                 EnablePublicProjectsPage         bool
 234                 FileViewersConfigURL             string
 235                 LogViewerMaxBytes                ByteSize
 236                 MultiSiteSearch                  string
 237                 ProfilingEnabled                 bool
 238                 Repositories                     bool
 239                 RepositoryCache                  string
 240                 RunningJobLogRecordsToFetch      int
 241                 SecretKeyBase                    string
 242                 ShowRecentCollectionsOnDashboard bool
 243                 ShowUserAgreementInline          bool
 244                 ShowUserNotifications            bool
 245                 SiteName                         string
 246                 Theme                            string
 247                 UserProfileFormFields            map[string]struct {
 248                         Type                 string
 249                         FormFieldTitle       string
 250                         FormFieldDescription string
 251                         Required             bool
 252                         Position             int
 253                         Options              map[string]struct{}
 254                 }
 255                 UserProfileFormMessage string
 256                 VocabularyURL          string
 257                 WelcomePageHTML        string
 258                 InactivePageHTML       string
 259                 SSHHelpPageHTML        string
 260                 SSHHelpHostSuffix      string
 261         }
 262
 263         ForceLegacyAPI14 bool
 264 }
 265
 266 type Volume struct {
 267         AccessViaHosts   map[URL]VolumeAccess
 268         ReadOnly         bool
 269         Replication      int
 270         StorageClasses   map[string]bool
 271         Driver           string
 272         DriverParameters json.RawMessage
 273 }
 274
 275 type S3VolumeDriverParameters struct {
 276         IAMRole            string
 277         AccessKey          string
 278         SecretKey          string
 279         Endpoint           string
 280         Region             string
 281         Bucket             string
 282         LocationConstraint bool
 283         V2Signature        bool
 284         UseAWSS3v2Driver   bool
 285         IndexPageSize      int
 286         ConnectTimeout     Duration
 287         ReadTimeout        Duration
 288         RaceWindow         Duration
 289         UnsafeDelete       bool
 290 }
 291
 292 type AzureVolumeDriverParameters struct {
 293         StorageAccountName   string
 294         StorageAccountKey    string
 295         StorageBaseURL       string
 296         ContainerName        string
 297         RequestTimeout       Duration
 298         ListBlobsRetryDelay  Duration
 299         ListBlobsMaxAttempts int
 300 }
 301
 302 type DirectoryVolumeDriverParameters struct {
 303         Root      string
 304         Serialize bool
 305 }
 306
 307 type VolumeAccess struct {
 308         ReadOnly bool
 309 }
 310
 311 type Services struct {
 312         Composer       Service
 313         Controller     Service
 314         DispatchCloud  Service
 315         GitHTTP        Service
 316         GitSSH         Service
 317         Health         Service
 318         Keepbalance    Service
 319         Keepproxy      Service
 320         Keepstore      Service
 321         RailsAPI       Service
 322         SSO            Service
 323         WebDAVDownload Service
 324         WebDAV         Service
 325         WebShell       Service
 326         Websocket      Service
 327         Workbench1     Service
 328         Workbench2     Service
 329 }
 330
 331 type Service struct {
 332         InternalURLs map[URL]ServiceInstance
 333         ExternalURL  URL
 334 }
 335
 336 type TestUser struct {
 337         Email    string
 338         Password string
 339 }
 340
 341 // URL is a url.URL that is also usable as a JSON key/value.
 342 type URL url.URL
 343
 344 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 345 // used as a JSON key/value.
 346 func (su *URL) UnmarshalText(text []byte) error {
 347         u, err := url.Parse(string(text))
 348         if err == nil {
 349                 *su = URL(*u)
 350                 if su.Path == "" && su.Host != "" {
 351                         // http://example really means http://example/
 352                         su.Path = "/"
 353                 }
 354         }
 355         return err
 356 }
 357
 358 func (su URL) MarshalText() ([]byte, error) {
 359         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 360 }
 361
 362 func (su URL) String() string {
 363         return (*url.URL)(&su).String()
 364 }
 365
 366 type ServiceInstance struct {
 367         Rendezvous string `json:",omitempty"`
 368 }
 369
 370 type PostgreSQL struct {
 371         Connection     PostgreSQLConnection
 372         ConnectionPool int
 373 }
 374
 375 type PostgreSQLConnection map[string]string
 376
 377 type RemoteCluster struct {
 378         Host          string
 379         Proxy         bool
 380         Scheme        string
 381         Insecure      bool
 382         ActivateUsers bool
 383 }
 384
 385 type InstanceType struct {
 386         Name            string
 387         ProviderType    string
 388         VCPUs           int
 389         RAM             ByteSize
 390         Scratch         ByteSize
 391         IncludedScratch ByteSize
 392         AddedScratch    ByteSize
 393         Price           float64
 394         Preemptible     bool
 395 }
 396
 397 type ContainersConfig struct {
 398         CloudVMs                    CloudVMsConfig
 399         CrunchRunCommand            string
 400         CrunchRunArgumentsList      []string
 401         DefaultKeepCacheRAM         ByteSize
 402         DispatchPrivateKey          string
 403         LogReuseDecisions           bool
 404         MaxComputeVMs               int
 405         MaxDispatchAttempts         int
 406         MaxRetryAttempts            int
 407         MinRetryPeriod              Duration
 408         ReserveExtraRAM             ByteSize
 409         StaleLockTimeout            Duration
 410         SupportedDockerImageFormats StringSet
 411         UsePreemptibleInstances     bool
 412
 413         JobsAPI struct {
 414                 Enable         string
 415                 GitInternalDir string
 416         }
 417         Logging struct {
 418                 MaxAge                       Duration
 419                 LogBytesPerEvent             int
 420                 LogSecondsBetweenEvents      Duration
 421                 LogThrottlePeriod            Duration
 422                 LogThrottleBytes             int
 423                 LogThrottleLines             int
 424                 LimitLogBytesPerJob          int
 425                 LogPartialLineThrottlePeriod Duration
 426                 LogUpdatePeriod              Duration
 427                 LogUpdateSize                ByteSize
 428         }
 429         SLURM struct {
 430                 PrioritySpread             int64
 431                 SbatchArgumentsList        []string
 432                 SbatchEnvironmentVariables map[string]string
 433                 Managed                    struct {
 434                         DNSServerConfDir       string
 435                         DNSServerConfTemplate  string
 436                         DNSServerReloadCommand string
 437                         DNSServerUpdateCommand string
 438                         ComputeNodeDomain      string
 439                         ComputeNodeNameservers StringSet
 440                         AssignNodeHostname     string
 441                 }
 442         }
 443 }
 444
 445 type CloudVMsConfig struct {
 446         Enable bool
 447
 448         BootProbeCommand     string
 449         DeployRunnerBinary   string
 450         ImageID              string
 451         MaxCloudOpsPerSecond int
 452         MaxProbesPerSecond   int
 453         PollInterval         Duration
 454         ProbeInterval        Duration
 455         SSHPort              string
 456         SyncInterval         Duration
 457         TimeoutBooting       Duration
 458         TimeoutIdle          Duration
 459         TimeoutProbe         Duration
 460         TimeoutShutdown      Duration
 461         TimeoutSignal        Duration
 462         TimeoutTERM          Duration
 463         ResourceTags         map[string]string
 464         TagKeyPrefix         string
 465
 466         Driver           string
 467         DriverParameters json.RawMessage
 468 }
 469
 470 type InstanceTypeMap map[string]InstanceType
 471
 472 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 473
 474 // UnmarshalJSON handles old config files that provide an array of
 475 // instance types instead of a hash.
 476 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 477         fixup := func(t InstanceType) (InstanceType, error) {
 478                 if t.ProviderType == "" {
 479                         t.ProviderType = t.Name
 480                 }
 481                 if t.Scratch == 0 {
 482                         t.Scratch = t.IncludedScratch + t.AddedScratch
 483                 } else if t.AddedScratch == 0 {
 484                         t.AddedScratch = t.Scratch - t.IncludedScratch
 485                 } else if t.IncludedScratch == 0 {
 486                         t.IncludedScratch = t.Scratch - t.AddedScratch
 487                 }
 488
 489                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 490                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 491                 }
 492                 return t, nil
 493         }
 494
 495         if len(data) > 0 && data[0] == '[' {
 496                 var arr []InstanceType
 497                 err := json.Unmarshal(data, &arr)
 498                 if err != nil {
 499                         return err
 500                 }
 501                 if len(arr) == 0 {
 502                         *it = nil
 503                         return nil
 504                 }
 505                 *it = make(map[string]InstanceType, len(arr))
 506                 for _, t := range arr {
 507                         if _, ok := (*it)[t.Name]; ok {
 508                                 return errDuplicateInstanceTypeName
 509                         }
 510                         t, err := fixup(t)
 511                         if err != nil {
 512                                 return err
 513                         }
 514                         (*it)[t.Name] = t
 515                 }
 516                 return nil
 517         }
 518         var hash map[string]InstanceType
 519         err := json.Unmarshal(data, &hash)
 520         if err != nil {
 521                 return err
 522         }
 523         // Fill in Name field (and ProviderType field, if not
 524         // specified) using hash key.
 525         *it = InstanceTypeMap(hash)
 526         for name, t := range *it {
 527                 t.Name = name
 528                 t, err := fixup(t)
 529                 if err != nil {
 530                         return err
 531                 }
 532                 (*it)[name] = t
 533         }
 534         return nil
 535 }
 536
 537 type StringSet map[string]struct{}
 538
 539 // UnmarshalJSON handles old config files that provide an array of
 540 // instance types instead of a hash.
 541 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 542         if len(data) > 0 && data[0] == '[' {
 543                 var arr []string
 544                 err := json.Unmarshal(data, &arr)
 545                 if err != nil {
 546                         return err
 547                 }
 548                 if len(arr) == 0 {
 549                         *ss = nil
 550                         return nil
 551                 }
 552                 *ss = make(map[string]struct{}, len(arr))
 553                 for _, t := range arr {
 554                         (*ss)[t] = struct{}{}
 555                 }
 556                 return nil
 557         }
 558         var hash map[string]struct{}
 559         err := json.Unmarshal(data, &hash)
 560         if err != nil {
 561                 return err
 562         }
 563         *ss = make(map[string]struct{}, len(hash))
 564         for t := range hash {
 565                 (*ss)[t] = struct{}{}
 566         }
 567
 568         return nil
 569 }
 570
 571 type ServiceName string
 572
 573 const (
 574         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 575         ServiceNameController    ServiceName = "arvados-controller"
 576         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 577         ServiceNameHealth        ServiceName = "arvados-health"
 578         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 579         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 580         ServiceNameWebsocket     ServiceName = "arvados-ws"
 581         ServiceNameKeepbalance   ServiceName = "keep-balance"
 582         ServiceNameKeepweb       ServiceName = "keep-web"
 583         ServiceNameKeepproxy     ServiceName = "keepproxy"
 584         ServiceNameKeepstore     ServiceName = "keepstore"
 585 )
 586
 587 // Map returns all services as a map, suitable for iterating over all
 588 // services or looking up a service by name.
 589 func (svcs Services) Map() map[ServiceName]Service {
 590         return map[ServiceName]Service{
 591                 ServiceNameRailsAPI:      svcs.RailsAPI,
 592                 ServiceNameController:    svcs.Controller,
 593                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 594                 ServiceNameHealth:        svcs.Health,
 595                 ServiceNameWorkbench1:    svcs.Workbench1,
 596                 ServiceNameWorkbench2:    svcs.Workbench2,
 597                 ServiceNameWebsocket:     svcs.Websocket,
 598                 ServiceNameKeepbalance:   svcs.Keepbalance,
 599                 ServiceNameKeepweb:       svcs.WebDAV,
 600                 ServiceNameKeepproxy:     svcs.Keepproxy,
 601                 ServiceNameKeepstore:     svcs.Keepstore,
 602         }
 603 }