sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.arvados.org/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         }
  21         return "/etc/arvados/config.yml"
  22 }()
  23
  24 type Config struct {
  25         Clusters         map[string]Cluster
  26         AutoReloadConfig bool
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         cc, ok := sc.Clusters[clusterID]
  53         if !ok {
  54                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  55         }
  56         cc.ClusterID = clusterID
  57         return &cc, nil
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxBlockEntries      int
  64         MaxCollectionEntries int
  65         MaxCollectionBytes   int64
  66         MaxPermissionEntries int
  67         MaxUUIDEntries       int
  68 }
  69
  70 type Cluster struct {
  71         ClusterID       string `json:"-"`
  72         ManagementToken string
  73         SystemRootToken string
  74         Services        Services
  75         InstanceTypes   InstanceTypeMap
  76         Containers      ContainersConfig
  77         RemoteClusters  map[string]RemoteCluster
  78         PostgreSQL      PostgreSQL
  79
  80         API struct {
  81                 AsyncPermissionsUpdateInterval Duration
  82                 DisabledAPIs                   StringSet
  83                 MaxIndexDatabaseRead           int
  84                 MaxItemsPerResponse            int
  85                 MaxConcurrentRequests          int
  86                 MaxKeepBlobBuffers             int
  87                 MaxRequestAmplification        int
  88                 MaxRequestSize                 int
  89                 RequestTimeout                 Duration
  90                 SendTimeout                    Duration
  91                 WebsocketClientEventQueue      int
  92                 WebsocketServerEventQueue      int
  93                 KeepServiceRequestTimeout      Duration
  94         }
  95         AuditLogs struct {
  96                 MaxAge             Duration
  97                 MaxDeleteBatch     int
  98                 UnloggedAttributes StringSet
  99         }
 100         Collections struct {
 101                 BlobSigning              bool
 102                 BlobSigningKey           string
 103                 BlobSigningTTL           Duration
 104                 BlobTrash                bool
 105                 BlobTrashLifetime        Duration
 106                 BlobTrashCheckInterval   Duration
 107                 BlobTrashConcurrency     int
 108                 BlobDeleteConcurrency    int
 109                 BlobReplicateConcurrency int
 110                 CollectionVersioning     bool
 111                 DefaultTrashLifetime     Duration
 112                 DefaultReplication       int
 113                 ManagedProperties        map[string]struct {
 114                         Value     interface{}
 115                         Function  string
 116                         Protected bool
 117                 }
 118                 PreserveVersionIfIdle        Duration
 119                 TrashSweepInterval           Duration
 120                 TrustAllContent              bool
 121                 ForwardSlashNameSubstitution string
 122                 S3FolderObjects              bool
 123
 124                 BlobMissingReport        string
 125                 BalancePeriod            Duration
 126                 BalanceCollectionBatch   int
 127                 BalanceCollectionBuffers int
 128                 BalanceTimeout           Duration
 129
 130                 WebDAVCache WebDAVCacheConfig
 131         }
 132         Git struct {
 133                 GitCommand   string
 134                 GitoliteHome string
 135                 Repositories string
 136         }
 137         Login struct {
 138                 LDAP struct {
 139                         Enable             bool
 140                         URL                URL
 141                         StartTLS           bool
 142                         InsecureTLS        bool
 143                         StripDomain        string
 144                         AppendDomain       string
 145                         SearchAttribute    string
 146                         SearchBindUser     string
 147                         SearchBindPassword string
 148                         SearchBase         string
 149                         SearchFilters      string
 150                         EmailAttribute     string
 151                         UsernameAttribute  string
 152                 }
 153                 Google struct {
 154                         Enable                          bool
 155                         ClientID                        string
 156                         ClientSecret                    string
 157                         AlternateEmailAddresses         bool
 158                         AuthenticationRequestParameters map[string]string
 159                 }
 160                 OpenIDConnect struct {
 161                         Enable                          bool
 162                         Issuer                          string
 163                         ClientID                        string
 164                         ClientSecret                    string
 165                         EmailClaim                      string
 166                         EmailVerifiedClaim              string
 167                         UsernameClaim                   string
 168                         AuthenticationRequestParameters map[string]string
 169                 }
 170                 PAM struct {
 171                         Enable             bool
 172                         Service            string
 173                         DefaultEmailDomain string
 174                 }
 175                 SSO struct {
 176                         Enable            bool
 177                         ProviderAppID     string
 178                         ProviderAppSecret string
 179                 }
 180                 Test struct {
 181                         Enable bool
 182                         Users  map[string]TestUser
 183                 }
 184                 LoginCluster       string
 185                 RemoteTokenRefresh Duration
 186                 TokenLifetime      Duration
 187                 TrustedClients     map[string]struct{}
 188         }
 189         Mail struct {
 190                 MailchimpAPIKey                string
 191                 MailchimpListID                string
 192                 SendUserSetupNotificationEmail bool
 193                 IssueReporterEmailFrom         string
 194                 IssueReporterEmailTo           string
 195                 SupportEmailAddress            string
 196                 EmailFrom                      string
 197         }
 198         SystemLogs struct {
 199                 LogLevel                string
 200                 Format                  string
 201                 MaxRequestLogParamsSize int
 202         }
 203         TLS struct {
 204                 Certificate string
 205                 Key         string
 206                 Insecure    bool
 207         }
 208         Users struct {
 209                 AnonymousUserToken                    string
 210                 AdminNotifierEmailFrom                string
 211                 AutoAdminFirstUser                    bool
 212                 AutoAdminUserWithEmail                string
 213                 AutoSetupNewUsers                     bool
 214                 AutoSetupNewUsersWithRepository       bool
 215                 AutoSetupNewUsersWithVmUUID           string
 216                 AutoSetupUsernameBlacklist            StringSet
 217                 EmailSubjectPrefix                    string
 218                 NewInactiveUserNotificationRecipients StringSet
 219                 NewUserNotificationRecipients         StringSet
 220                 NewUsersAreActive                     bool
 221                 UserNotifierEmailFrom                 string
 222                 UserProfileNotificationAddress        string
 223                 PreferDomainForUsername               string
 224                 UserSetupMailText                     string
 225         }
 226         Volumes   map[string]Volume
 227         Workbench struct {
 228                 ActivationContactLink            string
 229                 APIClientConnectTimeout          Duration
 230                 APIClientReceiveTimeout          Duration
 231                 APIResponseCompression           bool
 232                 ApplicationMimetypesWithViewIcon StringSet
 233                 ArvadosDocsite                   string
 234                 ArvadosPublicDataDocURL          string
 235                 DefaultOpenIdPrefix              string
 236                 EnableGettingStartedPopup        bool
 237                 EnablePublicProjectsPage         bool
 238                 FileViewersConfigURL             string
 239                 LogViewerMaxBytes                ByteSize
 240                 MultiSiteSearch                  string
 241                 ProfilingEnabled                 bool
 242                 Repositories                     bool
 243                 RepositoryCache                  string
 244                 RunningJobLogRecordsToFetch      int
 245                 SecretKeyBase                    string
 246                 ShowRecentCollectionsOnDashboard bool
 247                 ShowUserAgreementInline          bool
 248                 ShowUserNotifications            bool
 249                 SiteName                         string
 250                 Theme                            string
 251                 UserProfileFormFields            map[string]struct {
 252                         Type                 string
 253                         FormFieldTitle       string
 254                         FormFieldDescription string
 255                         Required             bool
 256                         Position             int
 257                         Options              map[string]struct{}
 258                 }
 259                 UserProfileFormMessage string
 260                 VocabularyURL          string
 261                 WelcomePageHTML        string
 262                 InactivePageHTML       string
 263                 SSHHelpPageHTML        string
 264                 SSHHelpHostSuffix      string
 265                 IdleTimeout            Duration
 266         }
 267
 268         ForceLegacyAPI14 bool
 269 }
 270
 271 type Volume struct {
 272         AccessViaHosts   map[URL]VolumeAccess
 273         ReadOnly         bool
 274         Replication      int
 275         StorageClasses   map[string]bool
 276         Driver           string
 277         DriverParameters json.RawMessage
 278 }
 279
 280 type S3VolumeDriverParameters struct {
 281         IAMRole            string
 282         AccessKey          string
 283         SecretKey          string
 284         Endpoint           string
 285         Region             string
 286         Bucket             string
 287         LocationConstraint bool
 288         V2Signature        bool
 289         UseAWSS3v2Driver   bool
 290         IndexPageSize      int
 291         ConnectTimeout     Duration
 292         ReadTimeout        Duration
 293         RaceWindow         Duration
 294         UnsafeDelete       bool
 295 }
 296
 297 type AzureVolumeDriverParameters struct {
 298         StorageAccountName   string
 299         StorageAccountKey    string
 300         StorageBaseURL       string
 301         ContainerName        string
 302         RequestTimeout       Duration
 303         ListBlobsRetryDelay  Duration
 304         ListBlobsMaxAttempts int
 305 }
 306
 307 type DirectoryVolumeDriverParameters struct {
 308         Root      string
 309         Serialize bool
 310 }
 311
 312 type VolumeAccess struct {
 313         ReadOnly bool
 314 }
 315
 316 type Services struct {
 317         Composer       Service
 318         Controller     Service
 319         DispatchCloud  Service
 320         GitHTTP        Service
 321         GitSSH         Service
 322         Health         Service
 323         Keepbalance    Service
 324         Keepproxy      Service
 325         Keepstore      Service
 326         RailsAPI       Service
 327         SSO            Service
 328         WebDAVDownload Service
 329         WebDAV         Service
 330         WebShell       Service
 331         Websocket      Service
 332         Workbench1     Service
 333         Workbench2     Service
 334 }
 335
 336 type Service struct {
 337         InternalURLs map[URL]ServiceInstance
 338         ExternalURL  URL
 339 }
 340
 341 type TestUser struct {
 342         Email    string
 343         Password string
 344 }
 345
 346 // URL is a url.URL that is also usable as a JSON key/value.
 347 type URL url.URL
 348
 349 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 350 // used as a JSON key/value.
 351 func (su *URL) UnmarshalText(text []byte) error {
 352         u, err := url.Parse(string(text))
 353         if err == nil {
 354                 *su = URL(*u)
 355                 if su.Path == "" && su.Host != "" {
 356                         // http://example really means http://example/
 357                         su.Path = "/"
 358                 }
 359         }
 360         return err
 361 }
 362
 363 func (su URL) MarshalText() ([]byte, error) {
 364         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 365 }
 366
 367 func (su URL) String() string {
 368         return (*url.URL)(&su).String()
 369 }
 370
 371 type ServiceInstance struct {
 372         Rendezvous string `json:",omitempty"`
 373 }
 374
 375 type PostgreSQL struct {
 376         Connection     PostgreSQLConnection
 377         ConnectionPool int
 378 }
 379
 380 type PostgreSQLConnection map[string]string
 381
 382 type RemoteCluster struct {
 383         Host          string
 384         Proxy         bool
 385         Scheme        string
 386         Insecure      bool
 387         ActivateUsers bool
 388 }
 389
 390 type InstanceType struct {
 391         Name            string
 392         ProviderType    string
 393         VCPUs           int
 394         RAM             ByteSize
 395         Scratch         ByteSize
 396         IncludedScratch ByteSize
 397         AddedScratch    ByteSize
 398         Price           float64
 399         Preemptible     bool
 400 }
 401
 402 type ContainersConfig struct {
 403         CloudVMs                    CloudVMsConfig
 404         CrunchRunCommand            string
 405         CrunchRunArgumentsList      []string
 406         DefaultKeepCacheRAM         ByteSize
 407         DispatchPrivateKey          string
 408         LogReuseDecisions           bool
 409         MaxComputeVMs               int
 410         MaxDispatchAttempts         int
 411         MaxRetryAttempts            int
 412         MinRetryPeriod              Duration
 413         ReserveExtraRAM             ByteSize
 414         StaleLockTimeout            Duration
 415         SupportedDockerImageFormats StringSet
 416         UsePreemptibleInstances     bool
 417
 418         JobsAPI struct {
 419                 Enable         string
 420                 GitInternalDir string
 421         }
 422         Logging struct {
 423                 MaxAge                       Duration
 424                 LogBytesPerEvent             int
 425                 LogSecondsBetweenEvents      Duration
 426                 LogThrottlePeriod            Duration
 427                 LogThrottleBytes             int
 428                 LogThrottleLines             int
 429                 LimitLogBytesPerJob          int
 430                 LogPartialLineThrottlePeriod Duration
 431                 LogUpdatePeriod              Duration
 432                 LogUpdateSize                ByteSize
 433         }
 434         SLURM struct {
 435                 PrioritySpread             int64
 436                 SbatchArgumentsList        []string
 437                 SbatchEnvironmentVariables map[string]string
 438                 Managed                    struct {
 439                         DNSServerConfDir       string
 440                         DNSServerConfTemplate  string
 441                         DNSServerReloadCommand string
 442                         DNSServerUpdateCommand string
 443                         ComputeNodeDomain      string
 444                         ComputeNodeNameservers StringSet
 445                         AssignNodeHostname     string
 446                 }
 447         }
 448 }
 449
 450 type CloudVMsConfig struct {
 451         Enable bool
 452
 453         BootProbeCommand               string
 454         DeployRunnerBinary             string
 455         ImageID                        string
 456         MaxCloudOpsPerSecond           int
 457         MaxProbesPerSecond             int
 458         MaxConcurrentInstanceCreateOps int
 459         PollInterval                   Duration
 460         ProbeInterval                  Duration
 461         SSHPort                        string
 462         SyncInterval                   Duration
 463         TimeoutBooting                 Duration
 464         TimeoutIdle                    Duration
 465         TimeoutProbe                   Duration
 466         TimeoutShutdown                Duration
 467         TimeoutSignal                  Duration
 468         TimeoutStaleRunLock            Duration
 469         TimeoutTERM                    Duration
 470         ResourceTags                   map[string]string
 471         TagKeyPrefix                   string
 472
 473         Driver           string
 474         DriverParameters json.RawMessage
 475 }
 476
 477 type InstanceTypeMap map[string]InstanceType
 478
 479 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 480
 481 // UnmarshalJSON handles old config files that provide an array of
 482 // instance types instead of a hash.
 483 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 484         fixup := func(t InstanceType) (InstanceType, error) {
 485                 if t.ProviderType == "" {
 486                         t.ProviderType = t.Name
 487                 }
 488                 if t.Scratch == 0 {
 489                         t.Scratch = t.IncludedScratch + t.AddedScratch
 490                 } else if t.AddedScratch == 0 {
 491                         t.AddedScratch = t.Scratch - t.IncludedScratch
 492                 } else if t.IncludedScratch == 0 {
 493                         t.IncludedScratch = t.Scratch - t.AddedScratch
 494                 }
 495
 496                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 497                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 498                 }
 499                 return t, nil
 500         }
 501
 502         if len(data) > 0 && data[0] == '[' {
 503                 var arr []InstanceType
 504                 err := json.Unmarshal(data, &arr)
 505                 if err != nil {
 506                         return err
 507                 }
 508                 if len(arr) == 0 {
 509                         *it = nil
 510                         return nil
 511                 }
 512                 *it = make(map[string]InstanceType, len(arr))
 513                 for _, t := range arr {
 514                         if _, ok := (*it)[t.Name]; ok {
 515                                 return errDuplicateInstanceTypeName
 516                         }
 517                         t, err := fixup(t)
 518                         if err != nil {
 519                                 return err
 520                         }
 521                         (*it)[t.Name] = t
 522                 }
 523                 return nil
 524         }
 525         var hash map[string]InstanceType
 526         err := json.Unmarshal(data, &hash)
 527         if err != nil {
 528                 return err
 529         }
 530         // Fill in Name field (and ProviderType field, if not
 531         // specified) using hash key.
 532         *it = InstanceTypeMap(hash)
 533         for name, t := range *it {
 534                 t.Name = name
 535                 t, err := fixup(t)
 536                 if err != nil {
 537                         return err
 538                 }
 539                 (*it)[name] = t
 540         }
 541         return nil
 542 }
 543
 544 type StringSet map[string]struct{}
 545
 546 // UnmarshalJSON handles old config files that provide an array of
 547 // instance types instead of a hash.
 548 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 549         if len(data) > 0 && data[0] == '[' {
 550                 var arr []string
 551                 err := json.Unmarshal(data, &arr)
 552                 if err != nil {
 553                         return err
 554                 }
 555                 if len(arr) == 0 {
 556                         *ss = nil
 557                         return nil
 558                 }
 559                 *ss = make(map[string]struct{}, len(arr))
 560                 for _, t := range arr {
 561                         (*ss)[t] = struct{}{}
 562                 }
 563                 return nil
 564         }
 565         var hash map[string]struct{}
 566         err := json.Unmarshal(data, &hash)
 567         if err != nil {
 568                 return err
 569         }
 570         *ss = make(map[string]struct{}, len(hash))
 571         for t := range hash {
 572                 (*ss)[t] = struct{}{}
 573         }
 574
 575         return nil
 576 }
 577
 578 type ServiceName string
 579
 580 const (
 581         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 582         ServiceNameController    ServiceName = "arvados-controller"
 583         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 584         ServiceNameHealth        ServiceName = "arvados-health"
 585         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 586         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 587         ServiceNameWebsocket     ServiceName = "arvados-ws"
 588         ServiceNameKeepbalance   ServiceName = "keep-balance"
 589         ServiceNameKeepweb       ServiceName = "keep-web"
 590         ServiceNameKeepproxy     ServiceName = "keepproxy"
 591         ServiceNameKeepstore     ServiceName = "keepstore"
 592 )
 593
 594 // Map returns all services as a map, suitable for iterating over all
 595 // services or looking up a service by name.
 596 func (svcs Services) Map() map[ServiceName]Service {
 597         return map[ServiceName]Service{
 598                 ServiceNameRailsAPI:      svcs.RailsAPI,
 599                 ServiceNameController:    svcs.Controller,
 600                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 601                 ServiceNameHealth:        svcs.Health,
 602                 ServiceNameWorkbench1:    svcs.Workbench1,
 603                 ServiceNameWorkbench2:    svcs.Workbench2,
 604                 ServiceNameWebsocket:     svcs.Websocket,
 605                 ServiceNameKeepbalance:   svcs.Keepbalance,
 606                 ServiceNameKeepweb:       svcs.WebDAV,
 607                 ServiceNameKeepproxy:     svcs.Keepproxy,
 608                 ServiceNameKeepstore:     svcs.Keepstore,
 609         }
 610 }