sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.arvados.org/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         }
  21         return "/etc/arvados/config.yml"
  22 }()
  23
  24 type Config struct {
  25         Clusters         map[string]Cluster
  26         AutoReloadConfig bool
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         cc, ok := sc.Clusters[clusterID]
  53         if !ok {
  54                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  55         }
  56         cc.ClusterID = clusterID
  57         return &cc, nil
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxBlockEntries      int
  64         MaxCollectionEntries int
  65         MaxCollectionBytes   int64
  66         MaxPermissionEntries int
  67         MaxUUIDEntries       int
  68 }
  69
  70 type Cluster struct {
  71         ClusterID       string `json:"-"`
  72         ManagementToken string
  73         SystemRootToken string
  74         Services        Services
  75         InstanceTypes   InstanceTypeMap
  76         Containers      ContainersConfig
  77         RemoteClusters  map[string]RemoteCluster
  78         PostgreSQL      PostgreSQL
  79
  80         API struct {
  81                 AsyncPermissionsUpdateInterval Duration
  82                 DisabledAPIs                   StringSet
  83                 MaxIndexDatabaseRead           int
  84                 MaxItemsPerResponse            int
  85                 MaxConcurrentRequests          int
  86                 MaxKeepBlobBuffers             int
  87                 MaxRequestAmplification        int
  88                 MaxRequestSize                 int
  89                 RequestTimeout                 Duration
  90                 SendTimeout                    Duration
  91                 WebsocketClientEventQueue      int
  92                 WebsocketServerEventQueue      int
  93                 KeepServiceRequestTimeout      Duration
  94         }
  95         AuditLogs struct {
  96                 MaxAge             Duration
  97                 MaxDeleteBatch     int
  98                 UnloggedAttributes StringSet
  99         }
 100         Collections struct {
 101                 BlobSigning              bool
 102                 BlobSigningKey           string
 103                 BlobSigningTTL           Duration
 104                 BlobTrash                bool
 105                 BlobTrashLifetime        Duration
 106                 BlobTrashCheckInterval   Duration
 107                 BlobTrashConcurrency     int
 108                 BlobDeleteConcurrency    int
 109                 BlobReplicateConcurrency int
 110                 CollectionVersioning     bool
 111                 DefaultTrashLifetime     Duration
 112                 DefaultReplication       int
 113                 ManagedProperties        map[string]struct {
 114                         Value     interface{}
 115                         Function  string
 116                         Protected bool
 117                 }
 118                 PreserveVersionIfIdle        Duration
 119                 TrashSweepInterval           Duration
 120                 TrustAllContent              bool
 121                 ForwardSlashNameSubstitution string
 122                 S3FolderObjects              bool
 123
 124                 BlobMissingReport        string
 125                 BalancePeriod            Duration
 126                 BalanceCollectionBatch   int
 127                 BalanceCollectionBuffers int
 128                 BalanceTimeout           Duration
 129
 130                 WebDAVCache WebDAVCacheConfig
 131         }
 132         Git struct {
 133                 GitCommand   string
 134                 GitoliteHome string
 135                 Repositories string
 136         }
 137         Login struct {
 138                 LDAP struct {
 139                         Enable             bool
 140                         URL                URL
 141                         StartTLS           bool
 142                         InsecureTLS        bool
 143                         StripDomain        string
 144                         AppendDomain       string
 145                         SearchAttribute    string
 146                         SearchBindUser     string
 147                         SearchBindPassword string
 148                         SearchBase         string
 149                         SearchFilters      string
 150                         EmailAttribute     string
 151                         UsernameAttribute  string
 152                 }
 153                 Google struct {
 154                         Enable                  bool
 155                         ClientID                string
 156                         ClientSecret            string
 157                         AlternateEmailAddresses bool
 158                 }
 159                 OpenIDConnect struct {
 160                         Enable             bool
 161                         Issuer             string
 162                         ClientID           string
 163                         ClientSecret       string
 164                         EmailClaim         string
 165                         EmailVerifiedClaim string
 166                         UsernameClaim      string
 167                 }
 168                 PAM struct {
 169                         Enable             bool
 170                         Service            string
 171                         DefaultEmailDomain string
 172                 }
 173                 SSO struct {
 174                         Enable            bool
 175                         ProviderAppID     string
 176                         ProviderAppSecret string
 177                 }
 178                 Test struct {
 179                         Enable bool
 180                         Users  map[string]TestUser
 181                 }
 182                 LoginCluster       string
 183                 RemoteTokenRefresh Duration
 184                 TokenLifetime      Duration
 185                 TrustedClients     map[string]struct{}
 186         }
 187         Mail struct {
 188                 MailchimpAPIKey                string
 189                 MailchimpListID                string
 190                 SendUserSetupNotificationEmail bool
 191                 IssueReporterEmailFrom         string
 192                 IssueReporterEmailTo           string
 193                 SupportEmailAddress            string
 194                 EmailFrom                      string
 195         }
 196         SystemLogs struct {
 197                 LogLevel                string
 198                 Format                  string
 199                 MaxRequestLogParamsSize int
 200         }
 201         TLS struct {
 202                 Certificate string
 203                 Key         string
 204                 Insecure    bool
 205         }
 206         Users struct {
 207                 AnonymousUserToken                    string
 208                 AdminNotifierEmailFrom                string
 209                 AutoAdminFirstUser                    bool
 210                 AutoAdminUserWithEmail                string
 211                 AutoSetupNewUsers                     bool
 212                 AutoSetupNewUsersWithRepository       bool
 213                 AutoSetupNewUsersWithVmUUID           string
 214                 AutoSetupUsernameBlacklist            StringSet
 215                 EmailSubjectPrefix                    string
 216                 NewInactiveUserNotificationRecipients StringSet
 217                 NewUserNotificationRecipients         StringSet
 218                 NewUsersAreActive                     bool
 219                 UserNotifierEmailFrom                 string
 220                 UserProfileNotificationAddress        string
 221                 PreferDomainForUsername               string
 222                 UserSetupMailText                     string
 223         }
 224         Volumes   map[string]Volume
 225         Workbench struct {
 226                 ActivationContactLink            string
 227                 APIClientConnectTimeout          Duration
 228                 APIClientReceiveTimeout          Duration
 229                 APIResponseCompression           bool
 230                 ApplicationMimetypesWithViewIcon StringSet
 231                 ArvadosDocsite                   string
 232                 ArvadosPublicDataDocURL          string
 233                 DefaultOpenIdPrefix              string
 234                 EnableGettingStartedPopup        bool
 235                 EnablePublicProjectsPage         bool
 236                 FileViewersConfigURL             string
 237                 LogViewerMaxBytes                ByteSize
 238                 MultiSiteSearch                  string
 239                 ProfilingEnabled                 bool
 240                 Repositories                     bool
 241                 RepositoryCache                  string
 242                 RunningJobLogRecordsToFetch      int
 243                 SecretKeyBase                    string
 244                 ShowRecentCollectionsOnDashboard bool
 245                 ShowUserAgreementInline          bool
 246                 ShowUserNotifications            bool
 247                 SiteName                         string
 248                 Theme                            string
 249                 UserProfileFormFields            map[string]struct {
 250                         Type                 string
 251                         FormFieldTitle       string
 252                         FormFieldDescription string
 253                         Required             bool
 254                         Position             int
 255                         Options              map[string]struct{}
 256                 }
 257                 UserProfileFormMessage string
 258                 VocabularyURL          string
 259                 WelcomePageHTML        string
 260                 InactivePageHTML       string
 261                 SSHHelpPageHTML        string
 262                 SSHHelpHostSuffix      string
 263                 IdleTimeout            Duration
 264         }
 265
 266         ForceLegacyAPI14 bool
 267 }
 268
 269 type Volume struct {
 270         AccessViaHosts   map[URL]VolumeAccess
 271         ReadOnly         bool
 272         Replication      int
 273         StorageClasses   map[string]bool
 274         Driver           string
 275         DriverParameters json.RawMessage
 276 }
 277
 278 type S3VolumeDriverParameters struct {
 279         IAMRole            string
 280         AccessKey          string
 281         SecretKey          string
 282         Endpoint           string
 283         Region             string
 284         Bucket             string
 285         LocationConstraint bool
 286         V2Signature        bool
 287         UseAWSS3v2Driver   bool
 288         IndexPageSize      int
 289         ConnectTimeout     Duration
 290         ReadTimeout        Duration
 291         RaceWindow         Duration
 292         UnsafeDelete       bool
 293 }
 294
 295 type AzureVolumeDriverParameters struct {
 296         StorageAccountName   string
 297         StorageAccountKey    string
 298         StorageBaseURL       string
 299         ContainerName        string
 300         RequestTimeout       Duration
 301         ListBlobsRetryDelay  Duration
 302         ListBlobsMaxAttempts int
 303 }
 304
 305 type DirectoryVolumeDriverParameters struct {
 306         Root      string
 307         Serialize bool
 308 }
 309
 310 type VolumeAccess struct {
 311         ReadOnly bool
 312 }
 313
 314 type Services struct {
 315         Composer       Service
 316         Controller     Service
 317         DispatchCloud  Service
 318         GitHTTP        Service
 319         GitSSH         Service
 320         Health         Service
 321         Keepbalance    Service
 322         Keepproxy      Service
 323         Keepstore      Service
 324         RailsAPI       Service
 325         SSO            Service
 326         WebDAVDownload Service
 327         WebDAV         Service
 328         WebShell       Service
 329         Websocket      Service
 330         Workbench1     Service
 331         Workbench2     Service
 332 }
 333
 334 type Service struct {
 335         InternalURLs map[URL]ServiceInstance
 336         ExternalURL  URL
 337 }
 338
 339 type TestUser struct {
 340         Email    string
 341         Password string
 342 }
 343
 344 // URL is a url.URL that is also usable as a JSON key/value.
 345 type URL url.URL
 346
 347 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 348 // used as a JSON key/value.
 349 func (su *URL) UnmarshalText(text []byte) error {
 350         u, err := url.Parse(string(text))
 351         if err == nil {
 352                 *su = URL(*u)
 353                 if su.Path == "" && su.Host != "" {
 354                         // http://example really means http://example/
 355                         su.Path = "/"
 356                 }
 357         }
 358         return err
 359 }
 360
 361 func (su URL) MarshalText() ([]byte, error) {
 362         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 363 }
 364
 365 func (su URL) String() string {
 366         return (*url.URL)(&su).String()
 367 }
 368
 369 type ServiceInstance struct {
 370         Rendezvous string `json:",omitempty"`
 371 }
 372
 373 type PostgreSQL struct {
 374         Connection     PostgreSQLConnection
 375         ConnectionPool int
 376 }
 377
 378 type PostgreSQLConnection map[string]string
 379
 380 type RemoteCluster struct {
 381         Host          string
 382         Proxy         bool
 383         Scheme        string
 384         Insecure      bool
 385         ActivateUsers bool
 386 }
 387
 388 type InstanceType struct {
 389         Name            string
 390         ProviderType    string
 391         VCPUs           int
 392         RAM             ByteSize
 393         Scratch         ByteSize
 394         IncludedScratch ByteSize
 395         AddedScratch    ByteSize
 396         Price           float64
 397         Preemptible     bool
 398 }
 399
 400 type ContainersConfig struct {
 401         CloudVMs                    CloudVMsConfig
 402         CrunchRunCommand            string
 403         CrunchRunArgumentsList      []string
 404         DefaultKeepCacheRAM         ByteSize
 405         DispatchPrivateKey          string
 406         LogReuseDecisions           bool
 407         MaxComputeVMs               int
 408         MaxDispatchAttempts         int
 409         MaxRetryAttempts            int
 410         MinRetryPeriod              Duration
 411         ReserveExtraRAM             ByteSize
 412         StaleLockTimeout            Duration
 413         SupportedDockerImageFormats StringSet
 414         UsePreemptibleInstances     bool
 415
 416         JobsAPI struct {
 417                 Enable         string
 418                 GitInternalDir string
 419         }
 420         Logging struct {
 421                 MaxAge                       Duration
 422                 LogBytesPerEvent             int
 423                 LogSecondsBetweenEvents      Duration
 424                 LogThrottlePeriod            Duration
 425                 LogThrottleBytes             int
 426                 LogThrottleLines             int
 427                 LimitLogBytesPerJob          int
 428                 LogPartialLineThrottlePeriod Duration
 429                 LogUpdatePeriod              Duration
 430                 LogUpdateSize                ByteSize
 431         }
 432         SLURM struct {
 433                 PrioritySpread             int64
 434                 SbatchArgumentsList        []string
 435                 SbatchEnvironmentVariables map[string]string
 436                 Managed                    struct {
 437                         DNSServerConfDir       string
 438                         DNSServerConfTemplate  string
 439                         DNSServerReloadCommand string
 440                         DNSServerUpdateCommand string
 441                         ComputeNodeDomain      string
 442                         ComputeNodeNameservers StringSet
 443                         AssignNodeHostname     string
 444                 }
 445         }
 446 }
 447
 448 type CloudVMsConfig struct {
 449         Enable bool
 450
 451         BootProbeCommand               string
 452         DeployRunnerBinary             string
 453         ImageID                        string
 454         MaxCloudOpsPerSecond           int
 455         MaxProbesPerSecond             int
 456         MaxConcurrentInstanceCreateOps int
 457         PollInterval                   Duration
 458         ProbeInterval                  Duration
 459         SSHPort                        string
 460         SyncInterval                   Duration
 461         TimeoutBooting                 Duration
 462         TimeoutIdle                    Duration
 463         TimeoutProbe                   Duration
 464         TimeoutShutdown                Duration
 465         TimeoutSignal                  Duration
 466         TimeoutStaleRunLock            Duration
 467         TimeoutTERM                    Duration
 468         ResourceTags                   map[string]string
 469         TagKeyPrefix                   string
 470
 471         Driver           string
 472         DriverParameters json.RawMessage
 473 }
 474
 475 type InstanceTypeMap map[string]InstanceType
 476
 477 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 478
 479 // UnmarshalJSON handles old config files that provide an array of
 480 // instance types instead of a hash.
 481 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 482         fixup := func(t InstanceType) (InstanceType, error) {
 483                 if t.ProviderType == "" {
 484                         t.ProviderType = t.Name
 485                 }
 486                 if t.Scratch == 0 {
 487                         t.Scratch = t.IncludedScratch + t.AddedScratch
 488                 } else if t.AddedScratch == 0 {
 489                         t.AddedScratch = t.Scratch - t.IncludedScratch
 490                 } else if t.IncludedScratch == 0 {
 491                         t.IncludedScratch = t.Scratch - t.AddedScratch
 492                 }
 493
 494                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 495                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 496                 }
 497                 return t, nil
 498         }
 499
 500         if len(data) > 0 && data[0] == '[' {
 501                 var arr []InstanceType
 502                 err := json.Unmarshal(data, &arr)
 503                 if err != nil {
 504                         return err
 505                 }
 506                 if len(arr) == 0 {
 507                         *it = nil
 508                         return nil
 509                 }
 510                 *it = make(map[string]InstanceType, len(arr))
 511                 for _, t := range arr {
 512                         if _, ok := (*it)[t.Name]; ok {
 513                                 return errDuplicateInstanceTypeName
 514                         }
 515                         t, err := fixup(t)
 516                         if err != nil {
 517                                 return err
 518                         }
 519                         (*it)[t.Name] = t
 520                 }
 521                 return nil
 522         }
 523         var hash map[string]InstanceType
 524         err := json.Unmarshal(data, &hash)
 525         if err != nil {
 526                 return err
 527         }
 528         // Fill in Name field (and ProviderType field, if not
 529         // specified) using hash key.
 530         *it = InstanceTypeMap(hash)
 531         for name, t := range *it {
 532                 t.Name = name
 533                 t, err := fixup(t)
 534                 if err != nil {
 535                         return err
 536                 }
 537                 (*it)[name] = t
 538         }
 539         return nil
 540 }
 541
 542 type StringSet map[string]struct{}
 543
 544 // UnmarshalJSON handles old config files that provide an array of
 545 // instance types instead of a hash.
 546 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 547         if len(data) > 0 && data[0] == '[' {
 548                 var arr []string
 549                 err := json.Unmarshal(data, &arr)
 550                 if err != nil {
 551                         return err
 552                 }
 553                 if len(arr) == 0 {
 554                         *ss = nil
 555                         return nil
 556                 }
 557                 *ss = make(map[string]struct{}, len(arr))
 558                 for _, t := range arr {
 559                         (*ss)[t] = struct{}{}
 560                 }
 561                 return nil
 562         }
 563         var hash map[string]struct{}
 564         err := json.Unmarshal(data, &hash)
 565         if err != nil {
 566                 return err
 567         }
 568         *ss = make(map[string]struct{}, len(hash))
 569         for t := range hash {
 570                 (*ss)[t] = struct{}{}
 571         }
 572
 573         return nil
 574 }
 575
 576 type ServiceName string
 577
 578 const (
 579         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 580         ServiceNameController    ServiceName = "arvados-controller"
 581         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 582         ServiceNameHealth        ServiceName = "arvados-health"
 583         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 584         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 585         ServiceNameWebsocket     ServiceName = "arvados-ws"
 586         ServiceNameKeepbalance   ServiceName = "keep-balance"
 587         ServiceNameKeepweb       ServiceName = "keep-web"
 588         ServiceNameKeepproxy     ServiceName = "keepproxy"
 589         ServiceNameKeepstore     ServiceName = "keepstore"
 590 )
 591
 592 // Map returns all services as a map, suitable for iterating over all
 593 // services or looking up a service by name.
 594 func (svcs Services) Map() map[ServiceName]Service {
 595         return map[ServiceName]Service{
 596                 ServiceNameRailsAPI:      svcs.RailsAPI,
 597                 ServiceNameController:    svcs.Controller,
 598                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 599                 ServiceNameHealth:        svcs.Health,
 600                 ServiceNameWorkbench1:    svcs.Workbench1,
 601                 ServiceNameWorkbench2:    svcs.Workbench2,
 602                 ServiceNameWebsocket:     svcs.Websocket,
 603                 ServiceNameKeepbalance:   svcs.Keepbalance,
 604                 ServiceNameKeepweb:       svcs.WebDAV,
 605                 ServiceNameKeepproxy:     svcs.Keepproxy,
 606                 ServiceNameKeepstore:     svcs.Keepstore,
 607         }
 608 }