sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.arvados.org/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         }
  21         return "/etc/arvados/config.yml"
  22 }()
  23
  24 type Config struct {
  25         Clusters         map[string]Cluster
  26         AutoReloadConfig bool
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         if cc, ok := sc.Clusters[clusterID]; !ok {
  53                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  54         } else {
  55                 cc.ClusterID = clusterID
  56                 return &cc, nil
  57         }
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxBlockEntries      int
  64         MaxCollectionEntries int
  65         MaxCollectionBytes   int64
  66         MaxPermissionEntries int
  67         MaxUUIDEntries       int
  68 }
  69
  70 type Cluster struct {
  71         ClusterID       string `json:"-"`
  72         ManagementToken string
  73         SystemRootToken string
  74         Services        Services
  75         InstanceTypes   InstanceTypeMap
  76         Containers      ContainersConfig
  77         RemoteClusters  map[string]RemoteCluster
  78         PostgreSQL      PostgreSQL
  79
  80         API struct {
  81                 AsyncPermissionsUpdateInterval Duration
  82                 DisabledAPIs                   StringSet
  83                 MaxIndexDatabaseRead           int
  84                 MaxItemsPerResponse            int
  85                 MaxConcurrentRequests          int
  86                 MaxKeepBlobBuffers             int
  87                 MaxRequestAmplification        int
  88                 MaxRequestSize                 int
  89                 RailsSessionSecretToken        string
  90                 RequestTimeout                 Duration
  91                 SendTimeout                    Duration
  92                 WebsocketClientEventQueue      int
  93                 WebsocketServerEventQueue      int
  94                 KeepServiceRequestTimeout      Duration
  95         }
  96         AuditLogs struct {
  97                 MaxAge             Duration
  98                 MaxDeleteBatch     int
  99                 UnloggedAttributes StringSet
 100         }
 101         Collections struct {
 102                 BlobSigning              bool
 103                 BlobSigningKey           string
 104                 BlobSigningTTL           Duration
 105                 BlobTrash                bool
 106                 BlobTrashLifetime        Duration
 107                 BlobTrashCheckInterval   Duration
 108                 BlobTrashConcurrency     int
 109                 BlobDeleteConcurrency    int
 110                 BlobReplicateConcurrency int
 111                 CollectionVersioning     bool
 112                 DefaultTrashLifetime     Duration
 113                 DefaultReplication       int
 114                 ManagedProperties        map[string]struct {
 115                         Value     interface{}
 116                         Function  string
 117                         Protected bool
 118                 }
 119                 PreserveVersionIfIdle        Duration
 120                 TrashSweepInterval           Duration
 121                 TrustAllContent              bool
 122                 ForwardSlashNameSubstitution string
 123                 S3FolderObjects              bool
 124
 125                 BlobMissingReport        string
 126                 BalancePeriod            Duration
 127                 BalanceCollectionBatch   int
 128                 BalanceCollectionBuffers int
 129                 BalanceTimeout           Duration
 130
 131                 WebDAVCache WebDAVCacheConfig
 132         }
 133         Git struct {
 134                 GitCommand   string
 135                 GitoliteHome string
 136                 Repositories string
 137         }
 138         Login struct {
 139                 LDAP struct {
 140                         Enable             bool
 141                         URL                URL
 142                         StartTLS           bool
 143                         InsecureTLS        bool
 144                         StripDomain        string
 145                         AppendDomain       string
 146                         SearchAttribute    string
 147                         SearchBindUser     string
 148                         SearchBindPassword string
 149                         SearchBase         string
 150                         SearchFilters      string
 151                         EmailAttribute     string
 152                         UsernameAttribute  string
 153                 }
 154                 Google struct {
 155                         Enable                  bool
 156                         ClientID                string
 157                         ClientSecret            string
 158                         AlternateEmailAddresses bool
 159                 }
 160                 OpenIDConnect struct {
 161                         Enable             bool
 162                         Issuer             string
 163                         ClientID           string
 164                         ClientSecret       string
 165                         EmailClaim         string
 166                         EmailVerifiedClaim string
 167                         UsernameClaim      string
 168                 }
 169                 PAM struct {
 170                         Enable             bool
 171                         Service            string
 172                         DefaultEmailDomain string
 173                 }
 174                 SSO struct {
 175                         Enable            bool
 176                         ProviderAppID     string
 177                         ProviderAppSecret string
 178                 }
 179                 Test struct {
 180                         Enable bool
 181                         Users  map[string]TestUser
 182                 }
 183                 LoginCluster       string
 184                 RemoteTokenRefresh Duration
 185                 TokenLifetime      Duration
 186         }
 187         Mail struct {
 188                 MailchimpAPIKey                string
 189                 MailchimpListID                string
 190                 SendUserSetupNotificationEmail bool
 191                 IssueReporterEmailFrom         string
 192                 IssueReporterEmailTo           string
 193                 SupportEmailAddress            string
 194                 EmailFrom                      string
 195         }
 196         SystemLogs struct {
 197                 LogLevel                string
 198                 Format                  string
 199                 MaxRequestLogParamsSize int
 200         }
 201         TLS struct {
 202                 Certificate string
 203                 Key         string
 204                 Insecure    bool
 205         }
 206         Users struct {
 207                 AnonymousUserToken                    string
 208                 AdminNotifierEmailFrom                string
 209                 AutoAdminFirstUser                    bool
 210                 AutoAdminUserWithEmail                string
 211                 AutoSetupNewUsers                     bool
 212                 AutoSetupNewUsersWithRepository       bool
 213                 AutoSetupNewUsersWithVmUUID           string
 214                 AutoSetupUsernameBlacklist            StringSet
 215                 EmailSubjectPrefix                    string
 216                 NewInactiveUserNotificationRecipients StringSet
 217                 NewUserNotificationRecipients         StringSet
 218                 NewUsersAreActive                     bool
 219                 UserNotifierEmailFrom                 string
 220                 UserProfileNotificationAddress        string
 221                 PreferDomainForUsername               string
 222         }
 223         Volumes   map[string]Volume
 224         Workbench struct {
 225                 ActivationContactLink            string
 226                 APIClientConnectTimeout          Duration
 227                 APIClientReceiveTimeout          Duration
 228                 APIResponseCompression           bool
 229                 ApplicationMimetypesWithViewIcon StringSet
 230                 ArvadosDocsite                   string
 231                 ArvadosPublicDataDocURL          string
 232                 DefaultOpenIdPrefix              string
 233                 EnableGettingStartedPopup        bool
 234                 EnablePublicProjectsPage         bool
 235                 FileViewersConfigURL             string
 236                 LogViewerMaxBytes                ByteSize
 237                 MultiSiteSearch                  string
 238                 ProfilingEnabled                 bool
 239                 Repositories                     bool
 240                 RepositoryCache                  string
 241                 RunningJobLogRecordsToFetch      int
 242                 SecretKeyBase                    string
 243                 ShowRecentCollectionsOnDashboard bool
 244                 ShowUserAgreementInline          bool
 245                 ShowUserNotifications            bool
 246                 SiteName                         string
 247                 Theme                            string
 248                 UserProfileFormFields            map[string]struct {
 249                         Type                 string
 250                         FormFieldTitle       string
 251                         FormFieldDescription string
 252                         Required             bool
 253                         Position             int
 254                         Options              map[string]struct{}
 255                 }
 256                 UserProfileFormMessage string
 257                 VocabularyURL          string
 258                 WelcomePageHTML        string
 259                 InactivePageHTML       string
 260                 SSHHelpPageHTML        string
 261                 SSHHelpHostSuffix      string
 262                 IdleTimeout            Duration
 263         }
 264
 265         ForceLegacyAPI14 bool
 266 }
 267
 268 type Volume struct {
 269         AccessViaHosts   map[URL]VolumeAccess
 270         ReadOnly         bool
 271         Replication      int
 272         StorageClasses   map[string]bool
 273         Driver           string
 274         DriverParameters json.RawMessage
 275 }
 276
 277 type S3VolumeDriverParameters struct {
 278         IAMRole            string
 279         AccessKey          string
 280         SecretKey          string
 281         Endpoint           string
 282         Region             string
 283         Bucket             string
 284         LocationConstraint bool
 285         V2Signature        bool
 286         UseAWSS3v2Driver   bool
 287         IndexPageSize      int
 288         ConnectTimeout     Duration
 289         ReadTimeout        Duration
 290         RaceWindow         Duration
 291         UnsafeDelete       bool
 292 }
 293
 294 type AzureVolumeDriverParameters struct {
 295         StorageAccountName   string
 296         StorageAccountKey    string
 297         StorageBaseURL       string
 298         ContainerName        string
 299         RequestTimeout       Duration
 300         ListBlobsRetryDelay  Duration
 301         ListBlobsMaxAttempts int
 302 }
 303
 304 type DirectoryVolumeDriverParameters struct {
 305         Root      string
 306         Serialize bool
 307 }
 308
 309 type VolumeAccess struct {
 310         ReadOnly bool
 311 }
 312
 313 type Services struct {
 314         Composer       Service
 315         Controller     Service
 316         DispatchCloud  Service
 317         GitHTTP        Service
 318         GitSSH         Service
 319         Health         Service
 320         Keepbalance    Service
 321         Keepproxy      Service
 322         Keepstore      Service
 323         RailsAPI       Service
 324         SSO            Service
 325         WebDAVDownload Service
 326         WebDAV         Service
 327         WebShell       Service
 328         Websocket      Service
 329         Workbench1     Service
 330         Workbench2     Service
 331 }
 332
 333 type Service struct {
 334         InternalURLs map[URL]ServiceInstance
 335         ExternalURL  URL
 336 }
 337
 338 type TestUser struct {
 339         Email    string
 340         Password string
 341 }
 342
 343 // URL is a url.URL that is also usable as a JSON key/value.
 344 type URL url.URL
 345
 346 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 347 // used as a JSON key/value.
 348 func (su *URL) UnmarshalText(text []byte) error {
 349         u, err := url.Parse(string(text))
 350         if err == nil {
 351                 *su = URL(*u)
 352                 if su.Path == "" && su.Host != "" {
 353                         // http://example really means http://example/
 354                         su.Path = "/"
 355                 }
 356         }
 357         return err
 358 }
 359
 360 func (su URL) MarshalText() ([]byte, error) {
 361         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 362 }
 363
 364 func (su URL) String() string {
 365         return (*url.URL)(&su).String()
 366 }
 367
 368 type ServiceInstance struct {
 369         Rendezvous string `json:",omitempty"`
 370 }
 371
 372 type PostgreSQL struct {
 373         Connection     PostgreSQLConnection
 374         ConnectionPool int
 375 }
 376
 377 type PostgreSQLConnection map[string]string
 378
 379 type RemoteCluster struct {
 380         Host          string
 381         Proxy         bool
 382         Scheme        string
 383         Insecure      bool
 384         ActivateUsers bool
 385 }
 386
 387 type InstanceType struct {
 388         Name            string
 389         ProviderType    string
 390         VCPUs           int
 391         RAM             ByteSize
 392         Scratch         ByteSize
 393         IncludedScratch ByteSize
 394         AddedScratch    ByteSize
 395         Price           float64
 396         Preemptible     bool
 397 }
 398
 399 type ContainersConfig struct {
 400         CloudVMs                    CloudVMsConfig
 401         CrunchRunCommand            string
 402         CrunchRunArgumentsList      []string
 403         DefaultKeepCacheRAM         ByteSize
 404         DispatchPrivateKey          string
 405         LogReuseDecisions           bool
 406         MaxComputeVMs               int
 407         MaxDispatchAttempts         int
 408         MaxRetryAttempts            int
 409         MinRetryPeriod              Duration
 410         ReserveExtraRAM             ByteSize
 411         StaleLockTimeout            Duration
 412         SupportedDockerImageFormats StringSet
 413         UsePreemptibleInstances     bool
 414
 415         JobsAPI struct {
 416                 Enable         string
 417                 GitInternalDir string
 418         }
 419         Logging struct {
 420                 MaxAge                       Duration
 421                 LogBytesPerEvent             int
 422                 LogSecondsBetweenEvents      Duration
 423                 LogThrottlePeriod            Duration
 424                 LogThrottleBytes             int
 425                 LogThrottleLines             int
 426                 LimitLogBytesPerJob          int
 427                 LogPartialLineThrottlePeriod Duration
 428                 LogUpdatePeriod              Duration
 429                 LogUpdateSize                ByteSize
 430         }
 431         SLURM struct {
 432                 PrioritySpread             int64
 433                 SbatchArgumentsList        []string
 434                 SbatchEnvironmentVariables map[string]string
 435                 Managed                    struct {
 436                         DNSServerConfDir       string
 437                         DNSServerConfTemplate  string
 438                         DNSServerReloadCommand string
 439                         DNSServerUpdateCommand string
 440                         ComputeNodeDomain      string
 441                         ComputeNodeNameservers StringSet
 442                         AssignNodeHostname     string
 443                 }
 444         }
 445 }
 446
 447 type CloudVMsConfig struct {
 448         Enable bool
 449
 450         BootProbeCommand               string
 451         DeployRunnerBinary             string
 452         ImageID                        string
 453         MaxCloudOpsPerSecond           int
 454         MaxProbesPerSecond             int
 455         MaxConcurrentInstanceCreateOps int
 456         PollInterval                   Duration
 457         ProbeInterval                  Duration
 458         SSHPort                        string
 459         SyncInterval                   Duration
 460         TimeoutBooting                 Duration
 461         TimeoutIdle                    Duration
 462         TimeoutProbe                   Duration
 463         TimeoutShutdown                Duration
 464         TimeoutSignal                  Duration
 465         TimeoutStaleRunLock            Duration
 466         TimeoutTERM                    Duration
 467         ResourceTags                   map[string]string
 468         TagKeyPrefix                   string
 469
 470         Driver           string
 471         DriverParameters json.RawMessage
 472 }
 473
 474 type InstanceTypeMap map[string]InstanceType
 475
 476 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 477
 478 // UnmarshalJSON handles old config files that provide an array of
 479 // instance types instead of a hash.
 480 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 481         fixup := func(t InstanceType) (InstanceType, error) {
 482                 if t.ProviderType == "" {
 483                         t.ProviderType = t.Name
 484                 }
 485                 if t.Scratch == 0 {
 486                         t.Scratch = t.IncludedScratch + t.AddedScratch
 487                 } else if t.AddedScratch == 0 {
 488                         t.AddedScratch = t.Scratch - t.IncludedScratch
 489                 } else if t.IncludedScratch == 0 {
 490                         t.IncludedScratch = t.Scratch - t.AddedScratch
 491                 }
 492
 493                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 494                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 495                 }
 496                 return t, nil
 497         }
 498
 499         if len(data) > 0 && data[0] == '[' {
 500                 var arr []InstanceType
 501                 err := json.Unmarshal(data, &arr)
 502                 if err != nil {
 503                         return err
 504                 }
 505                 if len(arr) == 0 {
 506                         *it = nil
 507                         return nil
 508                 }
 509                 *it = make(map[string]InstanceType, len(arr))
 510                 for _, t := range arr {
 511                         if _, ok := (*it)[t.Name]; ok {
 512                                 return errDuplicateInstanceTypeName
 513                         }
 514                         t, err := fixup(t)
 515                         if err != nil {
 516                                 return err
 517                         }
 518                         (*it)[t.Name] = t
 519                 }
 520                 return nil
 521         }
 522         var hash map[string]InstanceType
 523         err := json.Unmarshal(data, &hash)
 524         if err != nil {
 525                 return err
 526         }
 527         // Fill in Name field (and ProviderType field, if not
 528         // specified) using hash key.
 529         *it = InstanceTypeMap(hash)
 530         for name, t := range *it {
 531                 t.Name = name
 532                 t, err := fixup(t)
 533                 if err != nil {
 534                         return err
 535                 }
 536                 (*it)[name] = t
 537         }
 538         return nil
 539 }
 540
 541 type StringSet map[string]struct{}
 542
 543 // UnmarshalJSON handles old config files that provide an array of
 544 // instance types instead of a hash.
 545 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 546         if len(data) > 0 && data[0] == '[' {
 547                 var arr []string
 548                 err := json.Unmarshal(data, &arr)
 549                 if err != nil {
 550                         return err
 551                 }
 552                 if len(arr) == 0 {
 553                         *ss = nil
 554                         return nil
 555                 }
 556                 *ss = make(map[string]struct{}, len(arr))
 557                 for _, t := range arr {
 558                         (*ss)[t] = struct{}{}
 559                 }
 560                 return nil
 561         }
 562         var hash map[string]struct{}
 563         err := json.Unmarshal(data, &hash)
 564         if err != nil {
 565                 return err
 566         }
 567         *ss = make(map[string]struct{}, len(hash))
 568         for t := range hash {
 569                 (*ss)[t] = struct{}{}
 570         }
 571
 572         return nil
 573 }
 574
 575 type ServiceName string
 576
 577 const (
 578         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 579         ServiceNameController    ServiceName = "arvados-controller"
 580         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 581         ServiceNameHealth        ServiceName = "arvados-health"
 582         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 583         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 584         ServiceNameWebsocket     ServiceName = "arvados-ws"
 585         ServiceNameKeepbalance   ServiceName = "keep-balance"
 586         ServiceNameKeepweb       ServiceName = "keep-web"
 587         ServiceNameKeepproxy     ServiceName = "keepproxy"
 588         ServiceNameKeepstore     ServiceName = "keepstore"
 589 )
 590
 591 // Map returns all services as a map, suitable for iterating over all
 592 // services or looking up a service by name.
 593 func (svcs Services) Map() map[ServiceName]Service {
 594         return map[ServiceName]Service{
 595                 ServiceNameRailsAPI:      svcs.RailsAPI,
 596                 ServiceNameController:    svcs.Controller,
 597                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 598                 ServiceNameHealth:        svcs.Health,
 599                 ServiceNameWorkbench1:    svcs.Workbench1,
 600                 ServiceNameWorkbench2:    svcs.Workbench2,
 601                 ServiceNameWebsocket:     svcs.Websocket,
 602                 ServiceNameKeepbalance:   svcs.Keepbalance,
 603                 ServiceNameKeepweb:       svcs.WebDAV,
 604                 ServiceNameKeepproxy:     svcs.Keepproxy,
 605                 ServiceNameKeepstore:     svcs.Keepstore,
 606         }
 607 }