sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.arvados.org/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         }
  21         return "/etc/arvados/config.yml"
  22 }()
  23
  24 type Config struct {
  25         Clusters         map[string]Cluster
  26         AutoReloadConfig bool
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         cc, ok := sc.Clusters[clusterID]
  53         if !ok {
  54                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  55         }
  56         cc.ClusterID = clusterID
  57         return &cc, nil
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxBlockEntries      int
  64         MaxCollectionEntries int
  65         MaxCollectionBytes   int64
  66         MaxPermissionEntries int
  67         MaxUUIDEntries       int
  68         MaxSessions          int
  69 }
  70
  71 type UploadDownloadPermission struct {
  72         Upload   bool
  73         Download bool
  74 }
  75
  76 type UploadDownloadRolePermissions struct {
  77         User  UploadDownloadPermission
  78         Admin UploadDownloadPermission
  79 }
  80
  81 type Cluster struct {
  82         ClusterID       string `json:"-"`
  83         ManagementToken string
  84         SystemRootToken string
  85         Services        Services
  86         InstanceTypes   InstanceTypeMap
  87         Containers      ContainersConfig
  88         RemoteClusters  map[string]RemoteCluster
  89         PostgreSQL      PostgreSQL
  90
  91         API struct {
  92                 AsyncPermissionsUpdateInterval Duration
  93                 DisabledAPIs                   StringSet
  94                 MaxIndexDatabaseRead           int
  95                 MaxItemsPerResponse            int
  96                 MaxConcurrentRequests          int
  97                 MaxKeepBlobBuffers             int
  98                 MaxRequestAmplification        int
  99                 MaxRequestSize                 int
 100                 MaxTokenLifetime               Duration
 101                 RequestTimeout                 Duration
 102                 SendTimeout                    Duration
 103                 WebsocketClientEventQueue      int
 104                 WebsocketServerEventQueue      int
 105                 KeepServiceRequestTimeout      Duration
 106         }
 107         AuditLogs struct {
 108                 MaxAge             Duration
 109                 MaxDeleteBatch     int
 110                 UnloggedAttributes StringSet
 111         }
 112         Collections struct {
 113                 BlobSigning              bool
 114                 BlobSigningKey           string
 115                 BlobSigningTTL           Duration
 116                 BlobTrash                bool
 117                 BlobTrashLifetime        Duration
 118                 BlobTrashCheckInterval   Duration
 119                 BlobTrashConcurrency     int
 120                 BlobDeleteConcurrency    int
 121                 BlobReplicateConcurrency int
 122                 CollectionVersioning     bool
 123                 DefaultTrashLifetime     Duration
 124                 DefaultReplication       int
 125                 ManagedProperties        map[string]struct {
 126                         Value     interface{}
 127                         Function  string
 128                         Protected bool
 129                 }
 130                 PreserveVersionIfIdle        Duration
 131                 TrashSweepInterval           Duration
 132                 TrustAllContent              bool
 133                 ForwardSlashNameSubstitution string
 134                 S3FolderObjects              bool
 135
 136                 BlobMissingReport        string
 137                 BalancePeriod            Duration
 138                 BalanceCollectionBatch   int
 139                 BalanceCollectionBuffers int
 140                 BalanceTimeout           Duration
 141
 142                 WebDAVCache WebDAVCacheConfig
 143
 144                 KeepproxyPermission UploadDownloadRolePermissions
 145                 WebDAVPermission    UploadDownloadRolePermissions
 146                 WebDAVLogEvents     bool
 147         }
 148         Git struct {
 149                 GitCommand   string
 150                 GitoliteHome string
 151                 Repositories string
 152         }
 153         Login struct {
 154                 LDAP struct {
 155                         Enable             bool
 156                         URL                URL
 157                         StartTLS           bool
 158                         InsecureTLS        bool
 159                         StripDomain        string
 160                         AppendDomain       string
 161                         SearchAttribute    string
 162                         SearchBindUser     string
 163                         SearchBindPassword string
 164                         SearchBase         string
 165                         SearchFilters      string
 166                         EmailAttribute     string
 167                         UsernameAttribute  string
 168                 }
 169                 Google struct {
 170                         Enable                          bool
 171                         ClientID                        string
 172                         ClientSecret                    string
 173                         AlternateEmailAddresses         bool
 174                         AuthenticationRequestParameters map[string]string
 175                 }
 176                 OpenIDConnect struct {
 177                         Enable                          bool
 178                         Issuer                          string
 179                         ClientID                        string
 180                         ClientSecret                    string
 181                         EmailClaim                      string
 182                         EmailVerifiedClaim              string
 183                         UsernameClaim                   string
 184                         AcceptAccessToken               bool
 185                         AcceptAccessTokenScope          string
 186                         AuthenticationRequestParameters map[string]string
 187                 }
 188                 PAM struct {
 189                         Enable             bool
 190                         Service            string
 191                         DefaultEmailDomain string
 192                 }
 193                 Test struct {
 194                         Enable bool
 195                         Users  map[string]TestUser
 196                 }
 197                 LoginCluster       string
 198                 RemoteTokenRefresh Duration
 199                 TokenLifetime      Duration
 200                 TrustedClients     map[string]struct{}
 201                 IssueTrustedTokens bool
 202         }
 203         Mail struct {
 204                 MailchimpAPIKey                string
 205                 MailchimpListID                string
 206                 SendUserSetupNotificationEmail bool
 207                 IssueReporterEmailFrom         string
 208                 IssueReporterEmailTo           string
 209                 SupportEmailAddress            string
 210                 EmailFrom                      string
 211         }
 212         SystemLogs struct {
 213                 LogLevel                string
 214                 Format                  string
 215                 MaxRequestLogParamsSize int
 216         }
 217         TLS struct {
 218                 Certificate string
 219                 Key         string
 220                 Insecure    bool
 221         }
 222         Users struct {
 223                 AnonymousUserToken                    string
 224                 AdminNotifierEmailFrom                string
 225                 AutoAdminFirstUser                    bool
 226                 AutoAdminUserWithEmail                string
 227                 AutoSetupNewUsers                     bool
 228                 AutoSetupNewUsersWithRepository       bool
 229                 AutoSetupNewUsersWithVmUUID           string
 230                 AutoSetupUsernameBlacklist            StringSet
 231                 EmailSubjectPrefix                    string
 232                 NewInactiveUserNotificationRecipients StringSet
 233                 NewUserNotificationRecipients         StringSet
 234                 NewUsersAreActive                     bool
 235                 UserNotifierEmailFrom                 string
 236                 UserProfileNotificationAddress        string
 237                 PreferDomainForUsername               string
 238                 UserSetupMailText                     string
 239         }
 240         Volumes   map[string]Volume
 241         Workbench struct {
 242                 ActivationContactLink            string
 243                 APIClientConnectTimeout          Duration
 244                 APIClientReceiveTimeout          Duration
 245                 APIResponseCompression           bool
 246                 ApplicationMimetypesWithViewIcon StringSet
 247                 ArvadosDocsite                   string
 248                 ArvadosPublicDataDocURL          string
 249                 DefaultOpenIdPrefix              string
 250                 EnableGettingStartedPopup        bool
 251                 EnablePublicProjectsPage         bool
 252                 FileViewersConfigURL             string
 253                 LogViewerMaxBytes                ByteSize
 254                 MultiSiteSearch                  string
 255                 ProfilingEnabled                 bool
 256                 Repositories                     bool
 257                 RepositoryCache                  string
 258                 RunningJobLogRecordsToFetch      int
 259                 SecretKeyBase                    string
 260                 ShowRecentCollectionsOnDashboard bool
 261                 ShowUserAgreementInline          bool
 262                 ShowUserNotifications            bool
 263                 SiteName                         string
 264                 Theme                            string
 265                 UserProfileFormFields            map[string]struct {
 266                         Type                 string
 267                         FormFieldTitle       string
 268                         FormFieldDescription string
 269                         Required             bool
 270                         Position             int
 271                         Options              map[string]struct{}
 272                 }
 273                 UserProfileFormMessage string
 274                 VocabularyURL          string
 275                 WelcomePageHTML        string
 276                 InactivePageHTML       string
 277                 SSHHelpPageHTML        string
 278                 SSHHelpHostSuffix      string
 279                 IdleTimeout            Duration
 280         }
 281 }
 282
 283 type Volume struct {
 284         AccessViaHosts   map[URL]VolumeAccess
 285         ReadOnly         bool
 286         Replication      int
 287         StorageClasses   map[string]bool
 288         Driver           string
 289         DriverParameters json.RawMessage
 290 }
 291
 292 type S3VolumeDriverParameters struct {
 293         IAMRole            string
 294         AccessKeyID        string
 295         SecretAccessKey    string
 296         Endpoint           string
 297         Region             string
 298         Bucket             string
 299         LocationConstraint bool
 300         V2Signature        bool
 301         UseAWSS3v2Driver   bool
 302         IndexPageSize      int
 303         ConnectTimeout     Duration
 304         ReadTimeout        Duration
 305         RaceWindow         Duration
 306         UnsafeDelete       bool
 307 }
 308
 309 type AzureVolumeDriverParameters struct {
 310         StorageAccountName   string
 311         StorageAccountKey    string
 312         StorageBaseURL       string
 313         ContainerName        string
 314         RequestTimeout       Duration
 315         ListBlobsRetryDelay  Duration
 316         ListBlobsMaxAttempts int
 317 }
 318
 319 type DirectoryVolumeDriverParameters struct {
 320         Root      string
 321         Serialize bool
 322 }
 323
 324 type VolumeAccess struct {
 325         ReadOnly bool
 326 }
 327
 328 type Services struct {
 329         Composer       Service
 330         Controller     Service
 331         DispatchCloud  Service
 332         GitHTTP        Service
 333         GitSSH         Service
 334         Health         Service
 335         Keepbalance    Service
 336         Keepproxy      Service
 337         Keepstore      Service
 338         RailsAPI       Service
 339         WebDAVDownload Service
 340         WebDAV         Service
 341         WebShell       Service
 342         Websocket      Service
 343         Workbench1     Service
 344         Workbench2     Service
 345 }
 346
 347 type Service struct {
 348         InternalURLs map[URL]ServiceInstance
 349         ExternalURL  URL
 350 }
 351
 352 type TestUser struct {
 353         Email    string
 354         Password string
 355 }
 356
 357 // URL is a url.URL that is also usable as a JSON key/value.
 358 type URL url.URL
 359
 360 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 361 // used as a JSON key/value.
 362 func (su *URL) UnmarshalText(text []byte) error {
 363         u, err := url.Parse(string(text))
 364         if err == nil {
 365                 *su = URL(*u)
 366                 if su.Path == "" && su.Host != "" {
 367                         // http://example really means http://example/
 368                         su.Path = "/"
 369                 }
 370         }
 371         return err
 372 }
 373
 374 func (su URL) MarshalText() ([]byte, error) {
 375         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 376 }
 377
 378 func (su URL) String() string {
 379         return (*url.URL)(&su).String()
 380 }
 381
 382 type ServiceInstance struct {
 383         Rendezvous string `json:",omitempty"`
 384 }
 385
 386 type PostgreSQL struct {
 387         Connection     PostgreSQLConnection
 388         ConnectionPool int
 389 }
 390
 391 type PostgreSQLConnection map[string]string
 392
 393 type RemoteCluster struct {
 394         Host          string
 395         Proxy         bool
 396         Scheme        string
 397         Insecure      bool
 398         ActivateUsers bool
 399 }
 400
 401 type InstanceType struct {
 402         Name            string
 403         ProviderType    string
 404         VCPUs           int
 405         RAM             ByteSize
 406         Scratch         ByteSize
 407         IncludedScratch ByteSize
 408         AddedScratch    ByteSize
 409         Price           float64
 410         Preemptible     bool
 411 }
 412
 413 type ContainersConfig struct {
 414         CloudVMs                    CloudVMsConfig
 415         CrunchRunCommand            string
 416         CrunchRunArgumentsList      []string
 417         DefaultKeepCacheRAM         ByteSize
 418         DispatchPrivateKey          string
 419         LogReuseDecisions           bool
 420         MaxComputeVMs               int
 421         MaxDispatchAttempts         int
 422         MaxRetryAttempts            int
 423         MinRetryPeriod              Duration
 424         ReserveExtraRAM             ByteSize
 425         StaleLockTimeout            Duration
 426         SupportedDockerImageFormats StringSet
 427         UsePreemptibleInstances     bool
 428         RuntimeEngine               string
 429
 430         JobsAPI struct {
 431                 Enable         string
 432                 GitInternalDir string
 433         }
 434         Logging struct {
 435                 MaxAge                       Duration
 436                 LogBytesPerEvent             int
 437                 LogSecondsBetweenEvents      Duration
 438                 LogThrottlePeriod            Duration
 439                 LogThrottleBytes             int
 440                 LogThrottleLines             int
 441                 LimitLogBytesPerJob          int
 442                 LogPartialLineThrottlePeriod Duration
 443                 LogUpdatePeriod              Duration
 444                 LogUpdateSize                ByteSize
 445         }
 446         ShellAccess struct {
 447                 Admin bool
 448                 User  bool
 449         }
 450         SLURM struct {
 451                 PrioritySpread             int64
 452                 SbatchArgumentsList        []string
 453                 SbatchEnvironmentVariables map[string]string
 454                 Managed                    struct {
 455                         DNSServerConfDir       string
 456                         DNSServerConfTemplate  string
 457                         DNSServerReloadCommand string
 458                         DNSServerUpdateCommand string
 459                         ComputeNodeDomain      string
 460                         ComputeNodeNameservers StringSet
 461                         AssignNodeHostname     string
 462                 }
 463         }
 464 }
 465
 466 type CloudVMsConfig struct {
 467         Enable bool
 468
 469         BootProbeCommand               string
 470         DeployRunnerBinary             string
 471         ImageID                        string
 472         MaxCloudOpsPerSecond           int
 473         MaxProbesPerSecond             int
 474         MaxConcurrentInstanceCreateOps int
 475         PollInterval                   Duration
 476         ProbeInterval                  Duration
 477         SSHPort                        string
 478         SyncInterval                   Duration
 479         TimeoutBooting                 Duration
 480         TimeoutIdle                    Duration
 481         TimeoutProbe                   Duration
 482         TimeoutShutdown                Duration
 483         TimeoutSignal                  Duration
 484         TimeoutStaleRunLock            Duration
 485         TimeoutTERM                    Duration
 486         ResourceTags                   map[string]string
 487         TagKeyPrefix                   string
 488
 489         Driver           string
 490         DriverParameters json.RawMessage
 491 }
 492
 493 type InstanceTypeMap map[string]InstanceType
 494
 495 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 496
 497 // UnmarshalJSON handles old config files that provide an array of
 498 // instance types instead of a hash.
 499 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 500         fixup := func(t InstanceType) (InstanceType, error) {
 501                 if t.ProviderType == "" {
 502                         t.ProviderType = t.Name
 503                 }
 504                 if t.Scratch == 0 {
 505                         t.Scratch = t.IncludedScratch + t.AddedScratch
 506                 } else if t.AddedScratch == 0 {
 507                         t.AddedScratch = t.Scratch - t.IncludedScratch
 508                 } else if t.IncludedScratch == 0 {
 509                         t.IncludedScratch = t.Scratch - t.AddedScratch
 510                 }
 511
 512                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 513                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 514                 }
 515                 return t, nil
 516         }
 517
 518         if len(data) > 0 && data[0] == '[' {
 519                 var arr []InstanceType
 520                 err := json.Unmarshal(data, &arr)
 521                 if err != nil {
 522                         return err
 523                 }
 524                 if len(arr) == 0 {
 525                         *it = nil
 526                         return nil
 527                 }
 528                 *it = make(map[string]InstanceType, len(arr))
 529                 for _, t := range arr {
 530                         if _, ok := (*it)[t.Name]; ok {
 531                                 return errDuplicateInstanceTypeName
 532                         }
 533                         t, err := fixup(t)
 534                         if err != nil {
 535                                 return err
 536                         }
 537                         (*it)[t.Name] = t
 538                 }
 539                 return nil
 540         }
 541         var hash map[string]InstanceType
 542         err := json.Unmarshal(data, &hash)
 543         if err != nil {
 544                 return err
 545         }
 546         // Fill in Name field (and ProviderType field, if not
 547         // specified) using hash key.
 548         *it = InstanceTypeMap(hash)
 549         for name, t := range *it {
 550                 t.Name = name
 551                 t, err := fixup(t)
 552                 if err != nil {
 553                         return err
 554                 }
 555                 (*it)[name] = t
 556         }
 557         return nil
 558 }
 559
 560 type StringSet map[string]struct{}
 561
 562 // UnmarshalJSON handles old config files that provide an array of
 563 // instance types instead of a hash.
 564 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 565         if len(data) > 0 && data[0] == '[' {
 566                 var arr []string
 567                 err := json.Unmarshal(data, &arr)
 568                 if err != nil {
 569                         return err
 570                 }
 571                 if len(arr) == 0 {
 572                         *ss = nil
 573                         return nil
 574                 }
 575                 *ss = make(map[string]struct{}, len(arr))
 576                 for _, t := range arr {
 577                         (*ss)[t] = struct{}{}
 578                 }
 579                 return nil
 580         }
 581         var hash map[string]struct{}
 582         err := json.Unmarshal(data, &hash)
 583         if err != nil {
 584                 return err
 585         }
 586         *ss = make(map[string]struct{}, len(hash))
 587         for t := range hash {
 588                 (*ss)[t] = struct{}{}
 589         }
 590
 591         return nil
 592 }
 593
 594 type ServiceName string
 595
 596 const (
 597         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 598         ServiceNameController    ServiceName = "arvados-controller"
 599         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 600         ServiceNameHealth        ServiceName = "arvados-health"
 601         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 602         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 603         ServiceNameWebsocket     ServiceName = "arvados-ws"
 604         ServiceNameKeepbalance   ServiceName = "keep-balance"
 605         ServiceNameKeepweb       ServiceName = "keep-web"
 606         ServiceNameKeepproxy     ServiceName = "keepproxy"
 607         ServiceNameKeepstore     ServiceName = "keepstore"
 608 )
 609
 610 // Map returns all services as a map, suitable for iterating over all
 611 // services or looking up a service by name.
 612 func (svcs Services) Map() map[ServiceName]Service {
 613         return map[ServiceName]Service{
 614                 ServiceNameRailsAPI:      svcs.RailsAPI,
 615                 ServiceNameController:    svcs.Controller,
 616                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 617                 ServiceNameHealth:        svcs.Health,
 618                 ServiceNameWorkbench1:    svcs.Workbench1,
 619                 ServiceNameWorkbench2:    svcs.Workbench2,
 620                 ServiceNameWebsocket:     svcs.Websocket,
 621                 ServiceNameKeepbalance:   svcs.Keepbalance,
 622                 ServiceNameKeepweb:       svcs.WebDAV,
 623                 ServiceNameKeepproxy:     svcs.Keepproxy,
 624                 ServiceNameKeepstore:     svcs.Keepstore,
 625         }
 626 }