sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "crypto/tls"
   9         "encoding/json"
  10         "errors"
  11         "fmt"
  12         "net/url"
  13         "os"
  14         "time"
  15
  16         "git.arvados.org/arvados.git/sdk/go/config"
  17 )
  18
  19 var DefaultConfigFile = func() string {
  20         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  21                 return path
  22         }
  23         return "/etc/arvados/config.yml"
  24 }()
  25
  26 type Config struct {
  27         Clusters         map[string]Cluster
  28         AutoReloadConfig bool
  29         SourceTimestamp  time.Time
  30         SourceSHA256     string
  31 }
  32
  33 // GetConfig returns the current system config, loading it from
  34 // configFile if needed.
  35 func GetConfig(configFile string) (*Config, error) {
  36         var cfg Config
  37         err := config.LoadFile(&cfg, configFile)
  38         return &cfg, err
  39 }
  40
  41 // GetCluster returns the cluster ID and config for the given
  42 // cluster, or the default/only configured cluster if clusterID is "".
  43 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  44         if clusterID == "" {
  45                 if len(sc.Clusters) == 0 {
  46                         return nil, fmt.Errorf("no clusters configured")
  47                 } else if len(sc.Clusters) > 1 {
  48                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  49                 } else {
  50                         for id, cc := range sc.Clusters {
  51                                 cc.ClusterID = id
  52                                 return &cc, nil
  53                         }
  54                 }
  55         }
  56         cc, ok := sc.Clusters[clusterID]
  57         if !ok {
  58                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  59         }
  60         cc.ClusterID = clusterID
  61         return &cc, nil
  62 }
  63
  64 type WebDAVCacheConfig struct {
  65         TTL                Duration
  66         DiskCacheSize      ByteSizeOrPercent
  67         MaxCollectionBytes ByteSize
  68         MaxSessions        int
  69 }
  70
  71 type UploadDownloadPermission struct {
  72         Upload   bool
  73         Download bool
  74 }
  75
  76 type UploadDownloadRolePermissions struct {
  77         User  UploadDownloadPermission
  78         Admin UploadDownloadPermission
  79 }
  80
  81 type ManagedProperties map[string]struct {
  82         Value     interface{}
  83         Function  string
  84         Protected bool
  85 }
  86
  87 type Cluster struct {
  88         ClusterID       string `json:"-"`
  89         ManagementToken string
  90         SystemRootToken string
  91         Services        Services
  92         InstanceTypes   InstanceTypeMap
  93         Containers      ContainersConfig
  94         RemoteClusters  map[string]RemoteCluster
  95         PostgreSQL      PostgreSQL
  96
  97         API struct {
  98                 AsyncPermissionsUpdateInterval   Duration
  99                 DisabledAPIs                     StringSet
 100                 MaxIndexDatabaseRead             int
 101                 MaxItemsPerResponse              int
 102                 MaxConcurrentRailsRequests       int
 103                 MaxConcurrentRequests            int
 104                 MaxQueuedRequests                int
 105                 MaxGatewayTunnels                int
 106                 MaxQueueTimeForLockRequests      Duration
 107                 LogCreateRequestFraction         float64
 108                 MaxKeepBlobBuffers               int
 109                 MaxRequestAmplification          int
 110                 MaxRequestSize                   int
 111                 MaxTokenLifetime                 Duration
 112                 RequestTimeout                   Duration
 113                 SendTimeout                      Duration
 114                 WebsocketClientEventQueue        int
 115                 WebsocketServerEventQueue        int
 116                 KeepServiceRequestTimeout        Duration
 117                 VocabularyPath                   string
 118                 FreezeProjectRequiresDescription bool
 119                 FreezeProjectRequiresProperties  StringSet
 120                 UnfreezeProjectRequiresAdmin     bool
 121                 LockBeforeUpdate                 bool
 122         }
 123         AuditLogs struct {
 124                 MaxAge             Duration
 125                 MaxDeleteBatch     int
 126                 UnloggedAttributes StringSet
 127         }
 128         Collections struct {
 129                 BlobSigning                  bool
 130                 BlobSigningKey               string
 131                 BlobSigningTTL               Duration
 132                 BlobTrash                    bool
 133                 BlobTrashLifetime            Duration
 134                 BlobTrashCheckInterval       Duration
 135                 BlobTrashConcurrency         int
 136                 BlobDeleteConcurrency        int
 137                 BlobReplicateConcurrency     int
 138                 CollectionVersioning         bool
 139                 DefaultTrashLifetime         Duration
 140                 DefaultReplication           int
 141                 ManagedProperties            ManagedProperties
 142                 PreserveVersionIfIdle        Duration
 143                 TrashSweepInterval           Duration
 144                 TrustAllContent              bool
 145                 ForwardSlashNameSubstitution string
 146                 S3FolderObjects              bool
 147
 148                 BlobMissingReport        string
 149                 BalancePeriod            Duration
 150                 BalanceCollectionBatch   int
 151                 BalanceCollectionBuffers int
 152                 BalanceTimeout           Duration
 153                 BalanceUpdateLimit       int
 154                 BalancePullLimit         int
 155                 BalanceTrashLimit        int
 156
 157                 WebDAVCache WebDAVCacheConfig
 158
 159                 KeepproxyPermission UploadDownloadRolePermissions
 160                 WebDAVPermission    UploadDownloadRolePermissions
 161                 WebDAVLogEvents     bool
 162                 WebDAVOutputBuffer  ByteSize
 163         }
 164         Git struct {
 165                 GitCommand   string
 166                 GitoliteHome string
 167                 Repositories string
 168         }
 169         Login struct {
 170                 LDAP struct {
 171                         Enable             bool
 172                         URL                URL
 173                         StartTLS           bool
 174                         InsecureTLS        bool
 175                         MinTLSVersion      TLSVersion
 176                         StripDomain        string
 177                         AppendDomain       string
 178                         SearchAttribute    string
 179                         SearchBindUser     string
 180                         SearchBindPassword string
 181                         SearchBase         string
 182                         SearchFilters      string
 183                         EmailAttribute     string
 184                         UsernameAttribute  string
 185                 }
 186                 Google struct {
 187                         Enable                          bool
 188                         ClientID                        string
 189                         ClientSecret                    string
 190                         AlternateEmailAddresses         bool
 191                         AuthenticationRequestParameters map[string]string
 192                 }
 193                 OpenIDConnect struct {
 194                         Enable                          bool
 195                         Issuer                          string
 196                         ClientID                        string
 197                         ClientSecret                    string
 198                         EmailClaim                      string
 199                         EmailVerifiedClaim              string
 200                         UsernameClaim                   string
 201                         AcceptAccessToken               bool
 202                         AcceptAccessTokenScope          string
 203                         AuthenticationRequestParameters map[string]string
 204                 }
 205                 PAM struct {
 206                         Enable             bool
 207                         Service            string
 208                         DefaultEmailDomain string
 209                 }
 210                 Test struct {
 211                         Enable bool
 212                         Users  map[string]TestUser
 213                 }
 214                 LoginCluster         string
 215                 RemoteTokenRefresh   Duration
 216                 TokenLifetime        Duration
 217                 TrustedClients       map[URL]struct{}
 218                 TrustPrivateNetworks bool
 219                 IssueTrustedTokens   bool
 220         }
 221         Mail struct {
 222                 MailchimpAPIKey                string
 223                 MailchimpListID                string
 224                 SendUserSetupNotificationEmail bool
 225                 IssueReporterEmailFrom         string
 226                 IssueReporterEmailTo           string
 227                 SupportEmailAddress            string
 228                 EmailFrom                      string
 229         }
 230         SystemLogs struct {
 231                 LogLevel                  string
 232                 Format                    string
 233                 MaxRequestLogParamsSize   int
 234                 RequestQueueDumpDirectory string
 235         }
 236         TLS struct {
 237                 Certificate string
 238                 Key         string
 239                 Insecure    bool
 240                 ACME        struct {
 241                         Server string
 242                 }
 243         }
 244         Users struct {
 245                 ActivatedUsersAreVisibleToOthers      bool
 246                 AnonymousUserToken                    string
 247                 AdminNotifierEmailFrom                string
 248                 AutoAdminFirstUser                    bool
 249                 AutoAdminUserWithEmail                string
 250                 AutoSetupNewUsers                     bool
 251                 AutoSetupNewUsersWithRepository       bool
 252                 AutoSetupNewUsersWithVmUUID           string
 253                 AutoSetupUsernameBlacklist            StringSet
 254                 EmailSubjectPrefix                    string
 255                 NewInactiveUserNotificationRecipients StringSet
 256                 NewUserNotificationRecipients         StringSet
 257                 NewUsersAreActive                     bool
 258                 UserNotifierEmailFrom                 string
 259                 UserNotifierEmailBcc                  StringSet
 260                 UserProfileNotificationAddress        string
 261                 PreferDomainForUsername               string
 262                 UserSetupMailText                     string
 263                 RoleGroupsVisibleToAll                bool
 264                 CanCreateRoleGroups                   bool
 265                 ActivityLoggingPeriod                 Duration
 266                 SyncIgnoredGroups                     []string
 267                 SyncRequiredGroups                    []string
 268                 SyncUserAccounts                      bool
 269                 SyncUserAPITokens                     bool
 270                 SyncUserGroups                        bool
 271                 SyncUserSSHKeys                       bool
 272         }
 273         StorageClasses map[string]StorageClassConfig
 274         Volumes        map[string]Volume
 275         Workbench      struct {
 276                 ActivationContactLink   string
 277                 ArvadosDocsite          string
 278                 ArvadosPublicDataDocURL string
 279                 DisableSharingURLsUI    bool
 280                 FileViewersConfigURL    string
 281                 ShowUserAgreementInline bool
 282                 SiteName                string
 283                 Theme                   string
 284                 UserProfileFormFields   map[string]struct {
 285                         Type                 string
 286                         FormFieldTitle       string
 287                         FormFieldDescription string
 288                         Required             bool
 289                         Position             int
 290                         Options              map[string]struct{}
 291                 }
 292                 UserProfileFormMessage string
 293                 WelcomePageHTML        string
 294                 InactivePageHTML       string
 295                 SSHHelpPageHTML        string
 296                 SSHHelpHostSuffix      string
 297                 IdleTimeout            Duration
 298                 BannerUUID             string
 299         }
 300 }
 301
 302 type StorageClassConfig struct {
 303         Default  bool
 304         Priority int
 305 }
 306
 307 type Volume struct {
 308         AccessViaHosts         map[URL]VolumeAccess
 309         ReadOnly               bool
 310         AllowTrashWhenReadOnly bool
 311         Replication            int
 312         StorageClasses         map[string]bool
 313         Driver                 string
 314         DriverParameters       json.RawMessage
 315 }
 316
 317 type S3VolumeDriverParameters struct {
 318         IAMRole            string
 319         AccessKeyID        string
 320         SecretAccessKey    string
 321         Endpoint           string
 322         Region             string
 323         Bucket             string
 324         LocationConstraint bool
 325         V2Signature        bool
 326         IndexPageSize      int
 327         ConnectTimeout     Duration
 328         ReadTimeout        Duration
 329         RaceWindow         Duration
 330         UnsafeDelete       bool
 331         PrefixLength       int
 332 }
 333
 334 type AzureVolumeDriverParameters struct {
 335         StorageAccountName   string
 336         StorageAccountKey    string
 337         StorageBaseURL       string
 338         ContainerName        string
 339         RequestTimeout       Duration
 340         ListBlobsRetryDelay  Duration
 341         ListBlobsMaxAttempts int
 342 }
 343
 344 type DirectoryVolumeDriverParameters struct {
 345         Root      string
 346         Serialize bool
 347 }
 348
 349 type VolumeAccess struct {
 350         ReadOnly bool
 351 }
 352
 353 type Services struct {
 354         Composer       Service
 355         Controller     Service
 356         DispatchCloud  Service
 357         DispatchLSF    Service
 358         DispatchSLURM  Service
 359         GitHTTP        Service
 360         GitSSH         Service
 361         Health         Service
 362         Keepbalance    Service
 363         Keepproxy      Service
 364         Keepstore      Service
 365         RailsAPI       Service
 366         WebDAVDownload Service
 367         WebDAV         Service
 368         WebShell       Service
 369         Websocket      Service
 370         Workbench1     Service
 371         Workbench2     Service
 372 }
 373
 374 type Service struct {
 375         InternalURLs map[URL]ServiceInstance
 376         ExternalURL  URL
 377 }
 378
 379 type TestUser struct {
 380         Email    string
 381         Password string
 382 }
 383
 384 // URL is a url.URL that is also usable as a JSON key/value.
 385 type URL url.URL
 386
 387 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 388 // used as a JSON key/value.
 389 func (su *URL) UnmarshalText(text []byte) error {
 390         u, err := url.Parse(string(text))
 391         if err == nil {
 392                 *su = URL(*u)
 393                 if su.Path == "" && su.Host != "" {
 394                         // http://example really means http://example/
 395                         su.Path = "/"
 396                 }
 397         }
 398         return err
 399 }
 400
 401 func (su URL) MarshalText() ([]byte, error) {
 402         return []byte(su.String()), nil
 403 }
 404
 405 func (su URL) String() string {
 406         return (*url.URL)(&su).String()
 407 }
 408
 409 type TLSVersion uint16
 410
 411 func (v TLSVersion) MarshalText() ([]byte, error) {
 412         switch v {
 413         case 0:
 414                 return []byte{}, nil
 415         case tls.VersionTLS10:
 416                 return []byte("1.0"), nil
 417         case tls.VersionTLS11:
 418                 return []byte("1.1"), nil
 419         case tls.VersionTLS12:
 420                 return []byte("1.2"), nil
 421         case tls.VersionTLS13:
 422                 return []byte("1.3"), nil
 423         default:
 424                 return nil, fmt.Errorf("unsupported TLSVersion %x", v)
 425         }
 426 }
 427
 428 func (v *TLSVersion) UnmarshalJSON(text []byte) error {
 429         if len(text) > 0 && text[0] == '"' {
 430                 var s string
 431                 err := json.Unmarshal(text, &s)
 432                 if err != nil {
 433                         return err
 434                 }
 435                 text = []byte(s)
 436         }
 437         switch string(text) {
 438         case "":
 439                 *v = 0
 440         case "1.0":
 441                 *v = tls.VersionTLS10
 442         case "1.1":
 443                 *v = tls.VersionTLS11
 444         case "1.2":
 445                 *v = tls.VersionTLS12
 446         case "1.3":
 447                 *v = tls.VersionTLS13
 448         default:
 449                 return fmt.Errorf("unsupported TLSVersion %q", text)
 450         }
 451         return nil
 452 }
 453
 454 type ServiceInstance struct {
 455         ListenURL  URL
 456         Rendezvous string `json:",omitempty"`
 457 }
 458
 459 type PostgreSQL struct {
 460         Connection     PostgreSQLConnection
 461         ConnectionPool int
 462 }
 463
 464 type PostgreSQLConnection map[string]string
 465
 466 type RemoteCluster struct {
 467         Host          string
 468         Proxy         bool
 469         Scheme        string
 470         Insecure      bool
 471         ActivateUsers bool
 472 }
 473
 474 type CUDAFeatures struct {
 475         DriverVersion      string
 476         HardwareCapability string
 477         DeviceCount        int
 478 }
 479
 480 type InstanceType struct {
 481         Name            string `json:"-"`
 482         ProviderType    string
 483         VCPUs           int
 484         RAM             ByteSize
 485         Scratch         ByteSize `json:"-"`
 486         IncludedScratch ByteSize
 487         AddedScratch    ByteSize
 488         Price           float64
 489         Preemptible     bool
 490         CUDA            CUDAFeatures
 491 }
 492
 493 type ContainersConfig struct {
 494         CloudVMs                      CloudVMsConfig
 495         CrunchRunCommand              string
 496         CrunchRunArgumentsList        []string
 497         DefaultKeepCacheRAM           ByteSize
 498         DispatchPrivateKey            string
 499         LogReuseDecisions             bool
 500         MaxDispatchAttempts           int
 501         MaxRetryAttempts              int
 502         MinRetryPeriod                Duration
 503         ReserveExtraRAM               ByteSize
 504         StaleLockTimeout              Duration
 505         SupportedDockerImageFormats   StringSet
 506         AlwaysUsePreemptibleInstances bool
 507         PreemptiblePriceFactor        float64
 508         MaximumPriceFactor            float64
 509         RuntimeEngine                 string
 510         LocalKeepBlobBuffersPerVCPU   int
 511         LocalKeepLogsToContainerLog   string
 512
 513         JobsAPI struct {
 514                 Enable         string
 515                 GitInternalDir string
 516         }
 517         Logging struct {
 518                 MaxAge                       Duration
 519                 SweepInterval                Duration
 520                 LogBytesPerEvent             int
 521                 LogSecondsBetweenEvents      Duration
 522                 LogThrottlePeriod            Duration
 523                 LogThrottleBytes             int
 524                 LogThrottleLines             int
 525                 LimitLogBytesPerJob          int
 526                 LogPartialLineThrottlePeriod Duration
 527                 LogUpdatePeriod              Duration
 528                 LogUpdateSize                ByteSize
 529         }
 530         ShellAccess struct {
 531                 Admin bool
 532                 User  bool
 533         }
 534         SLURM struct {
 535                 PrioritySpread             int64
 536                 SbatchArgumentsList        []string
 537                 SbatchEnvironmentVariables map[string]string
 538                 Managed                    struct {
 539                         DNSServerConfDir       string
 540                         DNSServerConfTemplate  string
 541                         DNSServerReloadCommand string
 542                         DNSServerUpdateCommand string
 543                         ComputeNodeDomain      string
 544                         ComputeNodeNameservers StringSet
 545                         AssignNodeHostname     string
 546                 }
 547         }
 548         LSF struct {
 549                 BsubSudoUser       string
 550                 BsubArgumentsList  []string
 551                 BsubCUDAArguments  []string
 552                 MaxRunTimeOverhead Duration
 553                 MaxRunTimeDefault  Duration
 554         }
 555 }
 556
 557 type CloudVMsConfig struct {
 558         Enable bool
 559
 560         BootProbeCommand               string
 561         InstanceInitCommand            string
 562         DeployRunnerBinary             string
 563         DeployPublicKey                bool
 564         ImageID                        string
 565         MaxCloudOpsPerSecond           int
 566         MaxProbesPerSecond             int
 567         MaxConcurrentInstanceCreateOps int
 568         MaxInstances                   int
 569         InitialQuotaEstimate           int
 570         SupervisorFraction             float64
 571         PollInterval                   Duration
 572         ProbeInterval                  Duration
 573         SSHPort                        string
 574         SyncInterval                   Duration
 575         TimeoutBooting                 Duration
 576         TimeoutIdle                    Duration
 577         TimeoutProbe                   Duration
 578         TimeoutShutdown                Duration
 579         TimeoutSignal                  Duration
 580         TimeoutStaleRunLock            Duration
 581         TimeoutTERM                    Duration
 582         ResourceTags                   map[string]string
 583         TagKeyPrefix                   string
 584
 585         Driver           string
 586         DriverParameters json.RawMessage
 587 }
 588
 589 type InstanceTypeMap map[string]InstanceType
 590
 591 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 592
 593 // UnmarshalJSON does special handling of InstanceTypes:
 594 //
 595 // - populate computed fields (Name and Scratch)
 596 //
 597 // - error out if InstancesTypes are populated as an array, which was
 598 // deprecated in Arvados 1.2.0
 599 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 600         fixup := func(t InstanceType) (InstanceType, error) {
 601                 if t.ProviderType == "" {
 602                         t.ProviderType = t.Name
 603                 }
 604                 // If t.Scratch is set in the configuration file, it will be ignored and overwritten.
 605                 // It will also generate a "deprecated or unknown config entry" warning.
 606                 t.Scratch = t.IncludedScratch + t.AddedScratch
 607                 return t, nil
 608         }
 609
 610         if len(data) > 0 && data[0] == '[' {
 611                 return fmt.Errorf("InstanceTypes must be specified as a map, not an array, see https://doc.arvados.org/admin/config.html")
 612         }
 613         var hash map[string]InstanceType
 614         err := json.Unmarshal(data, &hash)
 615         if err != nil {
 616                 return err
 617         }
 618         // Fill in Name field (and ProviderType field, if not
 619         // specified) using hash key.
 620         *it = InstanceTypeMap(hash)
 621         for name, t := range *it {
 622                 t.Name = name
 623                 t, err := fixup(t)
 624                 if err != nil {
 625                         return err
 626                 }
 627                 (*it)[name] = t
 628         }
 629         return nil
 630 }
 631
 632 type StringSet map[string]struct{}
 633
 634 // UnmarshalJSON handles old config files that provide an array of
 635 // instance types instead of a hash.
 636 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 637         if len(data) > 0 && data[0] == '[' {
 638                 var arr []string
 639                 err := json.Unmarshal(data, &arr)
 640                 if err != nil {
 641                         return err
 642                 }
 643                 if len(arr) == 0 {
 644                         *ss = nil
 645                         return nil
 646                 }
 647                 *ss = make(map[string]struct{}, len(arr))
 648                 for _, t := range arr {
 649                         (*ss)[t] = struct{}{}
 650                 }
 651                 return nil
 652         }
 653         var hash map[string]struct{}
 654         err := json.Unmarshal(data, &hash)
 655         if err != nil {
 656                 return err
 657         }
 658         *ss = make(map[string]struct{}, len(hash))
 659         for t := range hash {
 660                 (*ss)[t] = struct{}{}
 661         }
 662
 663         return nil
 664 }
 665
 666 type ServiceName string
 667
 668 const (
 669         ServiceNameController    ServiceName = "arvados-controller"
 670         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 671         ServiceNameDispatchLSF   ServiceName = "arvados-dispatch-lsf"
 672         ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm"
 673         ServiceNameGitHTTP       ServiceName = "arvados-git-httpd"
 674         ServiceNameHealth        ServiceName = "arvados-health"
 675         ServiceNameKeepbalance   ServiceName = "keep-balance"
 676         ServiceNameKeepproxy     ServiceName = "keepproxy"
 677         ServiceNameKeepstore     ServiceName = "keepstore"
 678         ServiceNameKeepweb       ServiceName = "keep-web"
 679         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 680         ServiceNameWebsocket     ServiceName = "arvados-ws"
 681         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 682         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 683 )
 684
 685 // Map returns all services as a map, suitable for iterating over all
 686 // services or looking up a service by name.
 687 func (svcs Services) Map() map[ServiceName]Service {
 688         return map[ServiceName]Service{
 689                 ServiceNameController:    svcs.Controller,
 690                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 691                 ServiceNameDispatchLSF:   svcs.DispatchLSF,
 692                 ServiceNameDispatchSLURM: svcs.DispatchSLURM,
 693                 ServiceNameGitHTTP:       svcs.GitHTTP,
 694                 ServiceNameHealth:        svcs.Health,
 695                 ServiceNameKeepbalance:   svcs.Keepbalance,
 696                 ServiceNameKeepproxy:     svcs.Keepproxy,
 697                 ServiceNameKeepstore:     svcs.Keepstore,
 698                 ServiceNameKeepweb:       svcs.WebDAV,
 699                 ServiceNameRailsAPI:      svcs.RailsAPI,
 700                 ServiceNameWebsocket:     svcs.Websocket,
 701                 ServiceNameWorkbench1:    svcs.Workbench1,
 702                 ServiceNameWorkbench2:    svcs.Workbench2,
 703         }
 704 }