sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "crypto/tls"
   9         "encoding/json"
  10         "errors"
  11         "fmt"
  12         "net/url"
  13         "os"
  14         "time"
  15
  16         "git.arvados.org/arvados.git/sdk/go/config"
  17 )
  18
  19 var DefaultConfigFile = func() string {
  20         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  21                 return path
  22         }
  23         return "/etc/arvados/config.yml"
  24 }()
  25
  26 type Config struct {
  27         Clusters         map[string]Cluster
  28         AutoReloadConfig bool
  29         SourceTimestamp  time.Time
  30         SourceSHA256     string
  31 }
  32
  33 // GetConfig returns the current system config, loading it from
  34 // configFile if needed.
  35 func GetConfig(configFile string) (*Config, error) {
  36         var cfg Config
  37         err := config.LoadFile(&cfg, configFile)
  38         return &cfg, err
  39 }
  40
  41 // GetCluster returns the cluster ID and config for the given
  42 // cluster, or the default/only configured cluster if clusterID is "".
  43 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  44         if clusterID == "" {
  45                 if len(sc.Clusters) == 0 {
  46                         return nil, fmt.Errorf("no clusters configured")
  47                 } else if len(sc.Clusters) > 1 {
  48                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  49                 } else {
  50                         for id, cc := range sc.Clusters {
  51                                 cc.ClusterID = id
  52                                 return &cc, nil
  53                         }
  54                 }
  55         }
  56         cc, ok := sc.Clusters[clusterID]
  57         if !ok {
  58                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  59         }
  60         cc.ClusterID = clusterID
  61         return &cc, nil
  62 }
  63
  64 type WebDAVCacheConfig struct {
  65         TTL                Duration
  66         DiskCacheSize      ByteSizeOrPercent
  67         MaxCollectionBytes ByteSize
  68         MaxSessions        int
  69 }
  70
  71 type UploadDownloadPermission struct {
  72         Upload   bool
  73         Download bool
  74 }
  75
  76 type UploadDownloadRolePermissions struct {
  77         User  UploadDownloadPermission
  78         Admin UploadDownloadPermission
  79 }
  80
  81 type ManagedProperties map[string]struct {
  82         Value     interface{}
  83         Function  string
  84         Protected bool
  85 }
  86
  87 type Cluster struct {
  88         ClusterID       string `json:"-"`
  89         ManagementToken string
  90         SystemRootToken string
  91         Services        Services
  92         InstanceTypes   InstanceTypeMap
  93         Containers      ContainersConfig
  94         RemoteClusters  map[string]RemoteCluster
  95         PostgreSQL      PostgreSQL
  96
  97         API struct {
  98                 AsyncPermissionsUpdateInterval   Duration
  99                 DisabledAPIs                     StringSet
 100                 MaxIndexDatabaseRead             int
 101                 MaxItemsPerResponse              int
 102                 MaxConcurrentRailsRequests       int
 103                 MaxConcurrentRequests            int
 104                 MaxQueuedRequests                int
 105                 MaxGatewayTunnels                int
 106                 MaxQueueTimeForLockRequests      Duration
 107                 LogCreateRequestFraction         float64
 108                 MaxKeepBlobBuffers               int
 109                 MaxRequestAmplification          int
 110                 MaxRequestSize                   int
 111                 MaxTokenLifetime                 Duration
 112                 RequestTimeout                   Duration
 113                 SendTimeout                      Duration
 114                 WebsocketClientEventQueue        int
 115                 WebsocketServerEventQueue        int
 116                 KeepServiceRequestTimeout        Duration
 117                 VocabularyPath                   string
 118                 FreezeProjectRequiresDescription bool
 119                 FreezeProjectRequiresProperties  StringSet
 120                 UnfreezeProjectRequiresAdmin     bool
 121                 LockBeforeUpdate                 bool
 122         }
 123         AuditLogs struct {
 124                 MaxAge             Duration
 125                 MaxDeleteBatch     int
 126                 UnloggedAttributes StringSet
 127         }
 128         Collections struct {
 129                 BlobSigning                  bool
 130                 BlobSigningKey               string
 131                 BlobSigningTTL               Duration
 132                 BlobTrash                    bool
 133                 BlobTrashLifetime            Duration
 134                 BlobTrashCheckInterval       Duration
 135                 BlobTrashConcurrency         int
 136                 BlobDeleteConcurrency        int
 137                 BlobReplicateConcurrency     int
 138                 CollectionVersioning         bool
 139                 DefaultTrashLifetime         Duration
 140                 DefaultReplication           int
 141                 ManagedProperties            ManagedProperties
 142                 PreserveVersionIfIdle        Duration
 143                 TrashSweepInterval           Duration
 144                 TrustAllContent              bool
 145                 ForwardSlashNameSubstitution string
 146                 S3FolderObjects              bool
 147
 148                 BlobMissingReport        string
 149                 BalancePeriod            Duration
 150                 BalanceCollectionBatch   int
 151                 BalanceCollectionBuffers int
 152                 BalanceTimeout           Duration
 153                 BalanceUpdateLimit       int
 154                 BalancePullLimit         int
 155                 BalanceTrashLimit        int
 156
 157                 WebDAVCache WebDAVCacheConfig
 158
 159                 KeepproxyPermission UploadDownloadRolePermissions
 160                 WebDAVPermission    UploadDownloadRolePermissions
 161                 WebDAVLogEvents     bool
 162         }
 163         Git struct {
 164                 GitCommand   string
 165                 GitoliteHome string
 166                 Repositories string
 167         }
 168         Login struct {
 169                 LDAP struct {
 170                         Enable             bool
 171                         URL                URL
 172                         StartTLS           bool
 173                         InsecureTLS        bool
 174                         MinTLSVersion      TLSVersion
 175                         StripDomain        string
 176                         AppendDomain       string
 177                         SearchAttribute    string
 178                         SearchBindUser     string
 179                         SearchBindPassword string
 180                         SearchBase         string
 181                         SearchFilters      string
 182                         EmailAttribute     string
 183                         UsernameAttribute  string
 184                 }
 185                 Google struct {
 186                         Enable                          bool
 187                         ClientID                        string
 188                         ClientSecret                    string
 189                         AlternateEmailAddresses         bool
 190                         AuthenticationRequestParameters map[string]string
 191                 }
 192                 OpenIDConnect struct {
 193                         Enable                          bool
 194                         Issuer                          string
 195                         ClientID                        string
 196                         ClientSecret                    string
 197                         EmailClaim                      string
 198                         EmailVerifiedClaim              string
 199                         UsernameClaim                   string
 200                         AcceptAccessToken               bool
 201                         AcceptAccessTokenScope          string
 202                         AuthenticationRequestParameters map[string]string
 203                 }
 204                 PAM struct {
 205                         Enable             bool
 206                         Service            string
 207                         DefaultEmailDomain string
 208                 }
 209                 Test struct {
 210                         Enable bool
 211                         Users  map[string]TestUser
 212                 }
 213                 LoginCluster         string
 214                 RemoteTokenRefresh   Duration
 215                 TokenLifetime        Duration
 216                 TrustedClients       map[URL]struct{}
 217                 TrustPrivateNetworks bool
 218                 IssueTrustedTokens   bool
 219         }
 220         Mail struct {
 221                 MailchimpAPIKey                string
 222                 MailchimpListID                string
 223                 SendUserSetupNotificationEmail bool
 224                 IssueReporterEmailFrom         string
 225                 IssueReporterEmailTo           string
 226                 SupportEmailAddress            string
 227                 EmailFrom                      string
 228         }
 229         SystemLogs struct {
 230                 LogLevel                  string
 231                 Format                    string
 232                 MaxRequestLogParamsSize   int
 233                 RequestQueueDumpDirectory string
 234         }
 235         TLS struct {
 236                 Certificate string
 237                 Key         string
 238                 Insecure    bool
 239                 ACME        struct {
 240                         Server string
 241                 }
 242         }
 243         Users struct {
 244                 ActivatedUsersAreVisibleToOthers      bool
 245                 AnonymousUserToken                    string
 246                 AdminNotifierEmailFrom                string
 247                 AutoAdminFirstUser                    bool
 248                 AutoAdminUserWithEmail                string
 249                 AutoSetupNewUsers                     bool
 250                 AutoSetupNewUsersWithRepository       bool
 251                 AutoSetupNewUsersWithVmUUID           string
 252                 AutoSetupUsernameBlacklist            StringSet
 253                 EmailSubjectPrefix                    string
 254                 NewInactiveUserNotificationRecipients StringSet
 255                 NewUserNotificationRecipients         StringSet
 256                 NewUsersAreActive                     bool
 257                 UserNotifierEmailFrom                 string
 258                 UserNotifierEmailBcc                  StringSet
 259                 UserProfileNotificationAddress        string
 260                 PreferDomainForUsername               string
 261                 UserSetupMailText                     string
 262                 RoleGroupsVisibleToAll                bool
 263                 CanCreateRoleGroups                   bool
 264                 ActivityLoggingPeriod                 Duration
 265                 SyncIgnoredGroups                     []string
 266                 SyncRequiredGroups                    []string
 267                 SyncUserAccounts                      bool
 268                 SyncUserAPITokens                     bool
 269                 SyncUserGroups                        bool
 270                 SyncUserSSHKeys                       bool
 271         }
 272         StorageClasses map[string]StorageClassConfig
 273         Volumes        map[string]Volume
 274         Workbench      struct {
 275                 ActivationContactLink   string
 276                 ArvadosDocsite          string
 277                 ArvadosPublicDataDocURL string
 278                 DisableSharingURLsUI    bool
 279                 FileViewersConfigURL    string
 280                 ShowUserAgreementInline bool
 281                 SiteName                string
 282                 Theme                   string
 283                 UserProfileFormFields   map[string]struct {
 284                         Type                 string
 285                         FormFieldTitle       string
 286                         FormFieldDescription string
 287                         Required             bool
 288                         Position             int
 289                         Options              map[string]struct{}
 290                 }
 291                 UserProfileFormMessage string
 292                 WelcomePageHTML        string
 293                 InactivePageHTML       string
 294                 SSHHelpPageHTML        string
 295                 SSHHelpHostSuffix      string
 296                 IdleTimeout            Duration
 297                 BannerUUID             string
 298         }
 299 }
 300
 301 type StorageClassConfig struct {
 302         Default  bool
 303         Priority int
 304 }
 305
 306 type Volume struct {
 307         AccessViaHosts         map[URL]VolumeAccess
 308         ReadOnly               bool
 309         AllowTrashWhenReadOnly bool
 310         Replication            int
 311         StorageClasses         map[string]bool
 312         Driver                 string
 313         DriverParameters       json.RawMessage
 314 }
 315
 316 type S3VolumeDriverParameters struct {
 317         IAMRole            string
 318         AccessKeyID        string
 319         SecretAccessKey    string
 320         Endpoint           string
 321         Region             string
 322         Bucket             string
 323         LocationConstraint bool
 324         V2Signature        bool
 325         IndexPageSize      int
 326         ConnectTimeout     Duration
 327         ReadTimeout        Duration
 328         RaceWindow         Duration
 329         UnsafeDelete       bool
 330         PrefixLength       int
 331 }
 332
 333 type AzureVolumeDriverParameters struct {
 334         StorageAccountName   string
 335         StorageAccountKey    string
 336         StorageBaseURL       string
 337         ContainerName        string
 338         RequestTimeout       Duration
 339         ListBlobsRetryDelay  Duration
 340         ListBlobsMaxAttempts int
 341 }
 342
 343 type DirectoryVolumeDriverParameters struct {
 344         Root      string
 345         Serialize bool
 346 }
 347
 348 type VolumeAccess struct {
 349         ReadOnly bool
 350 }
 351
 352 type Services struct {
 353         Composer       Service
 354         Controller     Service
 355         DispatchCloud  Service
 356         DispatchLSF    Service
 357         DispatchSLURM  Service
 358         GitHTTP        Service
 359         GitSSH         Service
 360         Health         Service
 361         Keepbalance    Service
 362         Keepproxy      Service
 363         Keepstore      Service
 364         RailsAPI       Service
 365         WebDAVDownload Service
 366         WebDAV         Service
 367         WebShell       Service
 368         Websocket      Service
 369         Workbench1     Service
 370         Workbench2     Service
 371 }
 372
 373 type Service struct {
 374         InternalURLs map[URL]ServiceInstance
 375         ExternalURL  URL
 376 }
 377
 378 type TestUser struct {
 379         Email    string
 380         Password string
 381 }
 382
 383 // URL is a url.URL that is also usable as a JSON key/value.
 384 type URL url.URL
 385
 386 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 387 // used as a JSON key/value.
 388 func (su *URL) UnmarshalText(text []byte) error {
 389         u, err := url.Parse(string(text))
 390         if err == nil {
 391                 *su = URL(*u)
 392                 if su.Path == "" && su.Host != "" {
 393                         // http://example really means http://example/
 394                         su.Path = "/"
 395                 }
 396         }
 397         return err
 398 }
 399
 400 func (su URL) MarshalText() ([]byte, error) {
 401         return []byte(su.String()), nil
 402 }
 403
 404 func (su URL) String() string {
 405         return (*url.URL)(&su).String()
 406 }
 407
 408 type TLSVersion uint16
 409
 410 func (v TLSVersion) MarshalText() ([]byte, error) {
 411         switch v {
 412         case 0:
 413                 return []byte{}, nil
 414         case tls.VersionTLS10:
 415                 return []byte("1.0"), nil
 416         case tls.VersionTLS11:
 417                 return []byte("1.1"), nil
 418         case tls.VersionTLS12:
 419                 return []byte("1.2"), nil
 420         case tls.VersionTLS13:
 421                 return []byte("1.3"), nil
 422         default:
 423                 return nil, fmt.Errorf("unsupported TLSVersion %x", v)
 424         }
 425 }
 426
 427 func (v *TLSVersion) UnmarshalJSON(text []byte) error {
 428         if len(text) > 0 && text[0] == '"' {
 429                 var s string
 430                 err := json.Unmarshal(text, &s)
 431                 if err != nil {
 432                         return err
 433                 }
 434                 text = []byte(s)
 435         }
 436         switch string(text) {
 437         case "":
 438                 *v = 0
 439         case "1.0":
 440                 *v = tls.VersionTLS10
 441         case "1.1":
 442                 *v = tls.VersionTLS11
 443         case "1.2":
 444                 *v = tls.VersionTLS12
 445         case "1.3":
 446                 *v = tls.VersionTLS13
 447         default:
 448                 return fmt.Errorf("unsupported TLSVersion %q", text)
 449         }
 450         return nil
 451 }
 452
 453 type ServiceInstance struct {
 454         ListenURL  URL
 455         Rendezvous string `json:",omitempty"`
 456 }
 457
 458 type PostgreSQL struct {
 459         Connection     PostgreSQLConnection
 460         ConnectionPool int
 461 }
 462
 463 type PostgreSQLConnection map[string]string
 464
 465 type RemoteCluster struct {
 466         Host          string
 467         Proxy         bool
 468         Scheme        string
 469         Insecure      bool
 470         ActivateUsers bool
 471 }
 472
 473 type CUDAFeatures struct {
 474         DriverVersion      string
 475         HardwareCapability string
 476         DeviceCount        int
 477 }
 478
 479 type InstanceType struct {
 480         Name            string `json:"-"`
 481         ProviderType    string
 482         VCPUs           int
 483         RAM             ByteSize
 484         Scratch         ByteSize `json:"-"`
 485         IncludedScratch ByteSize
 486         AddedScratch    ByteSize
 487         Price           float64
 488         Preemptible     bool
 489         CUDA            CUDAFeatures
 490 }
 491
 492 type ContainersConfig struct {
 493         CloudVMs                      CloudVMsConfig
 494         CrunchRunCommand              string
 495         CrunchRunArgumentsList        []string
 496         DefaultKeepCacheRAM           ByteSize
 497         DispatchPrivateKey            string
 498         LogReuseDecisions             bool
 499         MaxDispatchAttempts           int
 500         MaxRetryAttempts              int
 501         MinRetryPeriod                Duration
 502         ReserveExtraRAM               ByteSize
 503         StaleLockTimeout              Duration
 504         SupportedDockerImageFormats   StringSet
 505         AlwaysUsePreemptibleInstances bool
 506         PreemptiblePriceFactor        float64
 507         MaximumPriceFactor            float64
 508         RuntimeEngine                 string
 509         LocalKeepBlobBuffersPerVCPU   int
 510         LocalKeepLogsToContainerLog   string
 511
 512         JobsAPI struct {
 513                 Enable         string
 514                 GitInternalDir string
 515         }
 516         Logging struct {
 517                 MaxAge                       Duration
 518                 SweepInterval                Duration
 519                 LogBytesPerEvent             int
 520                 LogSecondsBetweenEvents      Duration
 521                 LogThrottlePeriod            Duration
 522                 LogThrottleBytes             int
 523                 LogThrottleLines             int
 524                 LimitLogBytesPerJob          int
 525                 LogPartialLineThrottlePeriod Duration
 526                 LogUpdatePeriod              Duration
 527                 LogUpdateSize                ByteSize
 528         }
 529         ShellAccess struct {
 530                 Admin bool
 531                 User  bool
 532         }
 533         SLURM struct {
 534                 PrioritySpread             int64
 535                 SbatchArgumentsList        []string
 536                 SbatchEnvironmentVariables map[string]string
 537                 Managed                    struct {
 538                         DNSServerConfDir       string
 539                         DNSServerConfTemplate  string
 540                         DNSServerReloadCommand string
 541                         DNSServerUpdateCommand string
 542                         ComputeNodeDomain      string
 543                         ComputeNodeNameservers StringSet
 544                         AssignNodeHostname     string
 545                 }
 546         }
 547         LSF struct {
 548                 BsubSudoUser       string
 549                 BsubArgumentsList  []string
 550                 BsubCUDAArguments  []string
 551                 MaxRunTimeOverhead Duration
 552                 MaxRunTimeDefault  Duration
 553         }
 554 }
 555
 556 type CloudVMsConfig struct {
 557         Enable bool
 558
 559         BootProbeCommand               string
 560         InstanceInitCommand            string
 561         DeployRunnerBinary             string
 562         DeployPublicKey                bool
 563         ImageID                        string
 564         MaxCloudOpsPerSecond           int
 565         MaxProbesPerSecond             int
 566         MaxConcurrentInstanceCreateOps int
 567         MaxInstances                   int
 568         InitialQuotaEstimate           int
 569         SupervisorFraction             float64
 570         PollInterval                   Duration
 571         ProbeInterval                  Duration
 572         SSHPort                        string
 573         SyncInterval                   Duration
 574         TimeoutBooting                 Duration
 575         TimeoutIdle                    Duration
 576         TimeoutProbe                   Duration
 577         TimeoutShutdown                Duration
 578         TimeoutSignal                  Duration
 579         TimeoutStaleRunLock            Duration
 580         TimeoutTERM                    Duration
 581         ResourceTags                   map[string]string
 582         TagKeyPrefix                   string
 583
 584         Driver           string
 585         DriverParameters json.RawMessage
 586 }
 587
 588 type InstanceTypeMap map[string]InstanceType
 589
 590 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 591
 592 // UnmarshalJSON does special handling of InstanceTypes:
 593 //
 594 // - populate computed fields (Name and Scratch)
 595 //
 596 // - error out if InstancesTypes are populated as an array, which was
 597 // deprecated in Arvados 1.2.0
 598 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 599         fixup := func(t InstanceType) (InstanceType, error) {
 600                 if t.ProviderType == "" {
 601                         t.ProviderType = t.Name
 602                 }
 603                 // If t.Scratch is set in the configuration file, it will be ignored and overwritten.
 604                 // It will also generate a "deprecated or unknown config entry" warning.
 605                 t.Scratch = t.IncludedScratch + t.AddedScratch
 606                 return t, nil
 607         }
 608
 609         if len(data) > 0 && data[0] == '[' {
 610                 return fmt.Errorf("InstanceTypes must be specified as a map, not an array, see https://doc.arvados.org/admin/config.html")
 611         }
 612         var hash map[string]InstanceType
 613         err := json.Unmarshal(data, &hash)
 614         if err != nil {
 615                 return err
 616         }
 617         // Fill in Name field (and ProviderType field, if not
 618         // specified) using hash key.
 619         *it = InstanceTypeMap(hash)
 620         for name, t := range *it {
 621                 t.Name = name
 622                 t, err := fixup(t)
 623                 if err != nil {
 624                         return err
 625                 }
 626                 (*it)[name] = t
 627         }
 628         return nil
 629 }
 630
 631 type StringSet map[string]struct{}
 632
 633 // UnmarshalJSON handles old config files that provide an array of
 634 // instance types instead of a hash.
 635 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 636         if len(data) > 0 && data[0] == '[' {
 637                 var arr []string
 638                 err := json.Unmarshal(data, &arr)
 639                 if err != nil {
 640                         return err
 641                 }
 642                 if len(arr) == 0 {
 643                         *ss = nil
 644                         return nil
 645                 }
 646                 *ss = make(map[string]struct{}, len(arr))
 647                 for _, t := range arr {
 648                         (*ss)[t] = struct{}{}
 649                 }
 650                 return nil
 651         }
 652         var hash map[string]struct{}
 653         err := json.Unmarshal(data, &hash)
 654         if err != nil {
 655                 return err
 656         }
 657         *ss = make(map[string]struct{}, len(hash))
 658         for t := range hash {
 659                 (*ss)[t] = struct{}{}
 660         }
 661
 662         return nil
 663 }
 664
 665 type ServiceName string
 666
 667 const (
 668         ServiceNameController    ServiceName = "arvados-controller"
 669         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 670         ServiceNameDispatchLSF   ServiceName = "arvados-dispatch-lsf"
 671         ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm"
 672         ServiceNameGitHTTP       ServiceName = "arvados-git-httpd"
 673         ServiceNameHealth        ServiceName = "arvados-health"
 674         ServiceNameKeepbalance   ServiceName = "keep-balance"
 675         ServiceNameKeepproxy     ServiceName = "keepproxy"
 676         ServiceNameKeepstore     ServiceName = "keepstore"
 677         ServiceNameKeepweb       ServiceName = "keep-web"
 678         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 679         ServiceNameWebsocket     ServiceName = "arvados-ws"
 680         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 681         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 682 )
 683
 684 // Map returns all services as a map, suitable for iterating over all
 685 // services or looking up a service by name.
 686 func (svcs Services) Map() map[ServiceName]Service {
 687         return map[ServiceName]Service{
 688                 ServiceNameController:    svcs.Controller,
 689                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 690                 ServiceNameDispatchLSF:   svcs.DispatchLSF,
 691                 ServiceNameDispatchSLURM: svcs.DispatchSLURM,
 692                 ServiceNameGitHTTP:       svcs.GitHTTP,
 693                 ServiceNameHealth:        svcs.Health,
 694                 ServiceNameKeepbalance:   svcs.Keepbalance,
 695                 ServiceNameKeepproxy:     svcs.Keepproxy,
 696                 ServiceNameKeepstore:     svcs.Keepstore,
 697                 ServiceNameKeepweb:       svcs.WebDAV,
 698                 ServiceNameRailsAPI:      svcs.RailsAPI,
 699                 ServiceNameWebsocket:     svcs.Websocket,
 700                 ServiceNameWorkbench1:    svcs.Workbench1,
 701                 ServiceNameWorkbench2:    svcs.Workbench2,
 702         }
 703 }