sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "crypto/tls"
   9         "encoding/json"
  10         "errors"
  11         "fmt"
  12         "net/url"
  13         "os"
  14         "time"
  15
  16         "git.arvados.org/arvados.git/sdk/go/config"
  17 )
  18
  19 var DefaultConfigFile = func() string {
  20         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  21                 return path
  22         }
  23         return "/etc/arvados/config.yml"
  24 }()
  25
  26 type Config struct {
  27         Clusters         map[string]Cluster
  28         AutoReloadConfig bool
  29         SourceTimestamp  time.Time
  30         SourceSHA256     string
  31 }
  32
  33 // GetConfig returns the current system config, loading it from
  34 // configFile if needed.
  35 func GetConfig(configFile string) (*Config, error) {
  36         var cfg Config
  37         err := config.LoadFile(&cfg, configFile)
  38         return &cfg, err
  39 }
  40
  41 // GetCluster returns the cluster ID and config for the given
  42 // cluster, or the default/only configured cluster if clusterID is "".
  43 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  44         if clusterID == "" {
  45                 if len(sc.Clusters) == 0 {
  46                         return nil, fmt.Errorf("no clusters configured")
  47                 } else if len(sc.Clusters) > 1 {
  48                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  49                 } else {
  50                         for id, cc := range sc.Clusters {
  51                                 cc.ClusterID = id
  52                                 return &cc, nil
  53                         }
  54                 }
  55         }
  56         cc, ok := sc.Clusters[clusterID]
  57         if !ok {
  58                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  59         }
  60         cc.ClusterID = clusterID
  61         return &cc, nil
  62 }
  63
  64 type WebDAVCacheConfig struct {
  65         TTL                Duration
  66         DiskCacheSize      ByteSizeOrPercent
  67         MaxCollectionBytes ByteSize
  68         MaxSessions        int
  69 }
  70
  71 type UploadDownloadPermission struct {
  72         Upload   bool
  73         Download bool
  74 }
  75
  76 type UploadDownloadRolePermissions struct {
  77         User  UploadDownloadPermission
  78         Admin UploadDownloadPermission
  79 }
  80
  81 type ManagedProperties map[string]struct {
  82         Value     interface{}
  83         Function  string
  84         Protected bool
  85 }
  86
  87 type Cluster struct {
  88         ClusterID       string `json:"-"`
  89         ManagementToken string
  90         SystemRootToken string
  91         Services        Services
  92         InstanceTypes   InstanceTypeMap
  93         Containers      ContainersConfig
  94         RemoteClusters  map[string]RemoteCluster
  95         PostgreSQL      PostgreSQL
  96
  97         API struct {
  98                 AsyncPermissionsUpdateInterval   Duration
  99                 DisabledAPIs                     StringSet
 100                 MaxIndexDatabaseRead             int
 101                 MaxItemsPerResponse              int
 102                 MaxConcurrentRailsRequests       int
 103                 MaxConcurrentRequests            int
 104                 MaxQueuedRequests                int
 105                 MaxGatewayTunnels                int
 106                 MaxQueueTimeForLockRequests      Duration
 107                 LogCreateRequestFraction         float64
 108                 MaxKeepBlobBuffers               int
 109                 MaxRequestAmplification          int
 110                 MaxRequestSize                   int
 111                 MaxTokenLifetime                 Duration
 112                 RequestTimeout                   Duration
 113                 SendTimeout                      Duration
 114                 WebsocketClientEventQueue        int
 115                 WebsocketServerEventQueue        int
 116                 KeepServiceRequestTimeout        Duration
 117                 VocabularyPath                   string
 118                 FreezeProjectRequiresDescription bool
 119                 FreezeProjectRequiresProperties  StringSet
 120                 UnfreezeProjectRequiresAdmin     bool
 121                 LockBeforeUpdate                 bool
 122         }
 123         AuditLogs struct {
 124                 MaxAge             Duration
 125                 MaxDeleteBatch     int
 126                 UnloggedAttributes StringSet
 127         }
 128         Collections struct {
 129                 BlobSigning                  bool
 130                 BlobSigningKey               string
 131                 BlobSigningTTL               Duration
 132                 BlobTrash                    bool
 133                 BlobTrashLifetime            Duration
 134                 BlobTrashCheckInterval       Duration
 135                 BlobTrashConcurrency         int
 136                 BlobDeleteConcurrency        int
 137                 BlobReplicateConcurrency     int
 138                 CollectionVersioning         bool
 139                 DefaultTrashLifetime         Duration
 140                 DefaultReplication           int
 141                 ManagedProperties            ManagedProperties
 142                 PreserveVersionIfIdle        Duration
 143                 TrashSweepInterval           Duration
 144                 TrustAllContent              bool
 145                 ForwardSlashNameSubstitution string
 146                 S3FolderObjects              bool
 147
 148                 BlobMissingReport        string
 149                 BalancePeriod            Duration
 150                 BalanceCollectionBatch   int
 151                 BalanceCollectionBuffers int
 152                 BalanceTimeout           Duration
 153                 BalanceUpdateLimit       int
 154                 BalancePullLimit         int
 155                 BalanceTrashLimit        int
 156
 157                 WebDAVCache WebDAVCacheConfig
 158
 159                 KeepproxyPermission UploadDownloadRolePermissions
 160                 WebDAVPermission    UploadDownloadRolePermissions
 161                 WebDAVLogEvents     bool
 162                 WebDAVOutputBuffer  ByteSize
 163         }
 164         Login struct {
 165                 LDAP struct {
 166                         Enable             bool
 167                         URL                URL
 168                         StartTLS           bool
 169                         InsecureTLS        bool
 170                         MinTLSVersion      TLSVersion
 171                         StripDomain        string
 172                         AppendDomain       string
 173                         SearchAttribute    string
 174                         SearchBindUser     string
 175                         SearchBindPassword string
 176                         SearchBase         string
 177                         SearchFilters      string
 178                         EmailAttribute     string
 179                         UsernameAttribute  string
 180                 }
 181                 Google struct {
 182                         Enable                          bool
 183                         ClientID                        string
 184                         ClientSecret                    string
 185                         AlternateEmailAddresses         bool
 186                         AuthenticationRequestParameters map[string]string
 187                 }
 188                 OpenIDConnect struct {
 189                         Enable                          bool
 190                         Issuer                          string
 191                         ClientID                        string
 192                         ClientSecret                    string
 193                         EmailClaim                      string
 194                         EmailVerifiedClaim              string
 195                         UsernameClaim                   string
 196                         AcceptAccessToken               bool
 197                         AcceptAccessTokenScope          string
 198                         AuthenticationRequestParameters map[string]string
 199                 }
 200                 PAM struct {
 201                         Enable             bool
 202                         Service            string
 203                         DefaultEmailDomain string
 204                 }
 205                 Test struct {
 206                         Enable bool
 207                         Users  map[string]TestUser
 208                 }
 209                 LoginCluster         string
 210                 RemoteTokenRefresh   Duration
 211                 TokenLifetime        Duration
 212                 TrustedClients       map[URL]struct{}
 213                 TrustPrivateNetworks bool
 214                 IssueTrustedTokens   bool
 215         }
 216         Mail struct {
 217                 MailchimpAPIKey                string
 218                 MailchimpListID                string
 219                 SendUserSetupNotificationEmail bool
 220                 IssueReporterEmailFrom         string
 221                 IssueReporterEmailTo           string
 222                 SupportEmailAddress            string
 223                 EmailFrom                      string
 224         }
 225         SystemLogs struct {
 226                 LogLevel                  string
 227                 Format                    string
 228                 MaxRequestLogParamsSize   int
 229                 RequestQueueDumpDirectory string
 230         }
 231         TLS struct {
 232                 Certificate string
 233                 Key         string
 234                 Insecure    bool
 235                 ACME        struct {
 236                         Server string
 237                 }
 238         }
 239         Users struct {
 240                 ActivatedUsersAreVisibleToOthers      bool
 241                 AnonymousUserToken                    string
 242                 AdminNotifierEmailFrom                string
 243                 AutoAdminFirstUser                    bool
 244                 AutoAdminUserWithEmail                string
 245                 AutoSetupNewUsers                     bool
 246                 AutoSetupNewUsersWithVmUUID           string
 247                 AutoSetupUsernameBlacklist            StringSet
 248                 EmailSubjectPrefix                    string
 249                 NewInactiveUserNotificationRecipients StringSet
 250                 NewUserNotificationRecipients         StringSet
 251                 NewUsersAreActive                     bool
 252                 UserNotifierEmailFrom                 string
 253                 UserNotifierEmailBcc                  StringSet
 254                 UserProfileNotificationAddress        string
 255                 PreferDomainForUsername               string
 256                 UserSetupMailText                     string
 257                 RoleGroupsVisibleToAll                bool
 258                 CanCreateRoleGroups                   bool
 259                 ActivityLoggingPeriod                 Duration
 260                 SyncIgnoredGroups                     []string
 261                 SyncRequiredGroups                    []string
 262                 SyncUserAccounts                      bool
 263                 SyncUserAPITokens                     bool
 264                 SyncUserGroups                        bool
 265                 SyncUserSSHKeys                       bool
 266         }
 267         StorageClasses map[string]StorageClassConfig
 268         Volumes        map[string]Volume
 269         Workbench      struct {
 270                 ActivationContactLink   string
 271                 ArvadosDocsite          string
 272                 ArvadosPublicDataDocURL string
 273                 DisableSharingURLsUI    bool
 274                 FileViewersConfigURL    string
 275                 ShowUserAgreementInline bool
 276                 SiteName                string
 277                 Theme                   string
 278                 UserProfileFormFields   map[string]struct {
 279                         Type                 string
 280                         FormFieldTitle       string
 281                         FormFieldDescription string
 282                         Required             bool
 283                         Position             int
 284                         Options              map[string]struct{}
 285                 }
 286                 UserProfileFormMessage string
 287                 WelcomePageHTML        string
 288                 InactivePageHTML       string
 289                 SSHHelpPageHTML        string
 290                 SSHHelpHostSuffix      string
 291                 IdleTimeout            Duration
 292                 BannerUUID             string
 293         }
 294 }
 295
 296 type StorageClassConfig struct {
 297         Default  bool
 298         Priority int
 299 }
 300
 301 type Volume struct {
 302         AccessViaHosts         map[URL]VolumeAccess
 303         ReadOnly               bool
 304         AllowTrashWhenReadOnly bool
 305         Replication            int
 306         StorageClasses         map[string]bool
 307         Driver                 string
 308         DriverParameters       json.RawMessage
 309 }
 310
 311 type S3VolumeDriverParameters struct {
 312         IAMRole            string
 313         AccessKeyID        string
 314         SecretAccessKey    string
 315         Endpoint           string
 316         Region             string
 317         Bucket             string
 318         LocationConstraint bool
 319         V2Signature        bool
 320         IndexPageSize      int
 321         ConnectTimeout     Duration
 322         ReadTimeout        Duration
 323         RaceWindow         Duration
 324         UnsafeDelete       bool
 325         PrefixLength       int
 326 }
 327
 328 type AzureVolumeDriverParameters struct {
 329         StorageAccountName   string
 330         StorageAccountKey    string
 331         StorageBaseURL       string
 332         ContainerName        string
 333         RequestTimeout       Duration
 334         ListBlobsRetryDelay  Duration
 335         ListBlobsMaxAttempts int
 336 }
 337
 338 type DirectoryVolumeDriverParameters struct {
 339         Root      string
 340         Serialize bool
 341 }
 342
 343 type VolumeAccess struct {
 344         ReadOnly bool
 345 }
 346
 347 type Services struct {
 348         Composer       Service
 349         Controller     Service
 350         DispatchCloud  Service
 351         DispatchLSF    Service
 352         DispatchSLURM  Service
 353         Health         Service
 354         Keepbalance    Service
 355         Keepproxy      Service
 356         Keepstore      Service
 357         RailsAPI       Service
 358         WebDAVDownload Service
 359         WebDAV         Service
 360         WebShell       Service
 361         Websocket      Service
 362         Workbench1     Service
 363         Workbench2     Service
 364 }
 365
 366 type Service struct {
 367         InternalURLs map[URL]ServiceInstance
 368         ExternalURL  URL
 369 }
 370
 371 type TestUser struct {
 372         Email    string
 373         Password string
 374 }
 375
 376 // URL is a url.URL that is also usable as a JSON key/value.
 377 type URL url.URL
 378
 379 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 380 // used as a JSON key/value.
 381 func (su *URL) UnmarshalText(text []byte) error {
 382         u, err := url.Parse(string(text))
 383         if err == nil {
 384                 *su = URL(*u)
 385                 if su.Path == "" && su.Host != "" {
 386                         // http://example really means http://example/
 387                         su.Path = "/"
 388                 }
 389         }
 390         return err
 391 }
 392
 393 func (su URL) MarshalText() ([]byte, error) {
 394         return []byte(su.String()), nil
 395 }
 396
 397 func (su URL) String() string {
 398         return (*url.URL)(&su).String()
 399 }
 400
 401 type TLSVersion uint16
 402
 403 func (v TLSVersion) MarshalText() ([]byte, error) {
 404         switch v {
 405         case 0:
 406                 return []byte{}, nil
 407         case tls.VersionTLS10:
 408                 return []byte("1.0"), nil
 409         case tls.VersionTLS11:
 410                 return []byte("1.1"), nil
 411         case tls.VersionTLS12:
 412                 return []byte("1.2"), nil
 413         case tls.VersionTLS13:
 414                 return []byte("1.3"), nil
 415         default:
 416                 return nil, fmt.Errorf("unsupported TLSVersion %x", v)
 417         }
 418 }
 419
 420 func (v *TLSVersion) UnmarshalJSON(text []byte) error {
 421         if len(text) > 0 && text[0] == '"' {
 422                 var s string
 423                 err := json.Unmarshal(text, &s)
 424                 if err != nil {
 425                         return err
 426                 }
 427                 text = []byte(s)
 428         }
 429         switch string(text) {
 430         case "":
 431                 *v = 0
 432         case "1.0":
 433                 *v = tls.VersionTLS10
 434         case "1.1":
 435                 *v = tls.VersionTLS11
 436         case "1.2":
 437                 *v = tls.VersionTLS12
 438         case "1.3":
 439                 *v = tls.VersionTLS13
 440         default:
 441                 return fmt.Errorf("unsupported TLSVersion %q", text)
 442         }
 443         return nil
 444 }
 445
 446 type ServiceInstance struct {
 447         ListenURL  URL
 448         Rendezvous string `json:",omitempty"`
 449 }
 450
 451 type PostgreSQL struct {
 452         Connection     PostgreSQLConnection
 453         ConnectionPool int
 454 }
 455
 456 type PostgreSQLConnection map[string]string
 457
 458 type RemoteCluster struct {
 459         Host          string
 460         Proxy         bool
 461         Scheme        string
 462         Insecure      bool
 463         ActivateUsers bool
 464 }
 465
 466 type CUDAFeatures struct {
 467         DriverVersion      string
 468         HardwareCapability string
 469         DeviceCount        int
 470 }
 471
 472 type InstanceType struct {
 473         Name            string `json:"-"`
 474         ProviderType    string
 475         VCPUs           int
 476         RAM             ByteSize
 477         Scratch         ByteSize `json:"-"`
 478         IncludedScratch ByteSize
 479         AddedScratch    ByteSize
 480         Price           float64
 481         Preemptible     bool
 482         CUDA            CUDAFeatures
 483 }
 484
 485 type ContainersConfig struct {
 486         CloudVMs                      CloudVMsConfig
 487         CrunchRunCommand              string
 488         CrunchRunArgumentsList        []string
 489         DefaultKeepCacheRAM           ByteSize
 490         DispatchPrivateKey            string
 491         LogReuseDecisions             bool
 492         MaxDispatchAttempts           int
 493         MaxRetryAttempts              int
 494         MinRetryPeriod                Duration
 495         ReserveExtraRAM               ByteSize
 496         StaleLockTimeout              Duration
 497         SupportedDockerImageFormats   StringSet
 498         AlwaysUsePreemptibleInstances bool
 499         PreemptiblePriceFactor        float64
 500         MaximumPriceFactor            float64
 501         RuntimeEngine                 string
 502         LocalKeepBlobBuffersPerVCPU   int
 503         LocalKeepLogsToContainerLog   string
 504
 505         JobsAPI struct {
 506                 Enable string
 507         }
 508         Logging struct {
 509                 LogUpdatePeriod Duration
 510                 LogUpdateSize   ByteSize
 511         }
 512         ShellAccess struct {
 513                 Admin bool
 514                 User  bool
 515         }
 516         SLURM struct {
 517                 PrioritySpread             int64
 518                 SbatchArgumentsList        []string
 519                 SbatchEnvironmentVariables map[string]string
 520         }
 521         LSF struct {
 522                 BsubSudoUser       string
 523                 BsubArgumentsList  []string
 524                 BsubCUDAArguments  []string
 525                 MaxRunTimeOverhead Duration
 526                 MaxRunTimeDefault  Duration
 527         }
 528 }
 529
 530 type CloudVMsConfig struct {
 531         Enable bool
 532
 533         BootProbeCommand               string
 534         InstanceInitCommand            string
 535         DeployRunnerBinary             string
 536         DeployPublicKey                bool
 537         ImageID                        string
 538         MaxCloudOpsPerSecond           int
 539         MaxProbesPerSecond             int
 540         MaxConcurrentInstanceCreateOps int
 541         MaxInstances                   int
 542         InitialQuotaEstimate           int
 543         SupervisorFraction             float64
 544         PollInterval                   Duration
 545         ProbeInterval                  Duration
 546         SSHPort                        string
 547         SyncInterval                   Duration
 548         TimeoutBooting                 Duration
 549         TimeoutIdle                    Duration
 550         TimeoutProbe                   Duration
 551         TimeoutShutdown                Duration
 552         TimeoutSignal                  Duration
 553         TimeoutStaleRunLock            Duration
 554         TimeoutTERM                    Duration
 555         ResourceTags                   map[string]string
 556         TagKeyPrefix                   string
 557
 558         Driver           string
 559         DriverParameters json.RawMessage
 560 }
 561
 562 type InstanceTypeMap map[string]InstanceType
 563
 564 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 565
 566 // UnmarshalJSON does special handling of InstanceTypes:
 567 //
 568 // - populate computed fields (Name and Scratch)
 569 //
 570 // - error out if InstancesTypes are populated as an array, which was
 571 // deprecated in Arvados 1.2.0
 572 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 573         fixup := func(t InstanceType) (InstanceType, error) {
 574                 if t.ProviderType == "" {
 575                         t.ProviderType = t.Name
 576                 }
 577                 // If t.Scratch is set in the configuration file, it will be ignored and overwritten.
 578                 // It will also generate a "deprecated or unknown config entry" warning.
 579                 t.Scratch = t.IncludedScratch + t.AddedScratch
 580                 return t, nil
 581         }
 582
 583         if len(data) > 0 && data[0] == '[' {
 584                 return fmt.Errorf("InstanceTypes must be specified as a map, not an array, see https://doc.arvados.org/admin/config.html")
 585         }
 586         var hash map[string]InstanceType
 587         err := json.Unmarshal(data, &hash)
 588         if err != nil {
 589                 return err
 590         }
 591         // Fill in Name field (and ProviderType field, if not
 592         // specified) using hash key.
 593         *it = InstanceTypeMap(hash)
 594         for name, t := range *it {
 595                 t.Name = name
 596                 t, err := fixup(t)
 597                 if err != nil {
 598                         return err
 599                 }
 600                 (*it)[name] = t
 601         }
 602         return nil
 603 }
 604
 605 type StringSet map[string]struct{}
 606
 607 // UnmarshalJSON handles old config files that provide an array of
 608 // instance types instead of a hash.
 609 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 610         if len(data) > 0 && data[0] == '[' {
 611                 var arr []string
 612                 err := json.Unmarshal(data, &arr)
 613                 if err != nil {
 614                         return err
 615                 }
 616                 if len(arr) == 0 {
 617                         *ss = nil
 618                         return nil
 619                 }
 620                 *ss = make(map[string]struct{}, len(arr))
 621                 for _, t := range arr {
 622                         (*ss)[t] = struct{}{}
 623                 }
 624                 return nil
 625         }
 626         var hash map[string]struct{}
 627         err := json.Unmarshal(data, &hash)
 628         if err != nil {
 629                 return err
 630         }
 631         *ss = make(map[string]struct{}, len(hash))
 632         for t := range hash {
 633                 (*ss)[t] = struct{}{}
 634         }
 635
 636         return nil
 637 }
 638
 639 type ServiceName string
 640
 641 const (
 642         ServiceNameController    ServiceName = "arvados-controller"
 643         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 644         ServiceNameDispatchLSF   ServiceName = "arvados-dispatch-lsf"
 645         ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm"
 646         ServiceNameHealth        ServiceName = "arvados-health"
 647         ServiceNameKeepbalance   ServiceName = "keep-balance"
 648         ServiceNameKeepproxy     ServiceName = "keepproxy"
 649         ServiceNameKeepstore     ServiceName = "keepstore"
 650         ServiceNameKeepweb       ServiceName = "keep-web"
 651         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 652         ServiceNameWebsocket     ServiceName = "arvados-ws"
 653         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 654         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 655 )
 656
 657 // Map returns all services as a map, suitable for iterating over all
 658 // services or looking up a service by name.
 659 func (svcs Services) Map() map[ServiceName]Service {
 660         return map[ServiceName]Service{
 661                 ServiceNameController:    svcs.Controller,
 662                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 663                 ServiceNameDispatchLSF:   svcs.DispatchLSF,
 664                 ServiceNameDispatchSLURM: svcs.DispatchSLURM,
 665                 ServiceNameHealth:        svcs.Health,
 666                 ServiceNameKeepbalance:   svcs.Keepbalance,
 667                 ServiceNameKeepproxy:     svcs.Keepproxy,
 668                 ServiceNameKeepstore:     svcs.Keepstore,
 669                 ServiceNameKeepweb:       svcs.WebDAV,
 670                 ServiceNameRailsAPI:      svcs.RailsAPI,
 671                 ServiceNameWebsocket:     svcs.Websocket,
 672                 ServiceNameWorkbench1:    svcs.Workbench1,
 673                 ServiceNameWorkbench2:    svcs.Workbench2,
 674         }
 675 }