sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "crypto/tls"
   9         "encoding/json"
  10         "errors"
  11         "fmt"
  12         "net/url"
  13         "os"
  14         "time"
  15
  16         "git.arvados.org/arvados.git/sdk/go/config"
  17 )
  18
  19 var DefaultConfigFile = func() string {
  20         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  21                 return path
  22         }
  23         return "/etc/arvados/config.yml"
  24 }()
  25
  26 type Config struct {
  27         Clusters         map[string]Cluster
  28         AutoReloadConfig bool
  29         SourceTimestamp  time.Time
  30         SourceSHA256     string
  31 }
  32
  33 // GetConfig returns the current system config, loading it from
  34 // configFile if needed.
  35 func GetConfig(configFile string) (*Config, error) {
  36         var cfg Config
  37         err := config.LoadFile(&cfg, configFile)
  38         return &cfg, err
  39 }
  40
  41 // GetCluster returns the cluster ID and config for the given
  42 // cluster, or the default/only configured cluster if clusterID is "".
  43 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  44         if clusterID == "" {
  45                 if len(sc.Clusters) == 0 {
  46                         return nil, fmt.Errorf("no clusters configured")
  47                 } else if len(sc.Clusters) > 1 {
  48                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  49                 } else {
  50                         for id, cc := range sc.Clusters {
  51                                 cc.ClusterID = id
  52                                 return &cc, nil
  53                         }
  54                 }
  55         }
  56         cc, ok := sc.Clusters[clusterID]
  57         if !ok {
  58                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  59         }
  60         cc.ClusterID = clusterID
  61         return &cc, nil
  62 }
  63
  64 type WebDAVCacheConfig struct {
  65         TTL                Duration
  66         DiskCacheSize      ByteSizeOrPercent
  67         MaxCollectionBytes ByteSize
  68         MaxSessions        int
  69 }
  70
  71 type UploadDownloadPermission struct {
  72         Upload   bool
  73         Download bool
  74 }
  75
  76 type UploadDownloadRolePermissions struct {
  77         User  UploadDownloadPermission
  78         Admin UploadDownloadPermission
  79 }
  80
  81 type ManagedProperties map[string]struct {
  82         Value     interface{}
  83         Function  string
  84         Protected bool
  85 }
  86
  87 type Cluster struct {
  88         ClusterID       string `json:"-"`
  89         ManagementToken string
  90         SystemRootToken string
  91         Services        Services
  92         InstanceTypes   InstanceTypeMap
  93         Containers      ContainersConfig
  94         RemoteClusters  map[string]RemoteCluster
  95         PostgreSQL      PostgreSQL
  96
  97         API struct {
  98                 AsyncPermissionsUpdateInterval   Duration
  99                 DisabledAPIs                     StringSet
 100                 MaxIndexDatabaseRead             int
 101                 MaxItemsPerResponse              int
 102                 MaxConcurrentRailsRequests       int
 103                 MaxConcurrentRequests            int
 104                 MaxQueuedRequests                int
 105                 MaxGatewayTunnels                int
 106                 MaxQueueTimeForLockRequests      Duration
 107                 LogCreateRequestFraction         float64
 108                 MaxKeepBlobBuffers               int
 109                 MaxRequestAmplification          int
 110                 MaxRequestSize                   int
 111                 MaxTokenLifetime                 Duration
 112                 RequestTimeout                   Duration
 113                 SendTimeout                      Duration
 114                 WebsocketClientEventQueue        int
 115                 WebsocketServerEventQueue        int
 116                 KeepServiceRequestTimeout        Duration
 117                 VocabularyPath                   string
 118                 FreezeProjectRequiresDescription bool
 119                 FreezeProjectRequiresProperties  StringSet
 120                 UnfreezeProjectRequiresAdmin     bool
 121                 LockBeforeUpdate                 bool
 122         }
 123         AuditLogs struct {
 124                 MaxAge             Duration
 125                 MaxDeleteBatch     int
 126                 UnloggedAttributes StringSet
 127         }
 128         Collections struct {
 129                 BlobSigning                  bool
 130                 BlobSigningKey               string
 131                 BlobSigningTTL               Duration
 132                 BlobTrash                    bool
 133                 BlobTrashLifetime            Duration
 134                 BlobTrashCheckInterval       Duration
 135                 BlobTrashConcurrency         int
 136                 BlobDeleteConcurrency        int
 137                 BlobReplicateConcurrency     int
 138                 CollectionVersioning         bool
 139                 DefaultTrashLifetime         Duration
 140                 DefaultReplication           int
 141                 ManagedProperties            ManagedProperties
 142                 PreserveVersionIfIdle        Duration
 143                 TrashSweepInterval           Duration
 144                 TrustAllContent              bool
 145                 ForwardSlashNameSubstitution string
 146                 S3FolderObjects              bool
 147
 148                 BlobMissingReport        string
 149                 BalancePeriod            Duration
 150                 BalanceCollectionBatch   int
 151                 BalanceCollectionBuffers int
 152                 BalanceTimeout           Duration
 153                 BalanceUpdateLimit       int
 154                 BalancePullLimit         int
 155                 BalanceTrashLimit        int
 156
 157                 WebDAVCache WebDAVCacheConfig
 158
 159                 KeepproxyPermission UploadDownloadRolePermissions
 160                 WebDAVPermission    UploadDownloadRolePermissions
 161                 WebDAVLogEvents     bool
 162                 WebDAVOutputBuffer  ByteSize
 163         }
 164         Login struct {
 165                 LDAP struct {
 166                         Enable             bool
 167                         URL                URL
 168                         StartTLS           bool
 169                         InsecureTLS        bool
 170                         MinTLSVersion      TLSVersion
 171                         StripDomain        string
 172                         AppendDomain       string
 173                         SearchAttribute    string
 174                         SearchBindUser     string
 175                         SearchBindPassword string
 176                         SearchBase         string
 177                         SearchFilters      string
 178                         EmailAttribute     string
 179                         UsernameAttribute  string
 180                 }
 181                 Google struct {
 182                         Enable                          bool
 183                         ClientID                        string
 184                         ClientSecret                    string
 185                         AlternateEmailAddresses         bool
 186                         AuthenticationRequestParameters map[string]string
 187                 }
 188                 OpenIDConnect struct {
 189                         Enable                          bool
 190                         Issuer                          string
 191                         ClientID                        string
 192                         ClientSecret                    string
 193                         EmailClaim                      string
 194                         EmailVerifiedClaim              string
 195                         UsernameClaim                   string
 196                         AcceptAccessToken               bool
 197                         AcceptAccessTokenScope          string
 198                         AuthenticationRequestParameters map[string]string
 199                 }
 200                 PAM struct {
 201                         Enable             bool
 202                         Service            string
 203                         DefaultEmailDomain string
 204                 }
 205                 Test struct {
 206                         Enable bool
 207                         Users  map[string]TestUser
 208                 }
 209                 LoginCluster         string
 210                 RemoteTokenRefresh   Duration
 211                 TokenLifetime        Duration
 212                 TrustedClients       map[URL]struct{}
 213                 TrustPrivateNetworks bool
 214                 IssueTrustedTokens   bool
 215         }
 216         Mail struct {
 217                 SendUserSetupNotificationEmail bool
 218                 SupportEmailAddress            string
 219         }
 220         SystemLogs struct {
 221                 LogLevel                  string
 222                 Format                    string
 223                 MaxRequestLogParamsSize   int
 224                 RequestQueueDumpDirectory string
 225         }
 226         TLS struct {
 227                 Certificate string
 228                 Key         string
 229                 Insecure    bool
 230                 ACME        struct {
 231                         Server string
 232                 }
 233         }
 234         Users struct {
 235                 ActivatedUsersAreVisibleToOthers      bool
 236                 AnonymousUserToken                    string
 237                 AdminNotifierEmailFrom                string
 238                 AutoAdminFirstUser                    bool
 239                 AutoAdminUserWithEmail                string
 240                 AutoSetupNewUsers                     bool
 241                 AutoSetupNewUsersWithVmUUID           string
 242                 AutoSetupUsernameBlacklist            StringSet
 243                 EmailSubjectPrefix                    string
 244                 NewInactiveUserNotificationRecipients StringSet
 245                 NewUserNotificationRecipients         StringSet
 246                 NewUsersAreActive                     bool
 247                 UserNotifierEmailFrom                 string
 248                 UserNotifierEmailBcc                  StringSet
 249                 UserProfileNotificationAddress        string
 250                 PreferDomainForUsername               string
 251                 UserSetupMailText                     string
 252                 RoleGroupsVisibleToAll                bool
 253                 CanCreateRoleGroups                   bool
 254                 ActivityLoggingPeriod                 Duration
 255                 SyncIgnoredGroups                     []string
 256                 SyncRequiredGroups                    []string
 257                 SyncUserAccounts                      bool
 258                 SyncUserAPITokens                     bool
 259                 SyncUserGroups                        bool
 260                 SyncUserSSHKeys                       bool
 261         }
 262         StorageClasses map[string]StorageClassConfig
 263         Volumes        map[string]Volume
 264         Workbench      struct {
 265                 ActivationContactLink   string
 266                 ArvadosDocsite          string
 267                 ArvadosPublicDataDocURL string
 268                 DisableSharingURLsUI    bool
 269                 FileViewersConfigURL    string
 270                 ShowUserAgreementInline bool
 271                 SiteName                string
 272                 Theme                   string
 273                 UserProfileFormFields   map[string]struct {
 274                         Type                 string
 275                         FormFieldTitle       string
 276                         FormFieldDescription string
 277                         Required             bool
 278                         Position             int
 279                         Options              map[string]struct{}
 280                 }
 281                 UserProfileFormMessage string
 282                 WelcomePageHTML        string
 283                 InactivePageHTML       string
 284                 SSHHelpPageHTML        string
 285                 SSHHelpHostSuffix      string
 286                 IdleTimeout            Duration
 287                 BannerUUID             string
 288         }
 289 }
 290
 291 type StorageClassConfig struct {
 292         Default  bool
 293         Priority int
 294 }
 295
 296 type Volume struct {
 297         AccessViaHosts         map[URL]VolumeAccess
 298         ReadOnly               bool
 299         AllowTrashWhenReadOnly bool
 300         Replication            int
 301         StorageClasses         map[string]bool
 302         Driver                 string
 303         DriverParameters       json.RawMessage
 304 }
 305
 306 type S3VolumeDriverParameters struct {
 307         IAMRole            string
 308         AccessKeyID        string
 309         SecretAccessKey    string
 310         Endpoint           string
 311         Region             string
 312         Bucket             string
 313         LocationConstraint bool
 314         V2Signature        bool
 315         IndexPageSize      int
 316         ConnectTimeout     Duration
 317         ReadTimeout        Duration
 318         RaceWindow         Duration
 319         UnsafeDelete       bool
 320         PrefixLength       int
 321 }
 322
 323 type AzureVolumeDriverParameters struct {
 324         StorageAccountName   string
 325         StorageAccountKey    string
 326         StorageBaseURL       string
 327         ContainerName        string
 328         RequestTimeout       Duration
 329         ListBlobsRetryDelay  Duration
 330         ListBlobsMaxAttempts int
 331 }
 332
 333 type DirectoryVolumeDriverParameters struct {
 334         Root      string
 335         Serialize bool
 336 }
 337
 338 type VolumeAccess struct {
 339         ReadOnly bool
 340 }
 341
 342 type Services struct {
 343         Composer       Service
 344         Controller     Service
 345         DispatchCloud  Service
 346         DispatchLSF    Service
 347         DispatchSLURM  Service
 348         Health         Service
 349         Keepbalance    Service
 350         Keepproxy      Service
 351         Keepstore      Service
 352         RailsAPI       Service
 353         WebDAVDownload Service
 354         WebDAV         Service
 355         WebShell       Service
 356         Websocket      Service
 357         Workbench1     Service
 358         Workbench2     Service
 359 }
 360
 361 type Service struct {
 362         InternalURLs map[URL]ServiceInstance
 363         ExternalURL  URL
 364 }
 365
 366 type TestUser struct {
 367         Email    string
 368         Password string
 369 }
 370
 371 // URL is a url.URL that is also usable as a JSON key/value.
 372 type URL url.URL
 373
 374 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 375 // used as a JSON key/value.
 376 func (su *URL) UnmarshalText(text []byte) error {
 377         u, err := url.Parse(string(text))
 378         if err == nil {
 379                 *su = URL(*u)
 380                 if su.Path == "" && su.Host != "" {
 381                         // http://example really means http://example/
 382                         su.Path = "/"
 383                 }
 384         }
 385         return err
 386 }
 387
 388 func (su URL) MarshalText() ([]byte, error) {
 389         return []byte(su.String()), nil
 390 }
 391
 392 func (su URL) String() string {
 393         return (*url.URL)(&su).String()
 394 }
 395
 396 type TLSVersion uint16
 397
 398 func (v TLSVersion) MarshalText() ([]byte, error) {
 399         switch v {
 400         case 0:
 401                 return []byte{}, nil
 402         case tls.VersionTLS10:
 403                 return []byte("1.0"), nil
 404         case tls.VersionTLS11:
 405                 return []byte("1.1"), nil
 406         case tls.VersionTLS12:
 407                 return []byte("1.2"), nil
 408         case tls.VersionTLS13:
 409                 return []byte("1.3"), nil
 410         default:
 411                 return nil, fmt.Errorf("unsupported TLSVersion %x", v)
 412         }
 413 }
 414
 415 func (v *TLSVersion) UnmarshalJSON(text []byte) error {
 416         if len(text) > 0 && text[0] == '"' {
 417                 var s string
 418                 err := json.Unmarshal(text, &s)
 419                 if err != nil {
 420                         return err
 421                 }
 422                 text = []byte(s)
 423         }
 424         switch string(text) {
 425         case "":
 426                 *v = 0
 427         case "1.0":
 428                 *v = tls.VersionTLS10
 429         case "1.1":
 430                 *v = tls.VersionTLS11
 431         case "1.2":
 432                 *v = tls.VersionTLS12
 433         case "1.3":
 434                 *v = tls.VersionTLS13
 435         default:
 436                 return fmt.Errorf("unsupported TLSVersion %q", text)
 437         }
 438         return nil
 439 }
 440
 441 type ServiceInstance struct {
 442         ListenURL  URL
 443         Rendezvous string `json:",omitempty"`
 444 }
 445
 446 type PostgreSQL struct {
 447         Connection     PostgreSQLConnection
 448         ConnectionPool int
 449 }
 450
 451 type PostgreSQLConnection map[string]string
 452
 453 type RemoteCluster struct {
 454         Host          string
 455         Proxy         bool
 456         Scheme        string
 457         Insecure      bool
 458         ActivateUsers bool
 459 }
 460
 461 type CUDAFeatures struct {
 462         DriverVersion      string
 463         HardwareCapability string
 464         DeviceCount        int
 465 }
 466
 467 type InstanceType struct {
 468         Name            string `json:"-"`
 469         ProviderType    string
 470         VCPUs           int
 471         RAM             ByteSize
 472         Scratch         ByteSize `json:"-"`
 473         IncludedScratch ByteSize
 474         AddedScratch    ByteSize
 475         Price           float64
 476         Preemptible     bool
 477         CUDA            CUDAFeatures
 478 }
 479
 480 type ContainersConfig struct {
 481         CloudVMs                      CloudVMsConfig
 482         CrunchRunCommand              string
 483         CrunchRunArgumentsList        []string
 484         DefaultKeepCacheRAM           ByteSize
 485         DispatchPrivateKey            string
 486         LogReuseDecisions             bool
 487         MaxDispatchAttempts           int
 488         MaxRetryAttempts              int
 489         MinRetryPeriod                Duration
 490         ReserveExtraRAM               ByteSize
 491         StaleLockTimeout              Duration
 492         SupportedDockerImageFormats   StringSet
 493         AlwaysUsePreemptibleInstances bool
 494         PreemptiblePriceFactor        float64
 495         MaximumPriceFactor            float64
 496         RuntimeEngine                 string
 497         LocalKeepBlobBuffersPerVCPU   int
 498         LocalKeepLogsToContainerLog   string
 499
 500         Logging struct {
 501                 LogUpdatePeriod Duration
 502                 LogUpdateSize   ByteSize
 503         }
 504         ShellAccess struct {
 505                 Admin bool
 506                 User  bool
 507         }
 508         SLURM struct {
 509                 PrioritySpread             int64
 510                 SbatchArgumentsList        []string
 511                 SbatchEnvironmentVariables map[string]string
 512         }
 513         LSF struct {
 514                 BsubSudoUser       string
 515                 BsubArgumentsList  []string
 516                 BsubCUDAArguments  []string
 517                 MaxRunTimeOverhead Duration
 518                 MaxRunTimeDefault  Duration
 519         }
 520 }
 521
 522 type CloudVMsConfig struct {
 523         Enable bool
 524
 525         BootProbeCommand               string
 526         InstanceInitCommand            string
 527         DeployRunnerBinary             string
 528         DeployPublicKey                bool
 529         ImageID                        string
 530         MaxCloudOpsPerSecond           int
 531         MaxProbesPerSecond             int
 532         MaxConcurrentInstanceCreateOps int
 533         MaxInstances                   int
 534         InitialQuotaEstimate           int
 535         SupervisorFraction             float64
 536         PollInterval                   Duration
 537         ProbeInterval                  Duration
 538         SSHPort                        string
 539         SyncInterval                   Duration
 540         TimeoutBooting                 Duration
 541         TimeoutIdle                    Duration
 542         TimeoutProbe                   Duration
 543         TimeoutShutdown                Duration
 544         TimeoutSignal                  Duration
 545         TimeoutStaleRunLock            Duration
 546         TimeoutTERM                    Duration
 547         ResourceTags                   map[string]string
 548         TagKeyPrefix                   string
 549
 550         Driver           string
 551         DriverParameters json.RawMessage
 552 }
 553
 554 type InstanceTypeMap map[string]InstanceType
 555
 556 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 557
 558 // UnmarshalJSON does special handling of InstanceTypes:
 559 //
 560 // - populate computed fields (Name and Scratch)
 561 //
 562 // - error out if InstancesTypes are populated as an array, which was
 563 // deprecated in Arvados 1.2.0
 564 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 565         fixup := func(t InstanceType) (InstanceType, error) {
 566                 if t.ProviderType == "" {
 567                         t.ProviderType = t.Name
 568                 }
 569                 // If t.Scratch is set in the configuration file, it will be ignored and overwritten.
 570                 // It will also generate a "deprecated or unknown config entry" warning.
 571                 t.Scratch = t.IncludedScratch + t.AddedScratch
 572                 return t, nil
 573         }
 574
 575         if len(data) > 0 && data[0] == '[' {
 576                 return fmt.Errorf("InstanceTypes must be specified as a map, not an array, see https://doc.arvados.org/admin/config.html")
 577         }
 578         var hash map[string]InstanceType
 579         err := json.Unmarshal(data, &hash)
 580         if err != nil {
 581                 return err
 582         }
 583         // Fill in Name field (and ProviderType field, if not
 584         // specified) using hash key.
 585         *it = InstanceTypeMap(hash)
 586         for name, t := range *it {
 587                 t.Name = name
 588                 t, err := fixup(t)
 589                 if err != nil {
 590                         return err
 591                 }
 592                 (*it)[name] = t
 593         }
 594         return nil
 595 }
 596
 597 type StringSet map[string]struct{}
 598
 599 // UnmarshalJSON handles old config files that provide an array of
 600 // instance types instead of a hash.
 601 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 602         if len(data) > 0 && data[0] == '[' {
 603                 var arr []string
 604                 err := json.Unmarshal(data, &arr)
 605                 if err != nil {
 606                         return err
 607                 }
 608                 if len(arr) == 0 {
 609                         *ss = nil
 610                         return nil
 611                 }
 612                 *ss = make(map[string]struct{}, len(arr))
 613                 for _, t := range arr {
 614                         (*ss)[t] = struct{}{}
 615                 }
 616                 return nil
 617         }
 618         var hash map[string]struct{}
 619         err := json.Unmarshal(data, &hash)
 620         if err != nil {
 621                 return err
 622         }
 623         *ss = make(map[string]struct{}, len(hash))
 624         for t := range hash {
 625                 (*ss)[t] = struct{}{}
 626         }
 627
 628         return nil
 629 }
 630
 631 type ServiceName string
 632
 633 const (
 634         ServiceNameController    ServiceName = "arvados-controller"
 635         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 636         ServiceNameDispatchLSF   ServiceName = "arvados-dispatch-lsf"
 637         ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm"
 638         ServiceNameHealth        ServiceName = "arvados-health"
 639         ServiceNameKeepbalance   ServiceName = "keep-balance"
 640         ServiceNameKeepproxy     ServiceName = "keepproxy"
 641         ServiceNameKeepstore     ServiceName = "keepstore"
 642         ServiceNameKeepweb       ServiceName = "keep-web"
 643         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 644         ServiceNameWebsocket     ServiceName = "arvados-ws"
 645         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 646         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 647 )
 648
 649 // Map returns all services as a map, suitable for iterating over all
 650 // services or looking up a service by name.
 651 func (svcs Services) Map() map[ServiceName]Service {
 652         return map[ServiceName]Service{
 653                 ServiceNameController:    svcs.Controller,
 654                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 655                 ServiceNameDispatchLSF:   svcs.DispatchLSF,
 656                 ServiceNameDispatchSLURM: svcs.DispatchSLURM,
 657                 ServiceNameHealth:        svcs.Health,
 658                 ServiceNameKeepbalance:   svcs.Keepbalance,
 659                 ServiceNameKeepproxy:     svcs.Keepproxy,
 660                 ServiceNameKeepstore:     svcs.Keepstore,
 661                 ServiceNameKeepweb:       svcs.WebDAV,
 662                 ServiceNameRailsAPI:      svcs.RailsAPI,
 663                 ServiceNameWebsocket:     svcs.Websocket,
 664                 ServiceNameWorkbench1:    svcs.Workbench1,
 665                 ServiceNameWorkbench2:    svcs.Workbench2,
 666         }
 667 }