sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "crypto/tls"
   9         "encoding/json"
  10         "errors"
  11         "fmt"
  12         "net/url"
  13         "os"
  14         "time"
  15
  16         "git.arvados.org/arvados.git/sdk/go/config"
  17 )
  18
  19 var DefaultConfigFile = func() string {
  20         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  21                 return path
  22         }
  23         return "/etc/arvados/config.yml"
  24 }()
  25
  26 type Config struct {
  27         Clusters         map[string]Cluster
  28         AutoReloadConfig bool
  29         SourceTimestamp  time.Time
  30         SourceSHA256     string
  31 }
  32
  33 // GetConfig returns the current system config, loading it from
  34 // configFile if needed.
  35 func GetConfig(configFile string) (*Config, error) {
  36         var cfg Config
  37         err := config.LoadFile(&cfg, configFile)
  38         return &cfg, err
  39 }
  40
  41 // GetCluster returns the cluster ID and config for the given
  42 // cluster, or the default/only configured cluster if clusterID is "".
  43 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  44         if clusterID == "" {
  45                 if len(sc.Clusters) == 0 {
  46                         return nil, fmt.Errorf("no clusters configured")
  47                 } else if len(sc.Clusters) > 1 {
  48                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  49                 } else {
  50                         for id, cc := range sc.Clusters {
  51                                 cc.ClusterID = id
  52                                 return &cc, nil
  53                         }
  54                 }
  55         }
  56         cc, ok := sc.Clusters[clusterID]
  57         if !ok {
  58                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  59         }
  60         cc.ClusterID = clusterID
  61         return &cc, nil
  62 }
  63
  64 type WebDAVCacheConfig struct {
  65         TTL                Duration
  66         DiskCacheSize      ByteSizeOrPercent
  67         MaxCollectionBytes ByteSize
  68         MaxSessions        int
  69 }
  70
  71 type UploadDownloadPermission struct {
  72         Upload   bool
  73         Download bool
  74 }
  75
  76 type UploadDownloadRolePermissions struct {
  77         User  UploadDownloadPermission
  78         Admin UploadDownloadPermission
  79 }
  80
  81 type ManagedProperties map[string]struct {
  82         Value     interface{}
  83         Function  string
  84         Protected bool
  85 }
  86
  87 type Cluster struct {
  88         ClusterID       string `json:"-"`
  89         ManagementToken string
  90         SystemRootToken string
  91         Services        Services
  92         InstanceTypes   InstanceTypeMap
  93         Containers      ContainersConfig
  94         RemoteClusters  map[string]RemoteCluster
  95         PostgreSQL      PostgreSQL
  96
  97         API struct {
  98                 AsyncPermissionsUpdateInterval   Duration
  99                 DisabledAPIs                     StringSet
 100                 MaxIndexDatabaseRead             int
 101                 MaxItemsPerResponse              int
 102                 MaxConcurrentRailsRequests       int
 103                 MaxConcurrentRequests            int
 104                 MaxQueuedRequests                int
 105                 MaxGatewayTunnels                int
 106                 MaxQueueTimeForLockRequests      Duration
 107                 MaxKeepBlobBuffers               int
 108                 MaxRequestAmplification          int
 109                 MaxRequestSize                   int
 110                 MaxTokenLifetime                 Duration
 111                 RequestTimeout                   Duration
 112                 SendTimeout                      Duration
 113                 WebsocketClientEventQueue        int
 114                 WebsocketServerEventQueue        int
 115                 KeepServiceRequestTimeout        Duration
 116                 VocabularyPath                   string
 117                 FreezeProjectRequiresDescription bool
 118                 FreezeProjectRequiresProperties  StringSet
 119                 UnfreezeProjectRequiresAdmin     bool
 120                 LockBeforeUpdate                 bool
 121         }
 122         AuditLogs struct {
 123                 MaxAge             Duration
 124                 MaxDeleteBatch     int
 125                 UnloggedAttributes StringSet
 126         }
 127         Collections struct {
 128                 BlobSigning                  bool
 129                 BlobSigningKey               string
 130                 BlobSigningTTL               Duration
 131                 BlobTrash                    bool
 132                 BlobTrashLifetime            Duration
 133                 BlobTrashCheckInterval       Duration
 134                 BlobTrashConcurrency         int
 135                 BlobDeleteConcurrency        int
 136                 BlobReplicateConcurrency     int
 137                 CollectionVersioning         bool
 138                 DefaultTrashLifetime         Duration
 139                 DefaultReplication           int
 140                 ManagedProperties            ManagedProperties
 141                 PreserveVersionIfIdle        Duration
 142                 TrashSweepInterval           Duration
 143                 TrustAllContent              bool
 144                 ForwardSlashNameSubstitution string
 145                 S3FolderObjects              bool
 146
 147                 BlobMissingReport        string
 148                 BalancePeriod            Duration
 149                 BalanceCollectionBatch   int
 150                 BalanceCollectionBuffers int
 151                 BalanceTimeout           Duration
 152                 BalanceUpdateLimit       int
 153                 BalancePullLimit         int
 154                 BalanceTrashLimit        int
 155
 156                 WebDAVCache WebDAVCacheConfig
 157
 158                 KeepproxyPermission       UploadDownloadRolePermissions
 159                 WebDAVPermission          UploadDownloadRolePermissions
 160                 WebDAVLogEvents           bool
 161                 WebDAVLogDownloadInterval Duration
 162                 WebDAVOutputBuffer        ByteSize
 163         }
 164         Login struct {
 165                 LDAP struct {
 166                         Enable             bool
 167                         URL                URL
 168                         StartTLS           bool
 169                         InsecureTLS        bool
 170                         MinTLSVersion      TLSVersion
 171                         StripDomain        string
 172                         AppendDomain       string
 173                         SearchAttribute    string
 174                         SearchBindUser     string
 175                         SearchBindPassword string
 176                         SearchBase         string
 177                         SearchFilters      string
 178                         EmailAttribute     string
 179                         UsernameAttribute  string
 180                 }
 181                 Google struct {
 182                         Enable                          bool
 183                         ClientID                        string
 184                         ClientSecret                    string
 185                         AlternateEmailAddresses         bool
 186                         AuthenticationRequestParameters map[string]string
 187                 }
 188                 OpenIDConnect struct {
 189                         Enable                          bool
 190                         Issuer                          string
 191                         ClientID                        string
 192                         ClientSecret                    string
 193                         EmailClaim                      string
 194                         EmailVerifiedClaim              string
 195                         UsernameClaim                   string
 196                         AcceptAccessToken               bool
 197                         AcceptAccessTokenScope          string
 198                         AuthenticationRequestParameters map[string]string
 199                 }
 200                 PAM struct {
 201                         Enable             bool
 202                         Service            string
 203                         DefaultEmailDomain string
 204                 }
 205                 Test struct {
 206                         Enable bool
 207                         Users  map[string]TestUser
 208                 }
 209                 LoginCluster         string
 210                 RemoteTokenRefresh   Duration
 211                 TokenLifetime        Duration
 212                 TrustedClients       map[URL]struct{}
 213                 TrustPrivateNetworks bool
 214                 IssueTrustedTokens   bool
 215         }
 216         SystemLogs struct {
 217                 LogLevel                  string
 218                 Format                    string
 219                 MaxRequestLogParamsSize   int
 220                 RequestQueueDumpDirectory string
 221         }
 222         TLS struct {
 223                 Certificate string
 224                 Key         string
 225                 Insecure    bool
 226                 ACME        struct {
 227                         Server string
 228                 }
 229         }
 230         Users struct {
 231                 ActivatedUsersAreVisibleToOthers      bool
 232                 AnonymousUserToken                    string
 233                 AdminNotifierEmailFrom                string
 234                 AutoAdminFirstUser                    bool
 235                 AutoAdminUserWithEmail                string
 236                 AutoSetupNewUsers                     bool
 237                 AutoSetupNewUsersWithVmUUID           string
 238                 AutoSetupUsernameBlacklist            StringSet
 239                 EmailSubjectPrefix                    string
 240                 NewInactiveUserNotificationRecipients StringSet
 241                 NewUserNotificationRecipients         StringSet
 242                 NewUsersAreActive                     bool
 243                 SendUserSetupNotificationEmail        bool
 244                 SupportEmailAddress                   string
 245                 UserNotifierEmailFrom                 string
 246                 UserNotifierEmailBcc                  StringSet
 247                 UserProfileNotificationAddress        string
 248                 PreferDomainForUsername               string
 249                 UserSetupMailText                     string
 250                 RoleGroupsVisibleToAll                bool
 251                 CanCreateRoleGroups                   bool
 252                 ActivityLoggingPeriod                 Duration
 253                 SyncIgnoredGroups                     []string
 254                 SyncRequiredGroups                    []string
 255                 SyncUserAccounts                      bool
 256                 SyncUserAPITokens                     bool
 257                 SyncUserGroups                        bool
 258                 SyncUserSSHKeys                       bool
 259         }
 260         StorageClasses map[string]StorageClassConfig
 261         Volumes        map[string]Volume
 262         Workbench      struct {
 263                 ActivationContactLink   string
 264                 ArvadosDocsite          string
 265                 ArvadosPublicDataDocURL string
 266                 DisableSharingURLsUI    bool
 267                 FileViewersConfigURL    string
 268                 ShowUserAgreementInline bool
 269                 SiteName                string
 270                 Theme                   string
 271                 UserProfileFormFields   map[string]struct {
 272                         Type                 string
 273                         FormFieldTitle       string
 274                         FormFieldDescription string
 275                         Required             bool
 276                         Position             int
 277                         Options              map[string]struct{}
 278                 }
 279                 UserProfileFormMessage string
 280                 WelcomePageHTML        string
 281                 InactivePageHTML       string
 282                 SSHHelpPageHTML        string
 283                 SSHHelpHostSuffix      string
 284                 IdleTimeout            Duration
 285                 BannerUUID             string
 286         }
 287 }
 288
 289 type StorageClassConfig struct {
 290         Default  bool
 291         Priority int
 292 }
 293
 294 type Volume struct {
 295         AccessViaHosts         map[URL]VolumeAccess
 296         ReadOnly               bool
 297         AllowTrashWhenReadOnly bool
 298         Replication            int
 299         StorageClasses         map[string]bool
 300         Driver                 string
 301         DriverParameters       json.RawMessage
 302 }
 303
 304 type S3VolumeDriverParameters struct {
 305         AccessKeyID        string
 306         SecretAccessKey    string
 307         Endpoint           string
 308         Region             string
 309         Bucket             string
 310         LocationConstraint bool
 311         V2Signature        bool
 312         IndexPageSize      int
 313         ConnectTimeout     Duration
 314         ReadTimeout        Duration
 315         RaceWindow         Duration
 316         UnsafeDelete       bool
 317         PrefixLength       int
 318         UsePathStyle       bool
 319 }
 320
 321 type AzureVolumeDriverParameters struct {
 322         StorageAccountName   string
 323         StorageAccountKey    string
 324         StorageBaseURL       string
 325         ContainerName        string
 326         RequestTimeout       Duration
 327         ListBlobsRetryDelay  Duration
 328         ListBlobsMaxAttempts int
 329 }
 330
 331 type DirectoryVolumeDriverParameters struct {
 332         Root      string
 333         Serialize bool
 334 }
 335
 336 type VolumeAccess struct {
 337         ReadOnly bool
 338 }
 339
 340 type Services struct {
 341         Composer             Service
 342         ContainerWebServices ServiceWithPortRange
 343         Controller           Service
 344         DispatchCloud        Service
 345         DispatchLSF          Service
 346         DispatchSLURM        Service
 347         Health               Service
 348         Keepbalance          Service
 349         Keepproxy            Service
 350         Keepstore            Service
 351         RailsAPI             Service
 352         WebDAVDownload       Service
 353         WebDAV               Service
 354         WebShell             Service
 355         Websocket            Service
 356         Workbench1           Service
 357         Workbench2           Service
 358 }
 359
 360 type Service struct {
 361         InternalURLs map[URL]ServiceInstance
 362         ExternalURL  URL
 363 }
 364
 365 type ServiceWithPortRange struct {
 366         Service
 367         ExternalPortMin int
 368         ExternalPortMax int
 369 }
 370
 371 type TestUser struct {
 372         Email    string
 373         Password string
 374 }
 375
 376 // URL is a url.URL that is also usable as a JSON key/value.
 377 type URL url.URL
 378
 379 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 380 // used as a JSON key/value.
 381 func (su *URL) UnmarshalText(text []byte) error {
 382         u, err := url.Parse(string(text))
 383         if err == nil {
 384                 *su = URL(*u)
 385                 if su.Path == "" && su.Host != "" {
 386                         // http://example really means http://example/
 387                         su.Path = "/"
 388                 }
 389         }
 390         return err
 391 }
 392
 393 func (su URL) MarshalText() ([]byte, error) {
 394         return []byte(su.String()), nil
 395 }
 396
 397 func (su URL) String() string {
 398         return (*url.URL)(&su).String()
 399 }
 400
 401 type TLSVersion uint16
 402
 403 func (v TLSVersion) MarshalText() ([]byte, error) {
 404         switch v {
 405         case 0:
 406                 return []byte{}, nil
 407         case tls.VersionTLS10:
 408                 return []byte("1.0"), nil
 409         case tls.VersionTLS11:
 410                 return []byte("1.1"), nil
 411         case tls.VersionTLS12:
 412                 return []byte("1.2"), nil
 413         case tls.VersionTLS13:
 414                 return []byte("1.3"), nil
 415         default:
 416                 return nil, fmt.Errorf("unsupported TLSVersion %x", v)
 417         }
 418 }
 419
 420 func (v *TLSVersion) UnmarshalJSON(text []byte) error {
 421         if len(text) > 0 && text[0] == '"' {
 422                 var s string
 423                 err := json.Unmarshal(text, &s)
 424                 if err != nil {
 425                         return err
 426                 }
 427                 text = []byte(s)
 428         }
 429         switch string(text) {
 430         case "":
 431                 *v = 0
 432         case "1.0":
 433                 *v = tls.VersionTLS10
 434         case "1.1":
 435                 *v = tls.VersionTLS11
 436         case "1.2":
 437                 *v = tls.VersionTLS12
 438         case "1.3":
 439                 *v = tls.VersionTLS13
 440         default:
 441                 return fmt.Errorf("unsupported TLSVersion %q", text)
 442         }
 443         return nil
 444 }
 445
 446 type ServiceInstance struct {
 447         ListenURL  URL
 448         Rendezvous string `json:",omitempty"`
 449 }
 450
 451 type PostgreSQL struct {
 452         Connection     PostgreSQLConnection
 453         ConnectionPool int
 454 }
 455
 456 type PostgreSQLConnection map[string]string
 457
 458 type RemoteCluster struct {
 459         Host          string
 460         Proxy         bool
 461         Scheme        string
 462         Insecure      bool
 463         ActivateUsers bool
 464 }
 465
 466 type GPUFeatures struct {
 467         // as of this writing, stack is "cuda" or "rocm"
 468         Stack          string
 469         DriverVersion  string
 470         HardwareTarget string
 471         DeviceCount    int
 472         VRAM           ByteSize
 473 }
 474
 475 type InstanceType struct {
 476         Name            string `json:"-"`
 477         ProviderType    string
 478         VCPUs           int
 479         RAM             ByteSize
 480         Scratch         ByteSize `json:"-"`
 481         IncludedScratch ByteSize
 482         AddedScratch    ByteSize
 483         Price           float64
 484         Preemptible     bool
 485         GPU             GPUFeatures
 486 }
 487
 488 type ContainersConfig struct {
 489         CloudVMs                      CloudVMsConfig
 490         CrunchRunCommand              string
 491         CrunchRunArgumentsList        []string
 492         DefaultKeepCacheRAM           ByteSize
 493         DispatchPrivateKey            string
 494         LogReuseDecisions             bool
 495         MaxDispatchAttempts           int
 496         MaxRetryAttempts              int
 497         MinRetryPeriod                Duration
 498         ReserveExtraRAM               ByteSize
 499         StaleLockTimeout              Duration
 500         SupportedDockerImageFormats   StringSet
 501         AlwaysUsePreemptibleInstances bool
 502         PreemptiblePriceFactor        float64
 503         MaximumPriceFactor            float64
 504         RuntimeEngine                 string
 505         LocalKeepBlobBuffersPerVCPU   int
 506         LocalKeepLogsToContainerLog   string
 507
 508         Logging struct {
 509                 LogUpdatePeriod Duration
 510                 LogUpdateSize   ByteSize
 511         }
 512         ShellAccess struct {
 513                 Admin bool
 514                 User  bool
 515         }
 516         SLURM struct {
 517                 PrioritySpread             int64
 518                 SbatchArgumentsList        []string
 519                 SbatchEnvironmentVariables map[string]string
 520         }
 521         LSF struct {
 522                 BsubSudoUser       string
 523                 BsubArgumentsList  []string
 524                 BsubGPUArguments   []string
 525                 MaxRunTimeOverhead Duration
 526                 MaxRunTimeDefault  Duration
 527         }
 528 }
 529
 530 type CloudVMsConfig struct {
 531         Enable bool
 532
 533         BootProbeCommand               string
 534         InstanceInitCommand            string
 535         DeployRunnerBinary             string
 536         DeployRunnerDirectory          string
 537         DeployPublicKey                bool
 538         ImageID                        string
 539         MaxCloudOpsPerSecond           int
 540         MaxProbesPerSecond             int
 541         MaxConcurrentInstanceCreateOps int
 542         MaxInstances                   int
 543         InitialQuotaEstimate           int
 544         SupervisorFraction             float64
 545         PollInterval                   Duration
 546         ProbeInterval                  Duration
 547         SSHPort                        string
 548         SyncInterval                   Duration
 549         TimeoutBooting                 Duration
 550         TimeoutIdle                    Duration
 551         TimeoutProbe                   Duration
 552         TimeoutShutdown                Duration
 553         TimeoutSignal                  Duration
 554         TimeoutStaleRunLock            Duration
 555         TimeoutTERM                    Duration
 556         ResourceTags                   map[string]string
 557         TagKeyPrefix                   string
 558
 559         Driver           string
 560         DriverParameters json.RawMessage
 561 }
 562
 563 type InstanceTypeMap map[string]InstanceType
 564
 565 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 566
 567 // UnmarshalJSON does special handling of InstanceTypes:
 568 //
 569 // - populate computed fields (Name and Scratch)
 570 //
 571 // - error out if InstancesTypes are populated as an array, which was
 572 // deprecated in Arvados 1.2.0
 573 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 574         fixup := func(t InstanceType) (InstanceType, error) {
 575                 if t.ProviderType == "" {
 576                         t.ProviderType = t.Name
 577                 }
 578                 // If t.Scratch is set in the configuration file, it will be ignored and overwritten.
 579                 // It will also generate a "deprecated or unknown config entry" warning.
 580                 t.Scratch = t.IncludedScratch + t.AddedScratch
 581                 return t, nil
 582         }
 583
 584         if len(data) > 0 && data[0] == '[' {
 585                 return fmt.Errorf("InstanceTypes must be specified as a map, not an array, see https://doc.arvados.org/admin/config.html")
 586         }
 587         var hash map[string]InstanceType
 588         err := json.Unmarshal(data, &hash)
 589         if err != nil {
 590                 return err
 591         }
 592         // Fill in Name field (and ProviderType field, if not
 593         // specified) using hash key.
 594         *it = InstanceTypeMap(hash)
 595         for name, t := range *it {
 596                 t.Name = name
 597                 t, err := fixup(t)
 598                 if err != nil {
 599                         return err
 600                 }
 601                 (*it)[name] = t
 602         }
 603         return nil
 604 }
 605
 606 type StringSet map[string]struct{}
 607
 608 // UnmarshalJSON handles old config files that provide an array of
 609 // instance types instead of a hash.
 610 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 611         if len(data) > 0 && data[0] == '[' {
 612                 var arr []string
 613                 err := json.Unmarshal(data, &arr)
 614                 if err != nil {
 615                         return err
 616                 }
 617                 if len(arr) == 0 {
 618                         *ss = nil
 619                         return nil
 620                 }
 621                 *ss = make(map[string]struct{}, len(arr))
 622                 for _, t := range arr {
 623                         (*ss)[t] = struct{}{}
 624                 }
 625                 return nil
 626         }
 627         var hash map[string]struct{}
 628         err := json.Unmarshal(data, &hash)
 629         if err != nil {
 630                 return err
 631         }
 632         *ss = make(map[string]struct{}, len(hash))
 633         for t := range hash {
 634                 (*ss)[t] = struct{}{}
 635         }
 636
 637         return nil
 638 }
 639
 640 type ServiceName string
 641
 642 const (
 643         ServiceNameController    ServiceName = "arvados-controller"
 644         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 645         ServiceNameDispatchLSF   ServiceName = "arvados-dispatch-lsf"
 646         ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm"
 647         ServiceNameHealth        ServiceName = "arvados-health"
 648         ServiceNameKeepbalance   ServiceName = "keep-balance"
 649         ServiceNameKeepproxy     ServiceName = "keepproxy"
 650         ServiceNameKeepstore     ServiceName = "keepstore"
 651         ServiceNameKeepweb       ServiceName = "keep-web"
 652         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 653         ServiceNameWebsocket     ServiceName = "arvados-ws"
 654         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 655         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 656 )
 657
 658 // Map returns all services as a map, suitable for iterating over all
 659 // services or looking up a service by name.
 660 func (svcs Services) Map() map[ServiceName]Service {
 661         return map[ServiceName]Service{
 662                 ServiceNameController:    svcs.Controller,
 663                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 664                 ServiceNameDispatchLSF:   svcs.DispatchLSF,
 665                 ServiceNameDispatchSLURM: svcs.DispatchSLURM,
 666                 ServiceNameHealth:        svcs.Health,
 667                 ServiceNameKeepbalance:   svcs.Keepbalance,
 668                 ServiceNameKeepproxy:     svcs.Keepproxy,
 669                 ServiceNameKeepstore:     svcs.Keepstore,
 670                 ServiceNameKeepweb:       svcs.WebDAV,
 671                 ServiceNameRailsAPI:      svcs.RailsAPI,
 672                 ServiceNameWebsocket:     svcs.Websocket,
 673                 ServiceNameWorkbench1:    svcs.Workbench1,
 674                 ServiceNameWorkbench2:    svcs.Workbench2,
 675         }
 676 }