sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "crypto/tls"
   9         "encoding/json"
  10         "errors"
  11         "fmt"
  12         "net/url"
  13         "os"
  14         "time"
  15
  16         "git.arvados.org/arvados.git/sdk/go/config"
  17 )
  18
  19 var DefaultConfigFile = func() string {
  20         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  21                 return path
  22         }
  23         return "/etc/arvados/config.yml"
  24 }()
  25
  26 type Config struct {
  27         Clusters         map[string]Cluster
  28         AutoReloadConfig bool
  29         SourceTimestamp  time.Time
  30         SourceSHA256     string
  31 }
  32
  33 // GetConfig returns the current system config, loading it from
  34 // configFile if needed.
  35 func GetConfig(configFile string) (*Config, error) {
  36         var cfg Config
  37         err := config.LoadFile(&cfg, configFile)
  38         return &cfg, err
  39 }
  40
  41 // GetCluster returns the cluster ID and config for the given
  42 // cluster, or the default/only configured cluster if clusterID is "".
  43 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  44         if clusterID == "" {
  45                 if len(sc.Clusters) == 0 {
  46                         return nil, fmt.Errorf("no clusters configured")
  47                 } else if len(sc.Clusters) > 1 {
  48                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  49                 } else {
  50                         for id, cc := range sc.Clusters {
  51                                 cc.ClusterID = id
  52                                 return &cc, nil
  53                         }
  54                 }
  55         }
  56         cc, ok := sc.Clusters[clusterID]
  57         if !ok {
  58                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  59         }
  60         cc.ClusterID = clusterID
  61         return &cc, nil
  62 }
  63
  64 type WebDAVCacheConfig struct {
  65         TTL                Duration
  66         DiskCacheSize      ByteSizeOrPercent
  67         MaxCollectionBytes ByteSize
  68         MaxSessions        int
  69 }
  70
  71 type UploadDownloadPermission struct {
  72         Upload   bool
  73         Download bool
  74 }
  75
  76 type UploadDownloadRolePermissions struct {
  77         User  UploadDownloadPermission
  78         Admin UploadDownloadPermission
  79 }
  80
  81 type ManagedProperties map[string]struct {
  82         Value     interface{}
  83         Function  string
  84         Protected bool
  85 }
  86
  87 type Cluster struct {
  88         ClusterID       string `json:"-"`
  89         ManagementToken string
  90         SystemRootToken string
  91         Services        Services
  92         InstanceTypes   InstanceTypeMap
  93         Containers      ContainersConfig
  94         RemoteClusters  map[string]RemoteCluster
  95         PostgreSQL      PostgreSQL
  96
  97         API struct {
  98                 AsyncPermissionsUpdateInterval   Duration
  99                 DisabledAPIs                     StringSet
 100                 MaxIndexDatabaseRead             int
 101                 MaxItemsPerResponse              int
 102                 MaxConcurrentRailsRequests       int
 103                 MaxConcurrentRequests            int
 104                 MaxQueuedRequests                int
 105                 MaxQueueTimeForLockRequests      Duration
 106                 LogCreateRequestFraction         float64
 107                 MaxKeepBlobBuffers               int
 108                 MaxRequestAmplification          int
 109                 MaxRequestSize                   int
 110                 MaxTokenLifetime                 Duration
 111                 RequestTimeout                   Duration
 112                 SendTimeout                      Duration
 113                 WebsocketClientEventQueue        int
 114                 WebsocketServerEventQueue        int
 115                 KeepServiceRequestTimeout        Duration
 116                 VocabularyPath                   string
 117                 FreezeProjectRequiresDescription bool
 118                 FreezeProjectRequiresProperties  StringSet
 119                 UnfreezeProjectRequiresAdmin     bool
 120                 LockBeforeUpdate                 bool
 121         }
 122         AuditLogs struct {
 123                 MaxAge             Duration
 124                 MaxDeleteBatch     int
 125                 UnloggedAttributes StringSet
 126         }
 127         Collections struct {
 128                 BlobSigning                  bool
 129                 BlobSigningKey               string
 130                 BlobSigningTTL               Duration
 131                 BlobTrash                    bool
 132                 BlobTrashLifetime            Duration
 133                 BlobTrashCheckInterval       Duration
 134                 BlobTrashConcurrency         int
 135                 BlobDeleteConcurrency        int
 136                 BlobReplicateConcurrency     int
 137                 CollectionVersioning         bool
 138                 DefaultTrashLifetime         Duration
 139                 DefaultReplication           int
 140                 ManagedProperties            ManagedProperties
 141                 PreserveVersionIfIdle        Duration
 142                 TrashSweepInterval           Duration
 143                 TrustAllContent              bool
 144                 ForwardSlashNameSubstitution string
 145                 S3FolderObjects              bool
 146
 147                 BlobMissingReport        string
 148                 BalancePeriod            Duration
 149                 BalanceCollectionBatch   int
 150                 BalanceCollectionBuffers int
 151                 BalanceTimeout           Duration
 152                 BalanceUpdateLimit       int
 153                 BalancePullLimit         int
 154                 BalanceTrashLimit        int
 155
 156                 WebDAVCache WebDAVCacheConfig
 157
 158                 KeepproxyPermission UploadDownloadRolePermissions
 159                 WebDAVPermission    UploadDownloadRolePermissions
 160                 WebDAVLogEvents     bool
 161         }
 162         Git struct {
 163                 GitCommand   string
 164                 GitoliteHome string
 165                 Repositories string
 166         }
 167         Login struct {
 168                 LDAP struct {
 169                         Enable             bool
 170                         URL                URL
 171                         StartTLS           bool
 172                         InsecureTLS        bool
 173                         MinTLSVersion      TLSVersion
 174                         StripDomain        string
 175                         AppendDomain       string
 176                         SearchAttribute    string
 177                         SearchBindUser     string
 178                         SearchBindPassword string
 179                         SearchBase         string
 180                         SearchFilters      string
 181                         EmailAttribute     string
 182                         UsernameAttribute  string
 183                 }
 184                 Google struct {
 185                         Enable                          bool
 186                         ClientID                        string
 187                         ClientSecret                    string
 188                         AlternateEmailAddresses         bool
 189                         AuthenticationRequestParameters map[string]string
 190                 }
 191                 OpenIDConnect struct {
 192                         Enable                          bool
 193                         Issuer                          string
 194                         ClientID                        string
 195                         ClientSecret                    string
 196                         EmailClaim                      string
 197                         EmailVerifiedClaim              string
 198                         UsernameClaim                   string
 199                         AcceptAccessToken               bool
 200                         AcceptAccessTokenScope          string
 201                         AuthenticationRequestParameters map[string]string
 202                 }
 203                 PAM struct {
 204                         Enable             bool
 205                         Service            string
 206                         DefaultEmailDomain string
 207                 }
 208                 Test struct {
 209                         Enable bool
 210                         Users  map[string]TestUser
 211                 }
 212                 LoginCluster         string
 213                 RemoteTokenRefresh   Duration
 214                 TokenLifetime        Duration
 215                 TrustedClients       map[URL]struct{}
 216                 TrustPrivateNetworks bool
 217                 IssueTrustedTokens   bool
 218         }
 219         Mail struct {
 220                 MailchimpAPIKey                string
 221                 MailchimpListID                string
 222                 SendUserSetupNotificationEmail bool
 223                 IssueReporterEmailFrom         string
 224                 IssueReporterEmailTo           string
 225                 SupportEmailAddress            string
 226                 EmailFrom                      string
 227         }
 228         SystemLogs struct {
 229                 LogLevel                  string
 230                 Format                    string
 231                 MaxRequestLogParamsSize   int
 232                 RequestQueueDumpDirectory string
 233         }
 234         TLS struct {
 235                 Certificate string
 236                 Key         string
 237                 Insecure    bool
 238                 ACME        struct {
 239                         Server string
 240                 }
 241         }
 242         Users struct {
 243                 ActivatedUsersAreVisibleToOthers      bool
 244                 AnonymousUserToken                    string
 245                 AdminNotifierEmailFrom                string
 246                 AutoAdminFirstUser                    bool
 247                 AutoAdminUserWithEmail                string
 248                 AutoSetupNewUsers                     bool
 249                 AutoSetupNewUsersWithRepository       bool
 250                 AutoSetupNewUsersWithVmUUID           string
 251                 AutoSetupUsernameBlacklist            StringSet
 252                 EmailSubjectPrefix                    string
 253                 NewInactiveUserNotificationRecipients StringSet
 254                 NewUserNotificationRecipients         StringSet
 255                 NewUsersAreActive                     bool
 256                 UserNotifierEmailFrom                 string
 257                 UserNotifierEmailBcc                  StringSet
 258                 UserProfileNotificationAddress        string
 259                 PreferDomainForUsername               string
 260                 UserSetupMailText                     string
 261                 RoleGroupsVisibleToAll                bool
 262                 CanCreateRoleGroups                   bool
 263                 ActivityLoggingPeriod                 Duration
 264                 SyncIgnoredGroups                     []string
 265                 SyncRequiredGroups                    []string
 266                 SyncUserAccounts                      bool
 267                 SyncUserAPITokens                     bool
 268                 SyncUserGroups                        bool
 269                 SyncUserSSHKeys                       bool
 270         }
 271         StorageClasses map[string]StorageClassConfig
 272         Volumes        map[string]Volume
 273         Workbench      struct {
 274                 ActivationContactLink   string
 275                 ArvadosDocsite          string
 276                 ArvadosPublicDataDocURL string
 277                 DisableSharingURLsUI    bool
 278                 FileViewersConfigURL    string
 279                 ShowUserAgreementInline bool
 280                 SiteName                string
 281                 Theme                   string
 282                 UserProfileFormFields   map[string]struct {
 283                         Type                 string
 284                         FormFieldTitle       string
 285                         FormFieldDescription string
 286                         Required             bool
 287                         Position             int
 288                         Options              map[string]struct{}
 289                 }
 290                 UserProfileFormMessage string
 291                 WelcomePageHTML        string
 292                 InactivePageHTML       string
 293                 SSHHelpPageHTML        string
 294                 SSHHelpHostSuffix      string
 295                 IdleTimeout            Duration
 296                 BannerUUID             string
 297         }
 298 }
 299
 300 type StorageClassConfig struct {
 301         Default  bool
 302         Priority int
 303 }
 304
 305 type Volume struct {
 306         AccessViaHosts         map[URL]VolumeAccess
 307         ReadOnly               bool
 308         AllowTrashWhenReadOnly bool
 309         Replication            int
 310         StorageClasses         map[string]bool
 311         Driver                 string
 312         DriverParameters       json.RawMessage
 313 }
 314
 315 type S3VolumeDriverParameters struct {
 316         IAMRole            string
 317         AccessKeyID        string
 318         SecretAccessKey    string
 319         Endpoint           string
 320         Region             string
 321         Bucket             string
 322         LocationConstraint bool
 323         V2Signature        bool
 324         IndexPageSize      int
 325         ConnectTimeout     Duration
 326         ReadTimeout        Duration
 327         RaceWindow         Duration
 328         UnsafeDelete       bool
 329         PrefixLength       int
 330 }
 331
 332 type AzureVolumeDriverParameters struct {
 333         StorageAccountName   string
 334         StorageAccountKey    string
 335         StorageBaseURL       string
 336         ContainerName        string
 337         RequestTimeout       Duration
 338         ListBlobsRetryDelay  Duration
 339         ListBlobsMaxAttempts int
 340 }
 341
 342 type DirectoryVolumeDriverParameters struct {
 343         Root      string
 344         Serialize bool
 345 }
 346
 347 type VolumeAccess struct {
 348         ReadOnly bool
 349 }
 350
 351 type Services struct {
 352         Composer       Service
 353         Controller     Service
 354         DispatchCloud  Service
 355         DispatchLSF    Service
 356         DispatchSLURM  Service
 357         GitHTTP        Service
 358         GitSSH         Service
 359         Health         Service
 360         Keepbalance    Service
 361         Keepproxy      Service
 362         Keepstore      Service
 363         RailsAPI       Service
 364         WebDAVDownload Service
 365         WebDAV         Service
 366         WebShell       Service
 367         Websocket      Service
 368         Workbench1     Service
 369         Workbench2     Service
 370 }
 371
 372 type Service struct {
 373         InternalURLs map[URL]ServiceInstance
 374         ExternalURL  URL
 375 }
 376
 377 type TestUser struct {
 378         Email    string
 379         Password string
 380 }
 381
 382 // URL is a url.URL that is also usable as a JSON key/value.
 383 type URL url.URL
 384
 385 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 386 // used as a JSON key/value.
 387 func (su *URL) UnmarshalText(text []byte) error {
 388         u, err := url.Parse(string(text))
 389         if err == nil {
 390                 *su = URL(*u)
 391                 if su.Path == "" && su.Host != "" {
 392                         // http://example really means http://example/
 393                         su.Path = "/"
 394                 }
 395         }
 396         return err
 397 }
 398
 399 func (su URL) MarshalText() ([]byte, error) {
 400         return []byte(su.String()), nil
 401 }
 402
 403 func (su URL) String() string {
 404         return (*url.URL)(&su).String()
 405 }
 406
 407 type TLSVersion uint16
 408
 409 func (v TLSVersion) MarshalText() ([]byte, error) {
 410         switch v {
 411         case 0:
 412                 return []byte{}, nil
 413         case tls.VersionTLS10:
 414                 return []byte("1.0"), nil
 415         case tls.VersionTLS11:
 416                 return []byte("1.1"), nil
 417         case tls.VersionTLS12:
 418                 return []byte("1.2"), nil
 419         case tls.VersionTLS13:
 420                 return []byte("1.3"), nil
 421         default:
 422                 return nil, fmt.Errorf("unsupported TLSVersion %x", v)
 423         }
 424 }
 425
 426 func (v *TLSVersion) UnmarshalJSON(text []byte) error {
 427         if len(text) > 0 && text[0] == '"' {
 428                 var s string
 429                 err := json.Unmarshal(text, &s)
 430                 if err != nil {
 431                         return err
 432                 }
 433                 text = []byte(s)
 434         }
 435         switch string(text) {
 436         case "":
 437                 *v = 0
 438         case "1.0":
 439                 *v = tls.VersionTLS10
 440         case "1.1":
 441                 *v = tls.VersionTLS11
 442         case "1.2":
 443                 *v = tls.VersionTLS12
 444         case "1.3":
 445                 *v = tls.VersionTLS13
 446         default:
 447                 return fmt.Errorf("unsupported TLSVersion %q", text)
 448         }
 449         return nil
 450 }
 451
 452 type ServiceInstance struct {
 453         ListenURL  URL
 454         Rendezvous string `json:",omitempty"`
 455 }
 456
 457 type PostgreSQL struct {
 458         Connection     PostgreSQLConnection
 459         ConnectionPool int
 460 }
 461
 462 type PostgreSQLConnection map[string]string
 463
 464 type RemoteCluster struct {
 465         Host          string
 466         Proxy         bool
 467         Scheme        string
 468         Insecure      bool
 469         ActivateUsers bool
 470 }
 471
 472 type CUDAFeatures struct {
 473         DriverVersion      string
 474         HardwareCapability string
 475         DeviceCount        int
 476 }
 477
 478 type InstanceType struct {
 479         Name            string `json:"-"`
 480         ProviderType    string
 481         VCPUs           int
 482         RAM             ByteSize
 483         Scratch         ByteSize `json:"-"`
 484         IncludedScratch ByteSize
 485         AddedScratch    ByteSize
 486         Price           float64
 487         Preemptible     bool
 488         CUDA            CUDAFeatures
 489 }
 490
 491 type ContainersConfig struct {
 492         CloudVMs                      CloudVMsConfig
 493         CrunchRunCommand              string
 494         CrunchRunArgumentsList        []string
 495         DefaultKeepCacheRAM           ByteSize
 496         DispatchPrivateKey            string
 497         LogReuseDecisions             bool
 498         MaxDispatchAttempts           int
 499         MaxRetryAttempts              int
 500         MinRetryPeriod                Duration
 501         ReserveExtraRAM               ByteSize
 502         StaleLockTimeout              Duration
 503         SupportedDockerImageFormats   StringSet
 504         AlwaysUsePreemptibleInstances bool
 505         PreemptiblePriceFactor        float64
 506         MaximumPriceFactor            float64
 507         RuntimeEngine                 string
 508         LocalKeepBlobBuffersPerVCPU   int
 509         LocalKeepLogsToContainerLog   string
 510
 511         JobsAPI struct {
 512                 Enable         string
 513                 GitInternalDir string
 514         }
 515         Logging struct {
 516                 MaxAge                       Duration
 517                 SweepInterval                Duration
 518                 LogBytesPerEvent             int
 519                 LogSecondsBetweenEvents      Duration
 520                 LogThrottlePeriod            Duration
 521                 LogThrottleBytes             int
 522                 LogThrottleLines             int
 523                 LimitLogBytesPerJob          int
 524                 LogPartialLineThrottlePeriod Duration
 525                 LogUpdatePeriod              Duration
 526                 LogUpdateSize                ByteSize
 527         }
 528         ShellAccess struct {
 529                 Admin bool
 530                 User  bool
 531         }
 532         SLURM struct {
 533                 PrioritySpread             int64
 534                 SbatchArgumentsList        []string
 535                 SbatchEnvironmentVariables map[string]string
 536                 Managed                    struct {
 537                         DNSServerConfDir       string
 538                         DNSServerConfTemplate  string
 539                         DNSServerReloadCommand string
 540                         DNSServerUpdateCommand string
 541                         ComputeNodeDomain      string
 542                         ComputeNodeNameservers StringSet
 543                         AssignNodeHostname     string
 544                 }
 545         }
 546         LSF struct {
 547                 BsubSudoUser      string
 548                 BsubArgumentsList []string
 549                 BsubCUDAArguments []string
 550         }
 551 }
 552
 553 type CloudVMsConfig struct {
 554         Enable bool
 555
 556         BootProbeCommand               string
 557         InstanceInitCommand            string
 558         DeployRunnerBinary             string
 559         DeployPublicKey                bool
 560         ImageID                        string
 561         MaxCloudOpsPerSecond           int
 562         MaxProbesPerSecond             int
 563         MaxConcurrentInstanceCreateOps int
 564         MaxInstances                   int
 565         InitialQuotaEstimate           int
 566         SupervisorFraction             float64
 567         PollInterval                   Duration
 568         ProbeInterval                  Duration
 569         SSHPort                        string
 570         SyncInterval                   Duration
 571         TimeoutBooting                 Duration
 572         TimeoutIdle                    Duration
 573         TimeoutProbe                   Duration
 574         TimeoutShutdown                Duration
 575         TimeoutSignal                  Duration
 576         TimeoutStaleRunLock            Duration
 577         TimeoutTERM                    Duration
 578         ResourceTags                   map[string]string
 579         TagKeyPrefix                   string
 580
 581         Driver           string
 582         DriverParameters json.RawMessage
 583 }
 584
 585 type InstanceTypeMap map[string]InstanceType
 586
 587 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 588
 589 // UnmarshalJSON does special handling of InstanceTypes:
 590 //
 591 // - populate computed fields (Name and Scratch)
 592 //
 593 // - error out if InstancesTypes are populated as an array, which was
 594 // deprecated in Arvados 1.2.0
 595 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 596         fixup := func(t InstanceType) (InstanceType, error) {
 597                 if t.ProviderType == "" {
 598                         t.ProviderType = t.Name
 599                 }
 600                 // If t.Scratch is set in the configuration file, it will be ignored and overwritten.
 601                 // It will also generate a "deprecated or unknown config entry" warning.
 602                 t.Scratch = t.IncludedScratch + t.AddedScratch
 603                 return t, nil
 604         }
 605
 606         if len(data) > 0 && data[0] == '[' {
 607                 return fmt.Errorf("InstanceTypes must be specified as a map, not an array, see https://doc.arvados.org/admin/config.html")
 608         }
 609         var hash map[string]InstanceType
 610         err := json.Unmarshal(data, &hash)
 611         if err != nil {
 612                 return err
 613         }
 614         // Fill in Name field (and ProviderType field, if not
 615         // specified) using hash key.
 616         *it = InstanceTypeMap(hash)
 617         for name, t := range *it {
 618                 t.Name = name
 619                 t, err := fixup(t)
 620                 if err != nil {
 621                         return err
 622                 }
 623                 (*it)[name] = t
 624         }
 625         return nil
 626 }
 627
 628 type StringSet map[string]struct{}
 629
 630 // UnmarshalJSON handles old config files that provide an array of
 631 // instance types instead of a hash.
 632 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 633         if len(data) > 0 && data[0] == '[' {
 634                 var arr []string
 635                 err := json.Unmarshal(data, &arr)
 636                 if err != nil {
 637                         return err
 638                 }
 639                 if len(arr) == 0 {
 640                         *ss = nil
 641                         return nil
 642                 }
 643                 *ss = make(map[string]struct{}, len(arr))
 644                 for _, t := range arr {
 645                         (*ss)[t] = struct{}{}
 646                 }
 647                 return nil
 648         }
 649         var hash map[string]struct{}
 650         err := json.Unmarshal(data, &hash)
 651         if err != nil {
 652                 return err
 653         }
 654         *ss = make(map[string]struct{}, len(hash))
 655         for t := range hash {
 656                 (*ss)[t] = struct{}{}
 657         }
 658
 659         return nil
 660 }
 661
 662 type ServiceName string
 663
 664 const (
 665         ServiceNameController    ServiceName = "arvados-controller"
 666         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 667         ServiceNameDispatchLSF   ServiceName = "arvados-dispatch-lsf"
 668         ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm"
 669         ServiceNameGitHTTP       ServiceName = "arvados-git-httpd"
 670         ServiceNameHealth        ServiceName = "arvados-health"
 671         ServiceNameKeepbalance   ServiceName = "keep-balance"
 672         ServiceNameKeepproxy     ServiceName = "keepproxy"
 673         ServiceNameKeepstore     ServiceName = "keepstore"
 674         ServiceNameKeepweb       ServiceName = "keep-web"
 675         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 676         ServiceNameWebsocket     ServiceName = "arvados-ws"
 677         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 678         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 679 )
 680
 681 // Map returns all services as a map, suitable for iterating over all
 682 // services or looking up a service by name.
 683 func (svcs Services) Map() map[ServiceName]Service {
 684         return map[ServiceName]Service{
 685                 ServiceNameController:    svcs.Controller,
 686                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 687                 ServiceNameDispatchLSF:   svcs.DispatchLSF,
 688                 ServiceNameDispatchSLURM: svcs.DispatchSLURM,
 689                 ServiceNameGitHTTP:       svcs.GitHTTP,
 690                 ServiceNameHealth:        svcs.Health,
 691                 ServiceNameKeepbalance:   svcs.Keepbalance,
 692                 ServiceNameKeepproxy:     svcs.Keepproxy,
 693                 ServiceNameKeepstore:     svcs.Keepstore,
 694                 ServiceNameKeepweb:       svcs.WebDAV,
 695                 ServiceNameRailsAPI:      svcs.RailsAPI,
 696                 ServiceNameWebsocket:     svcs.Websocket,
 697                 ServiceNameWorkbench1:    svcs.Workbench1,
 698                 ServiceNameWorkbench2:    svcs.Workbench2,
 699         }
 700 }