sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "crypto/tls"
   9         "encoding/json"
  10         "errors"
  11         "fmt"
  12         "net/url"
  13         "os"
  14         "time"
  15
  16         "git.arvados.org/arvados.git/sdk/go/config"
  17 )
  18
  19 var DefaultConfigFile = func() string {
  20         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  21                 return path
  22         }
  23         return "/etc/arvados/config.yml"
  24 }()
  25
  26 type Config struct {
  27         Clusters         map[string]Cluster
  28         AutoReloadConfig bool
  29         SourceTimestamp  time.Time
  30         SourceSHA256     string
  31 }
  32
  33 // GetConfig returns the current system config, loading it from
  34 // configFile if needed.
  35 func GetConfig(configFile string) (*Config, error) {
  36         var cfg Config
  37         err := config.LoadFile(&cfg, configFile)
  38         return &cfg, err
  39 }
  40
  41 // GetCluster returns the cluster ID and config for the given
  42 // cluster, or the default/only configured cluster if clusterID is "".
  43 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  44         if clusterID == "" {
  45                 if len(sc.Clusters) == 0 {
  46                         return nil, fmt.Errorf("no clusters configured")
  47                 } else if len(sc.Clusters) > 1 {
  48                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  49                 } else {
  50                         for id, cc := range sc.Clusters {
  51                                 cc.ClusterID = id
  52                                 return &cc, nil
  53                         }
  54                 }
  55         }
  56         cc, ok := sc.Clusters[clusterID]
  57         if !ok {
  58                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  59         }
  60         cc.ClusterID = clusterID
  61         return &cc, nil
  62 }
  63
  64 type WebDAVCacheConfig struct {
  65         TTL                Duration
  66         MaxBlockEntries    int
  67         MaxCollectionBytes int64
  68         MaxSessions        int
  69 }
  70
  71 type UploadDownloadPermission struct {
  72         Upload   bool
  73         Download bool
  74 }
  75
  76 type UploadDownloadRolePermissions struct {
  77         User  UploadDownloadPermission
  78         Admin UploadDownloadPermission
  79 }
  80
  81 type ManagedProperties map[string]struct {
  82         Value     interface{}
  83         Function  string
  84         Protected bool
  85 }
  86
  87 type Cluster struct {
  88         ClusterID       string `json:"-"`
  89         ManagementToken string
  90         SystemRootToken string
  91         Services        Services
  92         InstanceTypes   InstanceTypeMap
  93         Containers      ContainersConfig
  94         RemoteClusters  map[string]RemoteCluster
  95         PostgreSQL      PostgreSQL
  96
  97         API struct {
  98                 AsyncPermissionsUpdateInterval   Duration
  99                 DisabledAPIs                     StringSet
 100                 MaxIndexDatabaseRead             int
 101                 MaxItemsPerResponse              int
 102                 MaxConcurrentRequests            int
 103                 LogCreateRequestFraction         float64
 104                 MaxKeepBlobBuffers               int
 105                 MaxRequestAmplification          int
 106                 MaxRequestSize                   int
 107                 MaxTokenLifetime                 Duration
 108                 RequestTimeout                   Duration
 109                 SendTimeout                      Duration
 110                 WebsocketClientEventQueue        int
 111                 WebsocketServerEventQueue        int
 112                 KeepServiceRequestTimeout        Duration
 113                 VocabularyPath                   string
 114                 FreezeProjectRequiresDescription bool
 115                 FreezeProjectRequiresProperties  StringSet
 116                 UnfreezeProjectRequiresAdmin     bool
 117                 LockBeforeUpdate                 bool
 118         }
 119         AuditLogs struct {
 120                 MaxAge             Duration
 121                 MaxDeleteBatch     int
 122                 UnloggedAttributes StringSet
 123         }
 124         Collections struct {
 125                 BlobSigning                  bool
 126                 BlobSigningKey               string
 127                 BlobSigningTTL               Duration
 128                 BlobTrash                    bool
 129                 BlobTrashLifetime            Duration
 130                 BlobTrashCheckInterval       Duration
 131                 BlobTrashConcurrency         int
 132                 BlobDeleteConcurrency        int
 133                 BlobReplicateConcurrency     int
 134                 CollectionVersioning         bool
 135                 DefaultTrashLifetime         Duration
 136                 DefaultReplication           int
 137                 ManagedProperties            ManagedProperties
 138                 PreserveVersionIfIdle        Duration
 139                 TrashSweepInterval           Duration
 140                 TrustAllContent              bool
 141                 ForwardSlashNameSubstitution string
 142                 S3FolderObjects              bool
 143
 144                 BlobMissingReport        string
 145                 BalancePeriod            Duration
 146                 BalanceCollectionBatch   int
 147                 BalanceCollectionBuffers int
 148                 BalanceTimeout           Duration
 149                 BalanceUpdateLimit       int
 150
 151                 WebDAVCache WebDAVCacheConfig
 152
 153                 KeepproxyPermission UploadDownloadRolePermissions
 154                 WebDAVPermission    UploadDownloadRolePermissions
 155                 WebDAVLogEvents     bool
 156         }
 157         Git struct {
 158                 GitCommand   string
 159                 GitoliteHome string
 160                 Repositories string
 161         }
 162         Login struct {
 163                 LDAP struct {
 164                         Enable             bool
 165                         URL                URL
 166                         StartTLS           bool
 167                         InsecureTLS        bool
 168                         MinTLSVersion      TLSVersion
 169                         StripDomain        string
 170                         AppendDomain       string
 171                         SearchAttribute    string
 172                         SearchBindUser     string
 173                         SearchBindPassword string
 174                         SearchBase         string
 175                         SearchFilters      string
 176                         EmailAttribute     string
 177                         UsernameAttribute  string
 178                 }
 179                 Google struct {
 180                         Enable                          bool
 181                         ClientID                        string
 182                         ClientSecret                    string
 183                         AlternateEmailAddresses         bool
 184                         AuthenticationRequestParameters map[string]string
 185                 }
 186                 OpenIDConnect struct {
 187                         Enable                          bool
 188                         Issuer                          string
 189                         ClientID                        string
 190                         ClientSecret                    string
 191                         EmailClaim                      string
 192                         EmailVerifiedClaim              string
 193                         UsernameClaim                   string
 194                         AcceptAccessToken               bool
 195                         AcceptAccessTokenScope          string
 196                         AuthenticationRequestParameters map[string]string
 197                 }
 198                 PAM struct {
 199                         Enable             bool
 200                         Service            string
 201                         DefaultEmailDomain string
 202                 }
 203                 Test struct {
 204                         Enable bool
 205                         Users  map[string]TestUser
 206                 }
 207                 LoginCluster         string
 208                 RemoteTokenRefresh   Duration
 209                 TokenLifetime        Duration
 210                 TrustedClients       map[URL]struct{}
 211                 TrustPrivateNetworks bool
 212                 IssueTrustedTokens   bool
 213         }
 214         Mail struct {
 215                 MailchimpAPIKey                string
 216                 MailchimpListID                string
 217                 SendUserSetupNotificationEmail bool
 218                 IssueReporterEmailFrom         string
 219                 IssueReporterEmailTo           string
 220                 SupportEmailAddress            string
 221                 EmailFrom                      string
 222         }
 223         SystemLogs struct {
 224                 LogLevel                string
 225                 Format                  string
 226                 MaxRequestLogParamsSize int
 227         }
 228         TLS struct {
 229                 Certificate string
 230                 Key         string
 231                 Insecure    bool
 232                 ACME        struct {
 233                         Server string
 234                 }
 235         }
 236         Users struct {
 237                 ActivatedUsersAreVisibleToOthers      bool
 238                 AnonymousUserToken                    string
 239                 AdminNotifierEmailFrom                string
 240                 AutoAdminFirstUser                    bool
 241                 AutoAdminUserWithEmail                string
 242                 AutoSetupNewUsers                     bool
 243                 AutoSetupNewUsersWithRepository       bool
 244                 AutoSetupNewUsersWithVmUUID           string
 245                 AutoSetupUsernameBlacklist            StringSet
 246                 EmailSubjectPrefix                    string
 247                 NewInactiveUserNotificationRecipients StringSet
 248                 NewUserNotificationRecipients         StringSet
 249                 NewUsersAreActive                     bool
 250                 UserNotifierEmailFrom                 string
 251                 UserNotifierEmailBcc                  StringSet
 252                 UserProfileNotificationAddress        string
 253                 PreferDomainForUsername               string
 254                 UserSetupMailText                     string
 255                 RoleGroupsVisibleToAll                bool
 256                 CanCreateRoleGroups                   bool
 257                 ActivityLoggingPeriod                 Duration
 258         }
 259         StorageClasses map[string]StorageClassConfig
 260         Volumes        map[string]Volume
 261         Workbench      struct {
 262                 ActivationContactLink            string
 263                 APIClientConnectTimeout          Duration
 264                 APIClientReceiveTimeout          Duration
 265                 APIResponseCompression           bool
 266                 ApplicationMimetypesWithViewIcon StringSet
 267                 ArvadosDocsite                   string
 268                 ArvadosPublicDataDocURL          string
 269                 DefaultOpenIdPrefix              string
 270                 DisableSharingURLsUI             bool
 271                 EnableGettingStartedPopup        bool
 272                 EnablePublicProjectsPage         bool
 273                 FileViewersConfigURL             string
 274                 LogViewerMaxBytes                ByteSize
 275                 MultiSiteSearch                  string
 276                 ProfilingEnabled                 bool
 277                 Repositories                     bool
 278                 RepositoryCache                  string
 279                 RunningJobLogRecordsToFetch      int
 280                 SecretKeyBase                    string
 281                 ShowRecentCollectionsOnDashboard bool
 282                 ShowUserAgreementInline          bool
 283                 ShowUserNotifications            bool
 284                 SiteName                         string
 285                 Theme                            string
 286                 UserProfileFormFields            map[string]struct {
 287                         Type                 string
 288                         FormFieldTitle       string
 289                         FormFieldDescription string
 290                         Required             bool
 291                         Position             int
 292                         Options              map[string]struct{}
 293                 }
 294                 UserProfileFormMessage string
 295                 WelcomePageHTML        string
 296                 InactivePageHTML       string
 297                 SSHHelpPageHTML        string
 298                 SSHHelpHostSuffix      string
 299                 IdleTimeout            Duration
 300                 BannerUUID             string
 301         }
 302 }
 303
 304 type StorageClassConfig struct {
 305         Default  bool
 306         Priority int
 307 }
 308
 309 type Volume struct {
 310         AccessViaHosts   map[URL]VolumeAccess
 311         ReadOnly         bool
 312         Replication      int
 313         StorageClasses   map[string]bool
 314         Driver           string
 315         DriverParameters json.RawMessage
 316 }
 317
 318 type S3VolumeDriverParameters struct {
 319         IAMRole            string
 320         AccessKeyID        string
 321         SecretAccessKey    string
 322         Endpoint           string
 323         Region             string
 324         Bucket             string
 325         LocationConstraint bool
 326         V2Signature        bool
 327         IndexPageSize      int
 328         ConnectTimeout     Duration
 329         ReadTimeout        Duration
 330         RaceWindow         Duration
 331         UnsafeDelete       bool
 332         PrefixLength       int
 333 }
 334
 335 type AzureVolumeDriverParameters struct {
 336         StorageAccountName   string
 337         StorageAccountKey    string
 338         StorageBaseURL       string
 339         ContainerName        string
 340         RequestTimeout       Duration
 341         ListBlobsRetryDelay  Duration
 342         ListBlobsMaxAttempts int
 343 }
 344
 345 type DirectoryVolumeDriverParameters struct {
 346         Root      string
 347         Serialize bool
 348 }
 349
 350 type VolumeAccess struct {
 351         ReadOnly bool
 352 }
 353
 354 type Services struct {
 355         Composer       Service
 356         Controller     Service
 357         DispatchCloud  Service
 358         DispatchLSF    Service
 359         DispatchSLURM  Service
 360         GitHTTP        Service
 361         GitSSH         Service
 362         Health         Service
 363         Keepbalance    Service
 364         Keepproxy      Service
 365         Keepstore      Service
 366         RailsAPI       Service
 367         WebDAVDownload Service
 368         WebDAV         Service
 369         WebShell       Service
 370         Websocket      Service
 371         Workbench1     Service
 372         Workbench2     Service
 373 }
 374
 375 type Service struct {
 376         InternalURLs map[URL]ServiceInstance
 377         ExternalURL  URL
 378 }
 379
 380 type TestUser struct {
 381         Email    string
 382         Password string
 383 }
 384
 385 // URL is a url.URL that is also usable as a JSON key/value.
 386 type URL url.URL
 387
 388 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 389 // used as a JSON key/value.
 390 func (su *URL) UnmarshalText(text []byte) error {
 391         u, err := url.Parse(string(text))
 392         if err == nil {
 393                 *su = URL(*u)
 394                 if su.Path == "" && su.Host != "" {
 395                         // http://example really means http://example/
 396                         su.Path = "/"
 397                 }
 398         }
 399         return err
 400 }
 401
 402 func (su URL) MarshalText() ([]byte, error) {
 403         return []byte(su.String()), nil
 404 }
 405
 406 func (su URL) String() string {
 407         return (*url.URL)(&su).String()
 408 }
 409
 410 type TLSVersion uint16
 411
 412 func (v TLSVersion) MarshalText() ([]byte, error) {
 413         switch v {
 414         case 0:
 415                 return []byte{}, nil
 416         case tls.VersionTLS10:
 417                 return []byte("1.0"), nil
 418         case tls.VersionTLS11:
 419                 return []byte("1.1"), nil
 420         case tls.VersionTLS12:
 421                 return []byte("1.2"), nil
 422         case tls.VersionTLS13:
 423                 return []byte("1.3"), nil
 424         default:
 425                 return nil, fmt.Errorf("unsupported TLSVersion %x", v)
 426         }
 427 }
 428
 429 func (v *TLSVersion) UnmarshalJSON(text []byte) error {
 430         if len(text) > 0 && text[0] == '"' {
 431                 var s string
 432                 err := json.Unmarshal(text, &s)
 433                 if err != nil {
 434                         return err
 435                 }
 436                 text = []byte(s)
 437         }
 438         switch string(text) {
 439         case "":
 440                 *v = 0
 441         case "1.0":
 442                 *v = tls.VersionTLS10
 443         case "1.1":
 444                 *v = tls.VersionTLS11
 445         case "1.2":
 446                 *v = tls.VersionTLS12
 447         case "1.3":
 448                 *v = tls.VersionTLS13
 449         default:
 450                 return fmt.Errorf("unsupported TLSVersion %q", text)
 451         }
 452         return nil
 453 }
 454
 455 type ServiceInstance struct {
 456         ListenURL  URL
 457         Rendezvous string `json:",omitempty"`
 458 }
 459
 460 type PostgreSQL struct {
 461         Connection     PostgreSQLConnection
 462         ConnectionPool int
 463 }
 464
 465 type PostgreSQLConnection map[string]string
 466
 467 type RemoteCluster struct {
 468         Host          string
 469         Proxy         bool
 470         Scheme        string
 471         Insecure      bool
 472         ActivateUsers bool
 473 }
 474
 475 type CUDAFeatures struct {
 476         DriverVersion      string
 477         HardwareCapability string
 478         DeviceCount        int
 479 }
 480
 481 type InstanceType struct {
 482         Name            string `json:"-"`
 483         ProviderType    string
 484         VCPUs           int
 485         RAM             ByteSize
 486         Scratch         ByteSize `json:"-"`
 487         IncludedScratch ByteSize
 488         AddedScratch    ByteSize
 489         Price           float64
 490         Preemptible     bool
 491         CUDA            CUDAFeatures
 492 }
 493
 494 type ContainersConfig struct {
 495         CloudVMs                      CloudVMsConfig
 496         CrunchRunCommand              string
 497         CrunchRunArgumentsList        []string
 498         DefaultKeepCacheRAM           ByteSize
 499         DispatchPrivateKey            string
 500         LogReuseDecisions             bool
 501         MaxDispatchAttempts           int
 502         MaxRetryAttempts              int
 503         MinRetryPeriod                Duration
 504         ReserveExtraRAM               ByteSize
 505         StaleLockTimeout              Duration
 506         SupportedDockerImageFormats   StringSet
 507         AlwaysUsePreemptibleInstances bool
 508         PreemptiblePriceFactor        float64
 509         RuntimeEngine                 string
 510         LocalKeepBlobBuffersPerVCPU   int
 511         LocalKeepLogsToContainerLog   string
 512
 513         JobsAPI struct {
 514                 Enable         string
 515                 GitInternalDir string
 516         }
 517         Logging struct {
 518                 MaxAge                       Duration
 519                 SweepInterval                Duration
 520                 LogBytesPerEvent             int
 521                 LogSecondsBetweenEvents      Duration
 522                 LogThrottlePeriod            Duration
 523                 LogThrottleBytes             int
 524                 LogThrottleLines             int
 525                 LimitLogBytesPerJob          int
 526                 LogPartialLineThrottlePeriod Duration
 527                 LogUpdatePeriod              Duration
 528                 LogUpdateSize                ByteSize
 529         }
 530         ShellAccess struct {
 531                 Admin bool
 532                 User  bool
 533         }
 534         SLURM struct {
 535                 PrioritySpread             int64
 536                 SbatchArgumentsList        []string
 537                 SbatchEnvironmentVariables map[string]string
 538                 Managed                    struct {
 539                         DNSServerConfDir       string
 540                         DNSServerConfTemplate  string
 541                         DNSServerReloadCommand string
 542                         DNSServerUpdateCommand string
 543                         ComputeNodeDomain      string
 544                         ComputeNodeNameservers StringSet
 545                         AssignNodeHostname     string
 546                 }
 547         }
 548         LSF struct {
 549                 BsubSudoUser      string
 550                 BsubArgumentsList []string
 551                 BsubCUDAArguments []string
 552         }
 553 }
 554
 555 type CloudVMsConfig struct {
 556         Enable bool
 557
 558         BootProbeCommand               string
 559         DeployRunnerBinary             string
 560         ImageID                        string
 561         MaxCloudOpsPerSecond           int
 562         MaxProbesPerSecond             int
 563         MaxConcurrentInstanceCreateOps int
 564         MaxInstances                   int
 565         SupervisorFraction             float64
 566         PollInterval                   Duration
 567         ProbeInterval                  Duration
 568         SSHPort                        string
 569         SyncInterval                   Duration
 570         TimeoutBooting                 Duration
 571         TimeoutIdle                    Duration
 572         TimeoutProbe                   Duration
 573         TimeoutShutdown                Duration
 574         TimeoutSignal                  Duration
 575         TimeoutStaleRunLock            Duration
 576         TimeoutTERM                    Duration
 577         ResourceTags                   map[string]string
 578         TagKeyPrefix                   string
 579
 580         Driver           string
 581         DriverParameters json.RawMessage
 582 }
 583
 584 type InstanceTypeMap map[string]InstanceType
 585
 586 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 587
 588 // UnmarshalJSON does special handling of InstanceTypes:
 589 //
 590 // - populate computed fields (Name and Scratch)
 591 //
 592 // - error out if InstancesTypes are populated as an array, which was
 593 // deprecated in Arvados 1.2.0
 594 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 595         fixup := func(t InstanceType) (InstanceType, error) {
 596                 if t.ProviderType == "" {
 597                         t.ProviderType = t.Name
 598                 }
 599                 // If t.Scratch is set in the configuration file, it will be ignored and overwritten.
 600                 // It will also generate a "deprecated or unknown config entry" warning.
 601                 t.Scratch = t.IncludedScratch + t.AddedScratch
 602                 return t, nil
 603         }
 604
 605         if len(data) > 0 && data[0] == '[' {
 606                 return fmt.Errorf("InstanceTypes must be specified as a map, not an array, see https://doc.arvados.org/admin/config.html")
 607         }
 608         var hash map[string]InstanceType
 609         err := json.Unmarshal(data, &hash)
 610         if err != nil {
 611                 return err
 612         }
 613         // Fill in Name field (and ProviderType field, if not
 614         // specified) using hash key.
 615         *it = InstanceTypeMap(hash)
 616         for name, t := range *it {
 617                 t.Name = name
 618                 t, err := fixup(t)
 619                 if err != nil {
 620                         return err
 621                 }
 622                 (*it)[name] = t
 623         }
 624         return nil
 625 }
 626
 627 type StringSet map[string]struct{}
 628
 629 // UnmarshalJSON handles old config files that provide an array of
 630 // instance types instead of a hash.
 631 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 632         if len(data) > 0 && data[0] == '[' {
 633                 var arr []string
 634                 err := json.Unmarshal(data, &arr)
 635                 if err != nil {
 636                         return err
 637                 }
 638                 if len(arr) == 0 {
 639                         *ss = nil
 640                         return nil
 641                 }
 642                 *ss = make(map[string]struct{}, len(arr))
 643                 for _, t := range arr {
 644                         (*ss)[t] = struct{}{}
 645                 }
 646                 return nil
 647         }
 648         var hash map[string]struct{}
 649         err := json.Unmarshal(data, &hash)
 650         if err != nil {
 651                 return err
 652         }
 653         *ss = make(map[string]struct{}, len(hash))
 654         for t := range hash {
 655                 (*ss)[t] = struct{}{}
 656         }
 657
 658         return nil
 659 }
 660
 661 type ServiceName string
 662
 663 const (
 664         ServiceNameController    ServiceName = "arvados-controller"
 665         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 666         ServiceNameDispatchLSF   ServiceName = "arvados-dispatch-lsf"
 667         ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm"
 668         ServiceNameGitHTTP       ServiceName = "arvados-git-httpd"
 669         ServiceNameHealth        ServiceName = "arvados-health"
 670         ServiceNameKeepbalance   ServiceName = "keep-balance"
 671         ServiceNameKeepproxy     ServiceName = "keepproxy"
 672         ServiceNameKeepstore     ServiceName = "keepstore"
 673         ServiceNameKeepweb       ServiceName = "keep-web"
 674         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 675         ServiceNameWebsocket     ServiceName = "arvados-ws"
 676         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 677         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 678 )
 679
 680 // Map returns all services as a map, suitable for iterating over all
 681 // services or looking up a service by name.
 682 func (svcs Services) Map() map[ServiceName]Service {
 683         return map[ServiceName]Service{
 684                 ServiceNameController:    svcs.Controller,
 685                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 686                 ServiceNameDispatchLSF:   svcs.DispatchLSF,
 687                 ServiceNameDispatchSLURM: svcs.DispatchSLURM,
 688                 ServiceNameGitHTTP:       svcs.GitHTTP,
 689                 ServiceNameHealth:        svcs.Health,
 690                 ServiceNameKeepbalance:   svcs.Keepbalance,
 691                 ServiceNameKeepproxy:     svcs.Keepproxy,
 692                 ServiceNameKeepstore:     svcs.Keepstore,
 693                 ServiceNameKeepweb:       svcs.WebDAV,
 694                 ServiceNameRailsAPI:      svcs.RailsAPI,
 695                 ServiceNameWebsocket:     svcs.Websocket,
 696                 ServiceNameWorkbench1:    svcs.Workbench1,
 697                 ServiceNameWorkbench2:    svcs.Workbench2,
 698         }
 699 }