sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "crypto/tls"
   9         "encoding/json"
  10         "errors"
  11         "fmt"
  12         "net/url"
  13         "os"
  14         "time"
  15
  16         "git.arvados.org/arvados.git/sdk/go/config"
  17 )
  18
  19 var DefaultConfigFile = func() string {
  20         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  21                 return path
  22         }
  23         return "/etc/arvados/config.yml"
  24 }()
  25
  26 type Config struct {
  27         Clusters         map[string]Cluster
  28         AutoReloadConfig bool
  29         SourceTimestamp  time.Time
  30         SourceSHA256     string
  31 }
  32
  33 // GetConfig returns the current system config, loading it from
  34 // configFile if needed.
  35 func GetConfig(configFile string) (*Config, error) {
  36         var cfg Config
  37         err := config.LoadFile(&cfg, configFile)
  38         return &cfg, err
  39 }
  40
  41 // GetCluster returns the cluster ID and config for the given
  42 // cluster, or the default/only configured cluster if clusterID is "".
  43 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  44         if clusterID == "" {
  45                 if len(sc.Clusters) == 0 {
  46                         return nil, fmt.Errorf("no clusters configured")
  47                 } else if len(sc.Clusters) > 1 {
  48                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  49                 } else {
  50                         for id, cc := range sc.Clusters {
  51                                 cc.ClusterID = id
  52                                 return &cc, nil
  53                         }
  54                 }
  55         }
  56         cc, ok := sc.Clusters[clusterID]
  57         if !ok {
  58                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  59         }
  60         cc.ClusterID = clusterID
  61         return &cc, nil
  62 }
  63
  64 type WebDAVCacheConfig struct {
  65         TTL                Duration
  66         MaxBlockEntries    int
  67         MaxCollectionBytes int64
  68         MaxSessions        int
  69 }
  70
  71 type UploadDownloadPermission struct {
  72         Upload   bool
  73         Download bool
  74 }
  75
  76 type UploadDownloadRolePermissions struct {
  77         User  UploadDownloadPermission
  78         Admin UploadDownloadPermission
  79 }
  80
  81 type ManagedProperties map[string]struct {
  82         Value     interface{}
  83         Function  string
  84         Protected bool
  85 }
  86
  87 type Cluster struct {
  88         ClusterID       string `json:"-"`
  89         ManagementToken string
  90         SystemRootToken string
  91         Services        Services
  92         InstanceTypes   InstanceTypeMap
  93         Containers      ContainersConfig
  94         RemoteClusters  map[string]RemoteCluster
  95         PostgreSQL      PostgreSQL
  96
  97         API struct {
  98                 AsyncPermissionsUpdateInterval   Duration
  99                 DisabledAPIs                     StringSet
 100                 MaxIndexDatabaseRead             int
 101                 MaxItemsPerResponse              int
 102                 MaxConcurrentRequests            int
 103                 LogCreateRequestFraction         float64
 104                 MaxKeepBlobBuffers               int
 105                 MaxRequestAmplification          int
 106                 MaxRequestSize                   int
 107                 MaxTokenLifetime                 Duration
 108                 RequestTimeout                   Duration
 109                 SendTimeout                      Duration
 110                 WebsocketClientEventQueue        int
 111                 WebsocketServerEventQueue        int
 112                 KeepServiceRequestTimeout        Duration
 113                 VocabularyPath                   string
 114                 FreezeProjectRequiresDescription bool
 115                 FreezeProjectRequiresProperties  StringSet
 116                 UnfreezeProjectRequiresAdmin     bool
 117                 LockBeforeUpdate                 bool
 118         }
 119         AuditLogs struct {
 120                 MaxAge             Duration
 121                 MaxDeleteBatch     int
 122                 UnloggedAttributes StringSet
 123         }
 124         Collections struct {
 125                 BlobSigning                  bool
 126                 BlobSigningKey               string
 127                 BlobSigningTTL               Duration
 128                 BlobTrash                    bool
 129                 BlobTrashLifetime            Duration
 130                 BlobTrashCheckInterval       Duration
 131                 BlobTrashConcurrency         int
 132                 BlobDeleteConcurrency        int
 133                 BlobReplicateConcurrency     int
 134                 CollectionVersioning         bool
 135                 DefaultTrashLifetime         Duration
 136                 DefaultReplication           int
 137                 ManagedProperties            ManagedProperties
 138                 PreserveVersionIfIdle        Duration
 139                 TrashSweepInterval           Duration
 140                 TrustAllContent              bool
 141                 ForwardSlashNameSubstitution string
 142                 S3FolderObjects              bool
 143
 144                 BlobMissingReport        string
 145                 BalancePeriod            Duration
 146                 BalanceCollectionBatch   int
 147                 BalanceCollectionBuffers int
 148                 BalanceTimeout           Duration
 149                 BalanceUpdateLimit       int
 150
 151                 WebDAVCache WebDAVCacheConfig
 152
 153                 KeepproxyPermission UploadDownloadRolePermissions
 154                 WebDAVPermission    UploadDownloadRolePermissions
 155                 WebDAVLogEvents     bool
 156         }
 157         Git struct {
 158                 GitCommand   string
 159                 GitoliteHome string
 160                 Repositories string
 161         }
 162         Login struct {
 163                 LDAP struct {
 164                         Enable             bool
 165                         URL                URL
 166                         StartTLS           bool
 167                         InsecureTLS        bool
 168                         MinTLSVersion      TLSVersion
 169                         StripDomain        string
 170                         AppendDomain       string
 171                         SearchAttribute    string
 172                         SearchBindUser     string
 173                         SearchBindPassword string
 174                         SearchBase         string
 175                         SearchFilters      string
 176                         EmailAttribute     string
 177                         UsernameAttribute  string
 178                 }
 179                 Google struct {
 180                         Enable                          bool
 181                         ClientID                        string
 182                         ClientSecret                    string
 183                         AlternateEmailAddresses         bool
 184                         AuthenticationRequestParameters map[string]string
 185                 }
 186                 OpenIDConnect struct {
 187                         Enable                          bool
 188                         Issuer                          string
 189                         ClientID                        string
 190                         ClientSecret                    string
 191                         EmailClaim                      string
 192                         EmailVerifiedClaim              string
 193                         UsernameClaim                   string
 194                         AcceptAccessToken               bool
 195                         AcceptAccessTokenScope          string
 196                         AuthenticationRequestParameters map[string]string
 197                 }
 198                 PAM struct {
 199                         Enable             bool
 200                         Service            string
 201                         DefaultEmailDomain string
 202                 }
 203                 Test struct {
 204                         Enable bool
 205                         Users  map[string]TestUser
 206                 }
 207                 LoginCluster         string
 208                 RemoteTokenRefresh   Duration
 209                 TokenLifetime        Duration
 210                 TrustedClients       map[URL]struct{}
 211                 TrustPrivateNetworks bool
 212                 IssueTrustedTokens   bool
 213         }
 214         Mail struct {
 215                 MailchimpAPIKey                string
 216                 MailchimpListID                string
 217                 SendUserSetupNotificationEmail bool
 218                 IssueReporterEmailFrom         string
 219                 IssueReporterEmailTo           string
 220                 SupportEmailAddress            string
 221                 EmailFrom                      string
 222         }
 223         SystemLogs struct {
 224                 LogLevel                string
 225                 Format                  string
 226                 MaxRequestLogParamsSize int
 227         }
 228         TLS struct {
 229                 Certificate string
 230                 Key         string
 231                 Insecure    bool
 232                 ACME        struct {
 233                         Server string
 234                 }
 235         }
 236         Users struct {
 237                 ActivatedUsersAreVisibleToOthers      bool
 238                 AnonymousUserToken                    string
 239                 AdminNotifierEmailFrom                string
 240                 AutoAdminFirstUser                    bool
 241                 AutoAdminUserWithEmail                string
 242                 AutoSetupNewUsers                     bool
 243                 AutoSetupNewUsersWithRepository       bool
 244                 AutoSetupNewUsersWithVmUUID           string
 245                 AutoSetupUsernameBlacklist            StringSet
 246                 EmailSubjectPrefix                    string
 247                 NewInactiveUserNotificationRecipients StringSet
 248                 NewUserNotificationRecipients         StringSet
 249                 NewUsersAreActive                     bool
 250                 UserNotifierEmailFrom                 string
 251                 UserNotifierEmailBcc                  StringSet
 252                 UserProfileNotificationAddress        string
 253                 PreferDomainForUsername               string
 254                 UserSetupMailText                     string
 255                 RoleGroupsVisibleToAll                bool
 256                 CanCreateRoleGroups                   bool
 257                 ActivityLoggingPeriod                 Duration
 258         }
 259         StorageClasses map[string]StorageClassConfig
 260         Volumes        map[string]Volume
 261         Workbench      struct {
 262                 ActivationContactLink            string
 263                 APIClientConnectTimeout          Duration
 264                 APIClientReceiveTimeout          Duration
 265                 APIResponseCompression           bool
 266                 ApplicationMimetypesWithViewIcon StringSet
 267                 ArvadosDocsite                   string
 268                 ArvadosPublicDataDocURL          string
 269                 DefaultOpenIdPrefix              string
 270                 DisableSharingURLsUI             bool
 271                 EnableGettingStartedPopup        bool
 272                 EnablePublicProjectsPage         bool
 273                 FileViewersConfigURL             string
 274                 LogViewerMaxBytes                ByteSize
 275                 MultiSiteSearch                  string
 276                 ProfilingEnabled                 bool
 277                 Repositories                     bool
 278                 RepositoryCache                  string
 279                 RunningJobLogRecordsToFetch      int
 280                 SecretKeyBase                    string
 281                 ShowRecentCollectionsOnDashboard bool
 282                 ShowUserAgreementInline          bool
 283                 ShowUserNotifications            bool
 284                 SiteName                         string
 285                 Theme                            string
 286                 UserProfileFormFields            map[string]struct {
 287                         Type                 string
 288                         FormFieldTitle       string
 289                         FormFieldDescription string
 290                         Required             bool
 291                         Position             int
 292                         Options              map[string]struct{}
 293                 }
 294                 UserProfileFormMessage string
 295                 WelcomePageHTML        string
 296                 InactivePageHTML       string
 297                 SSHHelpPageHTML        string
 298                 SSHHelpHostSuffix      string
 299                 IdleTimeout            Duration
 300                 BannerUUID             string
 301         }
 302 }
 303
 304 type StorageClassConfig struct {
 305         Default  bool
 306         Priority int
 307 }
 308
 309 type Volume struct {
 310         AccessViaHosts   map[URL]VolumeAccess
 311         ReadOnly         bool
 312         Replication      int
 313         StorageClasses   map[string]bool
 314         Driver           string
 315         DriverParameters json.RawMessage
 316 }
 317
 318 type S3VolumeDriverParameters struct {
 319         IAMRole            string
 320         AccessKeyID        string
 321         SecretAccessKey    string
 322         Endpoint           string
 323         Region             string
 324         Bucket             string
 325         LocationConstraint bool
 326         V2Signature        bool
 327         UseAWSS3v2Driver   bool
 328         IndexPageSize      int
 329         ConnectTimeout     Duration
 330         ReadTimeout        Duration
 331         RaceWindow         Duration
 332         UnsafeDelete       bool
 333         PrefixLength       int
 334 }
 335
 336 type AzureVolumeDriverParameters struct {
 337         StorageAccountName   string
 338         StorageAccountKey    string
 339         StorageBaseURL       string
 340         ContainerName        string
 341         RequestTimeout       Duration
 342         ListBlobsRetryDelay  Duration
 343         ListBlobsMaxAttempts int
 344 }
 345
 346 type DirectoryVolumeDriverParameters struct {
 347         Root      string
 348         Serialize bool
 349 }
 350
 351 type VolumeAccess struct {
 352         ReadOnly bool
 353 }
 354
 355 type Services struct {
 356         Composer       Service
 357         Controller     Service
 358         DispatchCloud  Service
 359         DispatchLSF    Service
 360         DispatchSLURM  Service
 361         GitHTTP        Service
 362         GitSSH         Service
 363         Health         Service
 364         Keepbalance    Service
 365         Keepproxy      Service
 366         Keepstore      Service
 367         RailsAPI       Service
 368         WebDAVDownload Service
 369         WebDAV         Service
 370         WebShell       Service
 371         Websocket      Service
 372         Workbench1     Service
 373         Workbench2     Service
 374 }
 375
 376 type Service struct {
 377         InternalURLs map[URL]ServiceInstance
 378         ExternalURL  URL
 379 }
 380
 381 type TestUser struct {
 382         Email    string
 383         Password string
 384 }
 385
 386 // URL is a url.URL that is also usable as a JSON key/value.
 387 type URL url.URL
 388
 389 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 390 // used as a JSON key/value.
 391 func (su *URL) UnmarshalText(text []byte) error {
 392         u, err := url.Parse(string(text))
 393         if err == nil {
 394                 *su = URL(*u)
 395                 if su.Path == "" && su.Host != "" {
 396                         // http://example really means http://example/
 397                         su.Path = "/"
 398                 }
 399         }
 400         return err
 401 }
 402
 403 func (su URL) MarshalText() ([]byte, error) {
 404         return []byte(su.String()), nil
 405 }
 406
 407 func (su URL) String() string {
 408         return (*url.URL)(&su).String()
 409 }
 410
 411 type TLSVersion uint16
 412
 413 func (v TLSVersion) MarshalText() ([]byte, error) {
 414         switch v {
 415         case 0:
 416                 return []byte{}, nil
 417         case tls.VersionTLS10:
 418                 return []byte("1.0"), nil
 419         case tls.VersionTLS11:
 420                 return []byte("1.1"), nil
 421         case tls.VersionTLS12:
 422                 return []byte("1.2"), nil
 423         case tls.VersionTLS13:
 424                 return []byte("1.3"), nil
 425         default:
 426                 return nil, fmt.Errorf("unsupported TLSVersion %x", v)
 427         }
 428 }
 429
 430 func (v *TLSVersion) UnmarshalJSON(text []byte) error {
 431         if len(text) > 0 && text[0] == '"' {
 432                 var s string
 433                 err := json.Unmarshal(text, &s)
 434                 if err != nil {
 435                         return err
 436                 }
 437                 text = []byte(s)
 438         }
 439         switch string(text) {
 440         case "":
 441                 *v = 0
 442         case "1.0":
 443                 *v = tls.VersionTLS10
 444         case "1.1":
 445                 *v = tls.VersionTLS11
 446         case "1.2":
 447                 *v = tls.VersionTLS12
 448         case "1.3":
 449                 *v = tls.VersionTLS13
 450         default:
 451                 return fmt.Errorf("unsupported TLSVersion %q", text)
 452         }
 453         return nil
 454 }
 455
 456 type ServiceInstance struct {
 457         ListenURL  URL
 458         Rendezvous string `json:",omitempty"`
 459 }
 460
 461 type PostgreSQL struct {
 462         Connection     PostgreSQLConnection
 463         ConnectionPool int
 464 }
 465
 466 type PostgreSQLConnection map[string]string
 467
 468 type RemoteCluster struct {
 469         Host          string
 470         Proxy         bool
 471         Scheme        string
 472         Insecure      bool
 473         ActivateUsers bool
 474 }
 475
 476 type CUDAFeatures struct {
 477         DriverVersion      string
 478         HardwareCapability string
 479         DeviceCount        int
 480 }
 481
 482 type InstanceType struct {
 483         Name            string `json:"-"`
 484         ProviderType    string
 485         VCPUs           int
 486         RAM             ByteSize
 487         Scratch         ByteSize `json:"-"`
 488         IncludedScratch ByteSize
 489         AddedScratch    ByteSize
 490         Price           float64
 491         Preemptible     bool
 492         CUDA            CUDAFeatures
 493 }
 494
 495 type ContainersConfig struct {
 496         CloudVMs                      CloudVMsConfig
 497         CrunchRunCommand              string
 498         CrunchRunArgumentsList        []string
 499         DefaultKeepCacheRAM           ByteSize
 500         DispatchPrivateKey            string
 501         LogReuseDecisions             bool
 502         MaxDispatchAttempts           int
 503         MaxRetryAttempts              int
 504         MinRetryPeriod                Duration
 505         ReserveExtraRAM               ByteSize
 506         StaleLockTimeout              Duration
 507         SupportedDockerImageFormats   StringSet
 508         AlwaysUsePreemptibleInstances bool
 509         PreemptiblePriceFactor        float64
 510         RuntimeEngine                 string
 511         LocalKeepBlobBuffersPerVCPU   int
 512         LocalKeepLogsToContainerLog   string
 513
 514         JobsAPI struct {
 515                 Enable         string
 516                 GitInternalDir string
 517         }
 518         Logging struct {
 519                 MaxAge                       Duration
 520                 SweepInterval                Duration
 521                 LogBytesPerEvent             int
 522                 LogSecondsBetweenEvents      Duration
 523                 LogThrottlePeriod            Duration
 524                 LogThrottleBytes             int
 525                 LogThrottleLines             int
 526                 LimitLogBytesPerJob          int
 527                 LogPartialLineThrottlePeriod Duration
 528                 LogUpdatePeriod              Duration
 529                 LogUpdateSize                ByteSize
 530         }
 531         ShellAccess struct {
 532                 Admin bool
 533                 User  bool
 534         }
 535         SLURM struct {
 536                 PrioritySpread             int64
 537                 SbatchArgumentsList        []string
 538                 SbatchEnvironmentVariables map[string]string
 539                 Managed                    struct {
 540                         DNSServerConfDir       string
 541                         DNSServerConfTemplate  string
 542                         DNSServerReloadCommand string
 543                         DNSServerUpdateCommand string
 544                         ComputeNodeDomain      string
 545                         ComputeNodeNameservers StringSet
 546                         AssignNodeHostname     string
 547                 }
 548         }
 549         LSF struct {
 550                 BsubSudoUser      string
 551                 BsubArgumentsList []string
 552                 BsubCUDAArguments []string
 553         }
 554 }
 555
 556 type CloudVMsConfig struct {
 557         Enable bool
 558
 559         BootProbeCommand               string
 560         DeployRunnerBinary             string
 561         ImageID                        string
 562         MaxCloudOpsPerSecond           int
 563         MaxProbesPerSecond             int
 564         MaxConcurrentInstanceCreateOps int
 565         MaxInstances                   int
 566         SupervisorFraction             float64
 567         PollInterval                   Duration
 568         ProbeInterval                  Duration
 569         SSHPort                        string
 570         SyncInterval                   Duration
 571         TimeoutBooting                 Duration
 572         TimeoutIdle                    Duration
 573         TimeoutProbe                   Duration
 574         TimeoutShutdown                Duration
 575         TimeoutSignal                  Duration
 576         TimeoutStaleRunLock            Duration
 577         TimeoutTERM                    Duration
 578         ResourceTags                   map[string]string
 579         TagKeyPrefix                   string
 580
 581         Driver           string
 582         DriverParameters json.RawMessage
 583 }
 584
 585 type InstanceTypeMap map[string]InstanceType
 586
 587 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 588
 589 // UnmarshalJSON does special handling of InstanceTypes:
 590 //
 591 // - populate computed fields (Name and Scratch)
 592 //
 593 // - error out if InstancesTypes are populated as an array, which was
 594 // deprecated in Arvados 1.2.0
 595 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 596         fixup := func(t InstanceType) (InstanceType, error) {
 597                 if t.ProviderType == "" {
 598                         t.ProviderType = t.Name
 599                 }
 600                 // If t.Scratch is set in the configuration file, it will be ignored and overwritten.
 601                 // It will also generate a "deprecated or unknown config entry" warning.
 602                 t.Scratch = t.IncludedScratch + t.AddedScratch
 603                 return t, nil
 604         }
 605
 606         if len(data) > 0 && data[0] == '[' {
 607                 return fmt.Errorf("InstanceTypes must be specified as a map, not an array, see https://doc.arvados.org/admin/config.html")
 608         }
 609         var hash map[string]InstanceType
 610         err := json.Unmarshal(data, &hash)
 611         if err != nil {
 612                 return err
 613         }
 614         // Fill in Name field (and ProviderType field, if not
 615         // specified) using hash key.
 616         *it = InstanceTypeMap(hash)
 617         for name, t := range *it {
 618                 t.Name = name
 619                 t, err := fixup(t)
 620                 if err != nil {
 621                         return err
 622                 }
 623                 (*it)[name] = t
 624         }
 625         return nil
 626 }
 627
 628 type StringSet map[string]struct{}
 629
 630 // UnmarshalJSON handles old config files that provide an array of
 631 // instance types instead of a hash.
 632 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 633         if len(data) > 0 && data[0] == '[' {
 634                 var arr []string
 635                 err := json.Unmarshal(data, &arr)
 636                 if err != nil {
 637                         return err
 638                 }
 639                 if len(arr) == 0 {
 640                         *ss = nil
 641                         return nil
 642                 }
 643                 *ss = make(map[string]struct{}, len(arr))
 644                 for _, t := range arr {
 645                         (*ss)[t] = struct{}{}
 646                 }
 647                 return nil
 648         }
 649         var hash map[string]struct{}
 650         err := json.Unmarshal(data, &hash)
 651         if err != nil {
 652                 return err
 653         }
 654         *ss = make(map[string]struct{}, len(hash))
 655         for t := range hash {
 656                 (*ss)[t] = struct{}{}
 657         }
 658
 659         return nil
 660 }
 661
 662 type ServiceName string
 663
 664 const (
 665         ServiceNameController    ServiceName = "arvados-controller"
 666         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 667         ServiceNameDispatchLSF   ServiceName = "arvados-dispatch-lsf"
 668         ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm"
 669         ServiceNameGitHTTP       ServiceName = "arvados-git-httpd"
 670         ServiceNameHealth        ServiceName = "arvados-health"
 671         ServiceNameKeepbalance   ServiceName = "keep-balance"
 672         ServiceNameKeepproxy     ServiceName = "keepproxy"
 673         ServiceNameKeepstore     ServiceName = "keepstore"
 674         ServiceNameKeepweb       ServiceName = "keep-web"
 675         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 676         ServiceNameWebsocket     ServiceName = "arvados-ws"
 677         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 678         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 679 )
 680
 681 // Map returns all services as a map, suitable for iterating over all
 682 // services or looking up a service by name.
 683 func (svcs Services) Map() map[ServiceName]Service {
 684         return map[ServiceName]Service{
 685                 ServiceNameController:    svcs.Controller,
 686                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 687                 ServiceNameDispatchLSF:   svcs.DispatchLSF,
 688                 ServiceNameDispatchSLURM: svcs.DispatchSLURM,
 689                 ServiceNameGitHTTP:       svcs.GitHTTP,
 690                 ServiceNameHealth:        svcs.Health,
 691                 ServiceNameKeepbalance:   svcs.Keepbalance,
 692                 ServiceNameKeepproxy:     svcs.Keepproxy,
 693                 ServiceNameKeepstore:     svcs.Keepstore,
 694                 ServiceNameKeepweb:       svcs.WebDAV,
 695                 ServiceNameRailsAPI:      svcs.RailsAPI,
 696                 ServiceNameWebsocket:     svcs.Websocket,
 697                 ServiceNameWorkbench1:    svcs.Workbench1,
 698                 ServiceNameWorkbench2:    svcs.Workbench2,
 699         }
 700 }