sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "crypto/tls"
   9         "encoding/json"
  10         "errors"
  11         "fmt"
  12         "net/url"
  13         "os"
  14         "time"
  15
  16         "git.arvados.org/arvados.git/sdk/go/config"
  17 )
  18
  19 var DefaultConfigFile = func() string {
  20         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  21                 return path
  22         }
  23         return "/etc/arvados/config.yml"
  24 }()
  25
  26 type Config struct {
  27         Clusters         map[string]Cluster
  28         AutoReloadConfig bool
  29         SourceTimestamp  time.Time
  30         SourceSHA256     string
  31 }
  32
  33 // GetConfig returns the current system config, loading it from
  34 // configFile if needed.
  35 func GetConfig(configFile string) (*Config, error) {
  36         var cfg Config
  37         err := config.LoadFile(&cfg, configFile)
  38         return &cfg, err
  39 }
  40
  41 // GetCluster returns the cluster ID and config for the given
  42 // cluster, or the default/only configured cluster if clusterID is "".
  43 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  44         if clusterID == "" {
  45                 if len(sc.Clusters) == 0 {
  46                         return nil, fmt.Errorf("no clusters configured")
  47                 } else if len(sc.Clusters) > 1 {
  48                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  49                 } else {
  50                         for id, cc := range sc.Clusters {
  51                                 cc.ClusterID = id
  52                                 return &cc, nil
  53                         }
  54                 }
  55         }
  56         cc, ok := sc.Clusters[clusterID]
  57         if !ok {
  58                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  59         }
  60         cc.ClusterID = clusterID
  61         return &cc, nil
  62 }
  63
  64 type WebDAVCacheConfig struct {
  65         TTL                Duration
  66         MaxBlockEntries    int
  67         MaxCollectionBytes int64
  68         MaxSessions        int
  69 }
  70
  71 type UploadDownloadPermission struct {
  72         Upload   bool
  73         Download bool
  74 }
  75
  76 type UploadDownloadRolePermissions struct {
  77         User  UploadDownloadPermission
  78         Admin UploadDownloadPermission
  79 }
  80
  81 type ManagedProperties map[string]struct {
  82         Value     interface{}
  83         Function  string
  84         Protected bool
  85 }
  86
  87 type Cluster struct {
  88         ClusterID       string `json:"-"`
  89         ManagementToken string
  90         SystemRootToken string
  91         Services        Services
  92         InstanceTypes   InstanceTypeMap
  93         Containers      ContainersConfig
  94         RemoteClusters  map[string]RemoteCluster
  95         PostgreSQL      PostgreSQL
  96
  97         API struct {
  98                 AsyncPermissionsUpdateInterval   Duration
  99                 DisabledAPIs                     StringSet
 100                 MaxIndexDatabaseRead             int
 101                 MaxItemsPerResponse              int
 102                 MaxConcurrentRequests            int
 103                 MaxKeepBlobBuffers               int
 104                 MaxRequestAmplification          int
 105                 MaxRequestSize                   int
 106                 MaxTokenLifetime                 Duration
 107                 RequestTimeout                   Duration
 108                 SendTimeout                      Duration
 109                 WebsocketClientEventQueue        int
 110                 WebsocketServerEventQueue        int
 111                 KeepServiceRequestTimeout        Duration
 112                 VocabularyPath                   string
 113                 FreezeProjectRequiresDescription bool
 114                 FreezeProjectRequiresProperties  StringSet
 115                 UnfreezeProjectRequiresAdmin     bool
 116                 LockBeforeUpdate                 bool
 117         }
 118         AuditLogs struct {
 119                 MaxAge             Duration
 120                 MaxDeleteBatch     int
 121                 UnloggedAttributes StringSet
 122         }
 123         Collections struct {
 124                 BlobSigning                  bool
 125                 BlobSigningKey               string
 126                 BlobSigningTTL               Duration
 127                 BlobTrash                    bool
 128                 BlobTrashLifetime            Duration
 129                 BlobTrashCheckInterval       Duration
 130                 BlobTrashConcurrency         int
 131                 BlobDeleteConcurrency        int
 132                 BlobReplicateConcurrency     int
 133                 CollectionVersioning         bool
 134                 DefaultTrashLifetime         Duration
 135                 DefaultReplication           int
 136                 ManagedProperties            ManagedProperties
 137                 PreserveVersionIfIdle        Duration
 138                 TrashSweepInterval           Duration
 139                 TrustAllContent              bool
 140                 ForwardSlashNameSubstitution string
 141                 S3FolderObjects              bool
 142
 143                 BlobMissingReport        string
 144                 BalancePeriod            Duration
 145                 BalanceCollectionBatch   int
 146                 BalanceCollectionBuffers int
 147                 BalanceTimeout           Duration
 148                 BalanceUpdateLimit       int
 149
 150                 WebDAVCache WebDAVCacheConfig
 151
 152                 KeepproxyPermission UploadDownloadRolePermissions
 153                 WebDAVPermission    UploadDownloadRolePermissions
 154                 WebDAVLogEvents     bool
 155         }
 156         Git struct {
 157                 GitCommand   string
 158                 GitoliteHome string
 159                 Repositories string
 160         }
 161         Login struct {
 162                 LDAP struct {
 163                         Enable             bool
 164                         URL                URL
 165                         StartTLS           bool
 166                         InsecureTLS        bool
 167                         MinTLSVersion      TLSVersion
 168                         StripDomain        string
 169                         AppendDomain       string
 170                         SearchAttribute    string
 171                         SearchBindUser     string
 172                         SearchBindPassword string
 173                         SearchBase         string
 174                         SearchFilters      string
 175                         EmailAttribute     string
 176                         UsernameAttribute  string
 177                 }
 178                 Google struct {
 179                         Enable                          bool
 180                         ClientID                        string
 181                         ClientSecret                    string
 182                         AlternateEmailAddresses         bool
 183                         AuthenticationRequestParameters map[string]string
 184                 }
 185                 OpenIDConnect struct {
 186                         Enable                          bool
 187                         Issuer                          string
 188                         ClientID                        string
 189                         ClientSecret                    string
 190                         EmailClaim                      string
 191                         EmailVerifiedClaim              string
 192                         UsernameClaim                   string
 193                         AcceptAccessToken               bool
 194                         AcceptAccessTokenScope          string
 195                         AuthenticationRequestParameters map[string]string
 196                 }
 197                 PAM struct {
 198                         Enable             bool
 199                         Service            string
 200                         DefaultEmailDomain string
 201                 }
 202                 Test struct {
 203                         Enable bool
 204                         Users  map[string]TestUser
 205                 }
 206                 LoginCluster         string
 207                 RemoteTokenRefresh   Duration
 208                 TokenLifetime        Duration
 209                 TrustedClients       map[URL]struct{}
 210                 TrustPrivateNetworks bool
 211                 IssueTrustedTokens   bool
 212         }
 213         Mail struct {
 214                 MailchimpAPIKey                string
 215                 MailchimpListID                string
 216                 SendUserSetupNotificationEmail bool
 217                 IssueReporterEmailFrom         string
 218                 IssueReporterEmailTo           string
 219                 SupportEmailAddress            string
 220                 EmailFrom                      string
 221         }
 222         SystemLogs struct {
 223                 LogLevel                string
 224                 Format                  string
 225                 MaxRequestLogParamsSize int
 226         }
 227         TLS struct {
 228                 Certificate string
 229                 Key         string
 230                 Insecure    bool
 231                 ACME        struct {
 232                         Server string
 233                 }
 234         }
 235         Users struct {
 236                 ActivatedUsersAreVisibleToOthers      bool
 237                 AnonymousUserToken                    string
 238                 AdminNotifierEmailFrom                string
 239                 AutoAdminFirstUser                    bool
 240                 AutoAdminUserWithEmail                string
 241                 AutoSetupNewUsers                     bool
 242                 AutoSetupNewUsersWithRepository       bool
 243                 AutoSetupNewUsersWithVmUUID           string
 244                 AutoSetupUsernameBlacklist            StringSet
 245                 EmailSubjectPrefix                    string
 246                 NewInactiveUserNotificationRecipients StringSet
 247                 NewUserNotificationRecipients         StringSet
 248                 NewUsersAreActive                     bool
 249                 UserNotifierEmailFrom                 string
 250                 UserNotifierEmailBcc                  StringSet
 251                 UserProfileNotificationAddress        string
 252                 PreferDomainForUsername               string
 253                 UserSetupMailText                     string
 254                 RoleGroupsVisibleToAll                bool
 255                 CanCreateRoleGroups                   bool
 256                 ActivityLoggingPeriod                 Duration
 257         }
 258         StorageClasses map[string]StorageClassConfig
 259         Volumes        map[string]Volume
 260         Workbench      struct {
 261                 ActivationContactLink            string
 262                 APIClientConnectTimeout          Duration
 263                 APIClientReceiveTimeout          Duration
 264                 APIResponseCompression           bool
 265                 ApplicationMimetypesWithViewIcon StringSet
 266                 ArvadosDocsite                   string
 267                 ArvadosPublicDataDocURL          string
 268                 DefaultOpenIdPrefix              string
 269                 DisableSharingURLsUI             bool
 270                 EnableGettingStartedPopup        bool
 271                 EnablePublicProjectsPage         bool
 272                 FileViewersConfigURL             string
 273                 LogViewerMaxBytes                ByteSize
 274                 MultiSiteSearch                  string
 275                 ProfilingEnabled                 bool
 276                 Repositories                     bool
 277                 RepositoryCache                  string
 278                 RunningJobLogRecordsToFetch      int
 279                 SecretKeyBase                    string
 280                 ShowRecentCollectionsOnDashboard bool
 281                 ShowUserAgreementInline          bool
 282                 ShowUserNotifications            bool
 283                 SiteName                         string
 284                 Theme                            string
 285                 UserProfileFormFields            map[string]struct {
 286                         Type                 string
 287                         FormFieldTitle       string
 288                         FormFieldDescription string
 289                         Required             bool
 290                         Position             int
 291                         Options              map[string]struct{}
 292                 }
 293                 UserProfileFormMessage string
 294                 WelcomePageHTML        string
 295                 InactivePageHTML       string
 296                 SSHHelpPageHTML        string
 297                 SSHHelpHostSuffix      string
 298                 IdleTimeout            Duration
 299                 BannerURL              string
 300         }
 301 }
 302
 303 type StorageClassConfig struct {
 304         Default  bool
 305         Priority int
 306 }
 307
 308 type Volume struct {
 309         AccessViaHosts   map[URL]VolumeAccess
 310         ReadOnly         bool
 311         Replication      int
 312         StorageClasses   map[string]bool
 313         Driver           string
 314         DriverParameters json.RawMessage
 315 }
 316
 317 type S3VolumeDriverParameters struct {
 318         IAMRole            string
 319         AccessKeyID        string
 320         SecretAccessKey    string
 321         Endpoint           string
 322         Region             string
 323         Bucket             string
 324         LocationConstraint bool
 325         V2Signature        bool
 326         UseAWSS3v2Driver   bool
 327         IndexPageSize      int
 328         ConnectTimeout     Duration
 329         ReadTimeout        Duration
 330         RaceWindow         Duration
 331         UnsafeDelete       bool
 332         PrefixLength       int
 333 }
 334
 335 type AzureVolumeDriverParameters struct {
 336         StorageAccountName   string
 337         StorageAccountKey    string
 338         StorageBaseURL       string
 339         ContainerName        string
 340         RequestTimeout       Duration
 341         ListBlobsRetryDelay  Duration
 342         ListBlobsMaxAttempts int
 343 }
 344
 345 type DirectoryVolumeDriverParameters struct {
 346         Root      string
 347         Serialize bool
 348 }
 349
 350 type VolumeAccess struct {
 351         ReadOnly bool
 352 }
 353
 354 type Services struct {
 355         Composer       Service
 356         Controller     Service
 357         DispatchCloud  Service
 358         DispatchLSF    Service
 359         DispatchSLURM  Service
 360         GitHTTP        Service
 361         GitSSH         Service
 362         Health         Service
 363         Keepbalance    Service
 364         Keepproxy      Service
 365         Keepstore      Service
 366         RailsAPI       Service
 367         WebDAVDownload Service
 368         WebDAV         Service
 369         WebShell       Service
 370         Websocket      Service
 371         Workbench1     Service
 372         Workbench2     Service
 373 }
 374
 375 type Service struct {
 376         InternalURLs map[URL]ServiceInstance
 377         ExternalURL  URL
 378 }
 379
 380 type TestUser struct {
 381         Email    string
 382         Password string
 383 }
 384
 385 // URL is a url.URL that is also usable as a JSON key/value.
 386 type URL url.URL
 387
 388 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 389 // used as a JSON key/value.
 390 func (su *URL) UnmarshalText(text []byte) error {
 391         u, err := url.Parse(string(text))
 392         if err == nil {
 393                 *su = URL(*u)
 394                 if su.Path == "" && su.Host != "" {
 395                         // http://example really means http://example/
 396                         su.Path = "/"
 397                 }
 398         }
 399         return err
 400 }
 401
 402 func (su URL) MarshalText() ([]byte, error) {
 403         return []byte(su.String()), nil
 404 }
 405
 406 func (su URL) String() string {
 407         return (*url.URL)(&su).String()
 408 }
 409
 410 type TLSVersion uint16
 411
 412 func (v TLSVersion) MarshalText() ([]byte, error) {
 413         switch v {
 414         case 0:
 415                 return []byte{}, nil
 416         case tls.VersionTLS10:
 417                 return []byte("1.0"), nil
 418         case tls.VersionTLS11:
 419                 return []byte("1.1"), nil
 420         case tls.VersionTLS12:
 421                 return []byte("1.2"), nil
 422         case tls.VersionTLS13:
 423                 return []byte("1.3"), nil
 424         default:
 425                 return nil, fmt.Errorf("unsupported TLSVersion %x", v)
 426         }
 427 }
 428
 429 func (v *TLSVersion) UnmarshalJSON(text []byte) error {
 430         if len(text) > 0 && text[0] == '"' {
 431                 var s string
 432                 err := json.Unmarshal(text, &s)
 433                 if err != nil {
 434                         return err
 435                 }
 436                 text = []byte(s)
 437         }
 438         switch string(text) {
 439         case "":
 440                 *v = 0
 441         case "1.0":
 442                 *v = tls.VersionTLS10
 443         case "1.1":
 444                 *v = tls.VersionTLS11
 445         case "1.2":
 446                 *v = tls.VersionTLS12
 447         case "1.3":
 448                 *v = tls.VersionTLS13
 449         default:
 450                 return fmt.Errorf("unsupported TLSVersion %q", text)
 451         }
 452         return nil
 453 }
 454
 455 type ServiceInstance struct {
 456         ListenURL  URL
 457         Rendezvous string `json:",omitempty"`
 458 }
 459
 460 type PostgreSQL struct {
 461         Connection     PostgreSQLConnection
 462         ConnectionPool int
 463 }
 464
 465 type PostgreSQLConnection map[string]string
 466
 467 type RemoteCluster struct {
 468         Host          string
 469         Proxy         bool
 470         Scheme        string
 471         Insecure      bool
 472         ActivateUsers bool
 473 }
 474
 475 type CUDAFeatures struct {
 476         DriverVersion      string
 477         HardwareCapability string
 478         DeviceCount        int
 479 }
 480
 481 type InstanceType struct {
 482         Name            string `json:"-"`
 483         ProviderType    string
 484         VCPUs           int
 485         RAM             ByteSize
 486         Scratch         ByteSize `json:"-"`
 487         IncludedScratch ByteSize
 488         AddedScratch    ByteSize
 489         Price           float64
 490         Preemptible     bool
 491         CUDA            CUDAFeatures
 492 }
 493
 494 type ContainersConfig struct {
 495         CloudVMs                      CloudVMsConfig
 496         CrunchRunCommand              string
 497         CrunchRunArgumentsList        []string
 498         DefaultKeepCacheRAM           ByteSize
 499         DispatchPrivateKey            string
 500         LogReuseDecisions             bool
 501         MaxDispatchAttempts           int
 502         MaxRetryAttempts              int
 503         MinRetryPeriod                Duration
 504         ReserveExtraRAM               ByteSize
 505         StaleLockTimeout              Duration
 506         SupportedDockerImageFormats   StringSet
 507         AlwaysUsePreemptibleInstances bool
 508         PreemptiblePriceFactor        float64
 509         RuntimeEngine                 string
 510         LocalKeepBlobBuffersPerVCPU   int
 511         LocalKeepLogsToContainerLog   string
 512
 513         JobsAPI struct {
 514                 Enable         string
 515                 GitInternalDir string
 516         }
 517         Logging struct {
 518                 MaxAge                       Duration
 519                 SweepInterval                Duration
 520                 LogBytesPerEvent             int
 521                 LogSecondsBetweenEvents      Duration
 522                 LogThrottlePeriod            Duration
 523                 LogThrottleBytes             int
 524                 LogThrottleLines             int
 525                 LimitLogBytesPerJob          int
 526                 LogPartialLineThrottlePeriod Duration
 527                 LogUpdatePeriod              Duration
 528                 LogUpdateSize                ByteSize
 529         }
 530         ShellAccess struct {
 531                 Admin bool
 532                 User  bool
 533         }
 534         SLURM struct {
 535                 PrioritySpread             int64
 536                 SbatchArgumentsList        []string
 537                 SbatchEnvironmentVariables map[string]string
 538                 Managed                    struct {
 539                         DNSServerConfDir       string
 540                         DNSServerConfTemplate  string
 541                         DNSServerReloadCommand string
 542                         DNSServerUpdateCommand string
 543                         ComputeNodeDomain      string
 544                         ComputeNodeNameservers StringSet
 545                         AssignNodeHostname     string
 546                 }
 547         }
 548         LSF struct {
 549                 BsubSudoUser      string
 550                 BsubArgumentsList []string
 551                 BsubCUDAArguments []string
 552         }
 553 }
 554
 555 type CloudVMsConfig struct {
 556         Enable bool
 557
 558         BootProbeCommand               string
 559         DeployRunnerBinary             string
 560         ImageID                        string
 561         MaxCloudOpsPerSecond           int
 562         MaxProbesPerSecond             int
 563         MaxConcurrentInstanceCreateOps int
 564         MaxInstances                   int
 565         PollInterval                   Duration
 566         ProbeInterval                  Duration
 567         SSHPort                        string
 568         SyncInterval                   Duration
 569         TimeoutBooting                 Duration
 570         TimeoutIdle                    Duration
 571         TimeoutProbe                   Duration
 572         TimeoutShutdown                Duration
 573         TimeoutSignal                  Duration
 574         TimeoutStaleRunLock            Duration
 575         TimeoutTERM                    Duration
 576         ResourceTags                   map[string]string
 577         TagKeyPrefix                   string
 578
 579         Driver           string
 580         DriverParameters json.RawMessage
 581 }
 582
 583 type InstanceTypeMap map[string]InstanceType
 584
 585 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 586
 587 // UnmarshalJSON does special handling of InstanceTypes:
 588 //
 589 // - populate computed fields (Name and Scratch)
 590 //
 591 // - error out if InstancesTypes are populated as an array, which was
 592 // deprecated in Arvados 1.2.0
 593 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 594         fixup := func(t InstanceType) (InstanceType, error) {
 595                 if t.ProviderType == "" {
 596                         t.ProviderType = t.Name
 597                 }
 598                 // If t.Scratch is set in the configuration file, it will be ignored and overwritten.
 599                 // It will also generate a "deprecated or unknown config entry" warning.
 600                 t.Scratch = t.IncludedScratch + t.AddedScratch
 601                 return t, nil
 602         }
 603
 604         if len(data) > 0 && data[0] == '[' {
 605                 return fmt.Errorf("InstanceTypes must be specified as a map, not an array, see https://doc.arvados.org/admin/config.html")
 606         }
 607         var hash map[string]InstanceType
 608         err := json.Unmarshal(data, &hash)
 609         if err != nil {
 610                 return err
 611         }
 612         // Fill in Name field (and ProviderType field, if not
 613         // specified) using hash key.
 614         *it = InstanceTypeMap(hash)
 615         for name, t := range *it {
 616                 t.Name = name
 617                 t, err := fixup(t)
 618                 if err != nil {
 619                         return err
 620                 }
 621                 (*it)[name] = t
 622         }
 623         return nil
 624 }
 625
 626 type StringSet map[string]struct{}
 627
 628 // UnmarshalJSON handles old config files that provide an array of
 629 // instance types instead of a hash.
 630 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 631         if len(data) > 0 && data[0] == '[' {
 632                 var arr []string
 633                 err := json.Unmarshal(data, &arr)
 634                 if err != nil {
 635                         return err
 636                 }
 637                 if len(arr) == 0 {
 638                         *ss = nil
 639                         return nil
 640                 }
 641                 *ss = make(map[string]struct{}, len(arr))
 642                 for _, t := range arr {
 643                         (*ss)[t] = struct{}{}
 644                 }
 645                 return nil
 646         }
 647         var hash map[string]struct{}
 648         err := json.Unmarshal(data, &hash)
 649         if err != nil {
 650                 return err
 651         }
 652         *ss = make(map[string]struct{}, len(hash))
 653         for t := range hash {
 654                 (*ss)[t] = struct{}{}
 655         }
 656
 657         return nil
 658 }
 659
 660 type ServiceName string
 661
 662 const (
 663         ServiceNameController    ServiceName = "arvados-controller"
 664         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 665         ServiceNameDispatchLSF   ServiceName = "arvados-dispatch-lsf"
 666         ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm"
 667         ServiceNameGitHTTP       ServiceName = "arvados-git-httpd"
 668         ServiceNameHealth        ServiceName = "arvados-health"
 669         ServiceNameKeepbalance   ServiceName = "keep-balance"
 670         ServiceNameKeepproxy     ServiceName = "keepproxy"
 671         ServiceNameKeepstore     ServiceName = "keepstore"
 672         ServiceNameKeepweb       ServiceName = "keep-web"
 673         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 674         ServiceNameWebsocket     ServiceName = "arvados-ws"
 675         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 676         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 677 )
 678
 679 // Map returns all services as a map, suitable for iterating over all
 680 // services or looking up a service by name.
 681 func (svcs Services) Map() map[ServiceName]Service {
 682         return map[ServiceName]Service{
 683                 ServiceNameController:    svcs.Controller,
 684                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 685                 ServiceNameDispatchLSF:   svcs.DispatchLSF,
 686                 ServiceNameDispatchSLURM: svcs.DispatchSLURM,
 687                 ServiceNameGitHTTP:       svcs.GitHTTP,
 688                 ServiceNameHealth:        svcs.Health,
 689                 ServiceNameKeepbalance:   svcs.Keepbalance,
 690                 ServiceNameKeepproxy:     svcs.Keepproxy,
 691                 ServiceNameKeepstore:     svcs.Keepstore,
 692                 ServiceNameKeepweb:       svcs.WebDAV,
 693                 ServiceNameRailsAPI:      svcs.RailsAPI,
 694                 ServiceNameWebsocket:     svcs.Websocket,
 695                 ServiceNameWorkbench1:    svcs.Workbench1,
 696                 ServiceNameWorkbench2:    svcs.Workbench2,
 697         }
 698 }