sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.arvados.org/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         }
  21         return "/etc/arvados/config.yml"
  22 }()
  23
  24 type Config struct {
  25         Clusters         map[string]Cluster
  26         AutoReloadConfig bool
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         cc, ok := sc.Clusters[clusterID]
  53         if !ok {
  54                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  55         }
  56         cc.ClusterID = clusterID
  57         return &cc, nil
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxBlockEntries      int
  64         MaxCollectionEntries int
  65         MaxCollectionBytes   int64
  66         MaxPermissionEntries int
  67         MaxUUIDEntries       int
  68         MaxSessions          int
  69 }
  70
  71 type UploadDownloadPermission struct {
  72         Upload   bool
  73         Download bool
  74 }
  75
  76 type UploadDownloadRolePermissions struct {
  77         User  UploadDownloadPermission
  78         Admin UploadDownloadPermission
  79 }
  80
  81 type Cluster struct {
  82         ClusterID       string `json:"-"`
  83         ManagementToken string
  84         SystemRootToken string
  85         Services        Services
  86         InstanceTypes   InstanceTypeMap
  87         Containers      ContainersConfig
  88         RemoteClusters  map[string]RemoteCluster
  89         PostgreSQL      PostgreSQL
  90
  91         API struct {
  92                 AsyncPermissionsUpdateInterval Duration
  93                 DisabledAPIs                   StringSet
  94                 MaxIndexDatabaseRead           int
  95                 MaxItemsPerResponse            int
  96                 MaxConcurrentRequests          int
  97                 MaxKeepBlobBuffers             int
  98                 MaxRequestAmplification        int
  99                 MaxRequestSize                 int
 100                 MaxTokenLifetime               Duration
 101                 RequestTimeout                 Duration
 102                 SendTimeout                    Duration
 103                 WebsocketClientEventQueue      int
 104                 WebsocketServerEventQueue      int
 105                 KeepServiceRequestTimeout      Duration
 106         }
 107         AuditLogs struct {
 108                 MaxAge             Duration
 109                 MaxDeleteBatch     int
 110                 UnloggedAttributes StringSet
 111         }
 112         Collections struct {
 113                 BlobSigning              bool
 114                 BlobSigningKey           string
 115                 BlobSigningTTL           Duration
 116                 BlobTrash                bool
 117                 BlobTrashLifetime        Duration
 118                 BlobTrashCheckInterval   Duration
 119                 BlobTrashConcurrency     int
 120                 BlobDeleteConcurrency    int
 121                 BlobReplicateConcurrency int
 122                 CollectionVersioning     bool
 123                 DefaultTrashLifetime     Duration
 124                 DefaultReplication       int
 125                 ManagedProperties        map[string]struct {
 126                         Value     interface{}
 127                         Function  string
 128                         Protected bool
 129                 }
 130                 PreserveVersionIfIdle        Duration
 131                 TrashSweepInterval           Duration
 132                 TrustAllContent              bool
 133                 ForwardSlashNameSubstitution string
 134                 S3FolderObjects              bool
 135
 136                 BlobMissingReport        string
 137                 BalancePeriod            Duration
 138                 BalanceCollectionBatch   int
 139                 BalanceCollectionBuffers int
 140                 BalanceTimeout           Duration
 141                 BalanceUpdateLimit       int
 142
 143                 WebDAVCache WebDAVCacheConfig
 144
 145                 KeepproxyPermission UploadDownloadRolePermissions
 146                 WebDAVPermission    UploadDownloadRolePermissions
 147                 WebDAVLogEvents     bool
 148         }
 149         Git struct {
 150                 GitCommand   string
 151                 GitoliteHome string
 152                 Repositories string
 153         }
 154         Login struct {
 155                 LDAP struct {
 156                         Enable             bool
 157                         URL                URL
 158                         StartTLS           bool
 159                         InsecureTLS        bool
 160                         StripDomain        string
 161                         AppendDomain       string
 162                         SearchAttribute    string
 163                         SearchBindUser     string
 164                         SearchBindPassword string
 165                         SearchBase         string
 166                         SearchFilters      string
 167                         EmailAttribute     string
 168                         UsernameAttribute  string
 169                 }
 170                 Google struct {
 171                         Enable                          bool
 172                         ClientID                        string
 173                         ClientSecret                    string
 174                         AlternateEmailAddresses         bool
 175                         AuthenticationRequestParameters map[string]string
 176                 }
 177                 OpenIDConnect struct {
 178                         Enable                          bool
 179                         Issuer                          string
 180                         ClientID                        string
 181                         ClientSecret                    string
 182                         EmailClaim                      string
 183                         EmailVerifiedClaim              string
 184                         UsernameClaim                   string
 185                         AcceptAccessToken               bool
 186                         AcceptAccessTokenScope          string
 187                         AuthenticationRequestParameters map[string]string
 188                 }
 189                 PAM struct {
 190                         Enable             bool
 191                         Service            string
 192                         DefaultEmailDomain string
 193                 }
 194                 Test struct {
 195                         Enable bool
 196                         Users  map[string]TestUser
 197                 }
 198                 LoginCluster       string
 199                 RemoteTokenRefresh Duration
 200                 TokenLifetime      Duration
 201                 TrustedClients     map[string]struct{}
 202                 IssueTrustedTokens bool
 203         }
 204         Mail struct {
 205                 MailchimpAPIKey                string
 206                 MailchimpListID                string
 207                 SendUserSetupNotificationEmail bool
 208                 IssueReporterEmailFrom         string
 209                 IssueReporterEmailTo           string
 210                 SupportEmailAddress            string
 211                 EmailFrom                      string
 212         }
 213         SystemLogs struct {
 214                 LogLevel                string
 215                 Format                  string
 216                 MaxRequestLogParamsSize int
 217         }
 218         TLS struct {
 219                 Certificate string
 220                 Key         string
 221                 Insecure    bool
 222         }
 223         Users struct {
 224                 AnonymousUserToken                    string
 225                 AdminNotifierEmailFrom                string
 226                 AutoAdminFirstUser                    bool
 227                 AutoAdminUserWithEmail                string
 228                 AutoSetupNewUsers                     bool
 229                 AutoSetupNewUsersWithRepository       bool
 230                 AutoSetupNewUsersWithVmUUID           string
 231                 AutoSetupUsernameBlacklist            StringSet
 232                 EmailSubjectPrefix                    string
 233                 NewInactiveUserNotificationRecipients StringSet
 234                 NewUserNotificationRecipients         StringSet
 235                 NewUsersAreActive                     bool
 236                 UserNotifierEmailFrom                 string
 237                 UserNotifierEmailBcc                  StringSet
 238                 UserProfileNotificationAddress        string
 239                 PreferDomainForUsername               string
 240                 UserSetupMailText                     string
 241         }
 242         StorageClasses map[string]StorageClassConfig
 243         Volumes        map[string]Volume
 244         Workbench      struct {
 245                 ActivationContactLink            string
 246                 APIClientConnectTimeout          Duration
 247                 APIClientReceiveTimeout          Duration
 248                 APIResponseCompression           bool
 249                 ApplicationMimetypesWithViewIcon StringSet
 250                 ArvadosDocsite                   string
 251                 ArvadosPublicDataDocURL          string
 252                 DefaultOpenIdPrefix              string
 253                 EnableGettingStartedPopup        bool
 254                 EnablePublicProjectsPage         bool
 255                 FileViewersConfigURL             string
 256                 LogViewerMaxBytes                ByteSize
 257                 MultiSiteSearch                  string
 258                 ProfilingEnabled                 bool
 259                 Repositories                     bool
 260                 RepositoryCache                  string
 261                 RunningJobLogRecordsToFetch      int
 262                 SecretKeyBase                    string
 263                 ShowRecentCollectionsOnDashboard bool
 264                 ShowUserAgreementInline          bool
 265                 ShowUserNotifications            bool
 266                 SiteName                         string
 267                 Theme                            string
 268                 UserProfileFormFields            map[string]struct {
 269                         Type                 string
 270                         FormFieldTitle       string
 271                         FormFieldDescription string
 272                         Required             bool
 273                         Position             int
 274                         Options              map[string]struct{}
 275                 }
 276                 UserProfileFormMessage string
 277                 VocabularyURL          string
 278                 WelcomePageHTML        string
 279                 InactivePageHTML       string
 280                 SSHHelpPageHTML        string
 281                 SSHHelpHostSuffix      string
 282                 IdleTimeout            Duration
 283         }
 284 }
 285
 286 type StorageClassConfig struct {
 287         Default  bool
 288         Priority int
 289 }
 290
 291 type Volume struct {
 292         AccessViaHosts   map[URL]VolumeAccess
 293         ReadOnly         bool
 294         Replication      int
 295         StorageClasses   map[string]bool
 296         Driver           string
 297         DriverParameters json.RawMessage
 298 }
 299
 300 type S3VolumeDriverParameters struct {
 301         IAMRole            string
 302         AccessKeyID        string
 303         SecretAccessKey    string
 304         Endpoint           string
 305         Region             string
 306         Bucket             string
 307         LocationConstraint bool
 308         V2Signature        bool
 309         UseAWSS3v2Driver   bool
 310         IndexPageSize      int
 311         ConnectTimeout     Duration
 312         ReadTimeout        Duration
 313         RaceWindow         Duration
 314         UnsafeDelete       bool
 315 }
 316
 317 type AzureVolumeDriverParameters struct {
 318         StorageAccountName   string
 319         StorageAccountKey    string
 320         StorageBaseURL       string
 321         ContainerName        string
 322         RequestTimeout       Duration
 323         ListBlobsRetryDelay  Duration
 324         ListBlobsMaxAttempts int
 325 }
 326
 327 type DirectoryVolumeDriverParameters struct {
 328         Root      string
 329         Serialize bool
 330 }
 331
 332 type VolumeAccess struct {
 333         ReadOnly bool
 334 }
 335
 336 type Services struct {
 337         Composer       Service
 338         Controller     Service
 339         DispatchCloud  Service
 340         DispatchLSF    Service
 341         GitHTTP        Service
 342         GitSSH         Service
 343         Health         Service
 344         Keepbalance    Service
 345         Keepproxy      Service
 346         Keepstore      Service
 347         RailsAPI       Service
 348         WebDAVDownload Service
 349         WebDAV         Service
 350         WebShell       Service
 351         Websocket      Service
 352         Workbench1     Service
 353         Workbench2     Service
 354 }
 355
 356 type Service struct {
 357         InternalURLs map[URL]ServiceInstance
 358         ExternalURL  URL
 359 }
 360
 361 type TestUser struct {
 362         Email    string
 363         Password string
 364 }
 365
 366 // URL is a url.URL that is also usable as a JSON key/value.
 367 type URL url.URL
 368
 369 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 370 // used as a JSON key/value.
 371 func (su *URL) UnmarshalText(text []byte) error {
 372         u, err := url.Parse(string(text))
 373         if err == nil {
 374                 *su = URL(*u)
 375                 if su.Path == "" && su.Host != "" {
 376                         // http://example really means http://example/
 377                         su.Path = "/"
 378                 }
 379         }
 380         return err
 381 }
 382
 383 func (su URL) MarshalText() ([]byte, error) {
 384         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 385 }
 386
 387 func (su URL) String() string {
 388         return (*url.URL)(&su).String()
 389 }
 390
 391 type ServiceInstance struct {
 392         Rendezvous string `json:",omitempty"`
 393 }
 394
 395 type PostgreSQL struct {
 396         Connection     PostgreSQLConnection
 397         ConnectionPool int
 398 }
 399
 400 type PostgreSQLConnection map[string]string
 401
 402 type RemoteCluster struct {
 403         Host          string
 404         Proxy         bool
 405         Scheme        string
 406         Insecure      bool
 407         ActivateUsers bool
 408 }
 409
 410 type InstanceType struct {
 411         Name            string
 412         ProviderType    string
 413         VCPUs           int
 414         RAM             ByteSize
 415         Scratch         ByteSize
 416         IncludedScratch ByteSize
 417         AddedScratch    ByteSize
 418         Price           float64
 419         Preemptible     bool
 420 }
 421
 422 type ContainersConfig struct {
 423         CloudVMs                    CloudVMsConfig
 424         CrunchRunCommand            string
 425         CrunchRunArgumentsList      []string
 426         DefaultKeepCacheRAM         ByteSize
 427         DispatchPrivateKey          string
 428         LogReuseDecisions           bool
 429         MaxComputeVMs               int
 430         MaxDispatchAttempts         int
 431         MaxRetryAttempts            int
 432         MinRetryPeriod              Duration
 433         ReserveExtraRAM             ByteSize
 434         StaleLockTimeout            Duration
 435         SupportedDockerImageFormats StringSet
 436         UsePreemptibleInstances     bool
 437         RuntimeEngine               string
 438
 439         JobsAPI struct {
 440                 Enable         string
 441                 GitInternalDir string
 442         }
 443         Logging struct {
 444                 MaxAge                       Duration
 445                 LogBytesPerEvent             int
 446                 LogSecondsBetweenEvents      Duration
 447                 LogThrottlePeriod            Duration
 448                 LogThrottleBytes             int
 449                 LogThrottleLines             int
 450                 LimitLogBytesPerJob          int
 451                 LogPartialLineThrottlePeriod Duration
 452                 LogUpdatePeriod              Duration
 453                 LogUpdateSize                ByteSize
 454         }
 455         ShellAccess struct {
 456                 Admin bool
 457                 User  bool
 458         }
 459         SLURM struct {
 460                 PrioritySpread             int64
 461                 SbatchArgumentsList        []string
 462                 SbatchEnvironmentVariables map[string]string
 463                 Managed                    struct {
 464                         DNSServerConfDir       string
 465                         DNSServerConfTemplate  string
 466                         DNSServerReloadCommand string
 467                         DNSServerUpdateCommand string
 468                         ComputeNodeDomain      string
 469                         ComputeNodeNameservers StringSet
 470                         AssignNodeHostname     string
 471                 }
 472         }
 473         LSF struct {
 474                 BsubSudoUser      string
 475                 BsubArgumentsList []string
 476         }
 477 }
 478
 479 type CloudVMsConfig struct {
 480         Enable bool
 481
 482         BootProbeCommand               string
 483         DeployRunnerBinary             string
 484         ImageID                        string
 485         MaxCloudOpsPerSecond           int
 486         MaxProbesPerSecond             int
 487         MaxConcurrentInstanceCreateOps int
 488         PollInterval                   Duration
 489         ProbeInterval                  Duration
 490         SSHPort                        string
 491         SyncInterval                   Duration
 492         TimeoutBooting                 Duration
 493         TimeoutIdle                    Duration
 494         TimeoutProbe                   Duration
 495         TimeoutShutdown                Duration
 496         TimeoutSignal                  Duration
 497         TimeoutStaleRunLock            Duration
 498         TimeoutTERM                    Duration
 499         ResourceTags                   map[string]string
 500         TagKeyPrefix                   string
 501
 502         Driver           string
 503         DriverParameters json.RawMessage
 504 }
 505
 506 type InstanceTypeMap map[string]InstanceType
 507
 508 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 509
 510 // UnmarshalJSON handles old config files that provide an array of
 511 // instance types instead of a hash.
 512 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 513         fixup := func(t InstanceType) (InstanceType, error) {
 514                 if t.ProviderType == "" {
 515                         t.ProviderType = t.Name
 516                 }
 517                 if t.Scratch == 0 {
 518                         t.Scratch = t.IncludedScratch + t.AddedScratch
 519                 } else if t.AddedScratch == 0 {
 520                         t.AddedScratch = t.Scratch - t.IncludedScratch
 521                 } else if t.IncludedScratch == 0 {
 522                         t.IncludedScratch = t.Scratch - t.AddedScratch
 523                 }
 524
 525                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 526                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 527                 }
 528                 return t, nil
 529         }
 530
 531         if len(data) > 0 && data[0] == '[' {
 532                 var arr []InstanceType
 533                 err := json.Unmarshal(data, &arr)
 534                 if err != nil {
 535                         return err
 536                 }
 537                 if len(arr) == 0 {
 538                         *it = nil
 539                         return nil
 540                 }
 541                 *it = make(map[string]InstanceType, len(arr))
 542                 for _, t := range arr {
 543                         if _, ok := (*it)[t.Name]; ok {
 544                                 return errDuplicateInstanceTypeName
 545                         }
 546                         t, err := fixup(t)
 547                         if err != nil {
 548                                 return err
 549                         }
 550                         (*it)[t.Name] = t
 551                 }
 552                 return nil
 553         }
 554         var hash map[string]InstanceType
 555         err := json.Unmarshal(data, &hash)
 556         if err != nil {
 557                 return err
 558         }
 559         // Fill in Name field (and ProviderType field, if not
 560         // specified) using hash key.
 561         *it = InstanceTypeMap(hash)
 562         for name, t := range *it {
 563                 t.Name = name
 564                 t, err := fixup(t)
 565                 if err != nil {
 566                         return err
 567                 }
 568                 (*it)[name] = t
 569         }
 570         return nil
 571 }
 572
 573 type StringSet map[string]struct{}
 574
 575 // UnmarshalJSON handles old config files that provide an array of
 576 // instance types instead of a hash.
 577 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 578         if len(data) > 0 && data[0] == '[' {
 579                 var arr []string
 580                 err := json.Unmarshal(data, &arr)
 581                 if err != nil {
 582                         return err
 583                 }
 584                 if len(arr) == 0 {
 585                         *ss = nil
 586                         return nil
 587                 }
 588                 *ss = make(map[string]struct{}, len(arr))
 589                 for _, t := range arr {
 590                         (*ss)[t] = struct{}{}
 591                 }
 592                 return nil
 593         }
 594         var hash map[string]struct{}
 595         err := json.Unmarshal(data, &hash)
 596         if err != nil {
 597                 return err
 598         }
 599         *ss = make(map[string]struct{}, len(hash))
 600         for t := range hash {
 601                 (*ss)[t] = struct{}{}
 602         }
 603
 604         return nil
 605 }
 606
 607 type ServiceName string
 608
 609 const (
 610         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 611         ServiceNameController    ServiceName = "arvados-controller"
 612         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 613         ServiceNameDispatchLSF   ServiceName = "arvados-dispatch-lsf"
 614         ServiceNameHealth        ServiceName = "arvados-health"
 615         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 616         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 617         ServiceNameWebsocket     ServiceName = "arvados-ws"
 618         ServiceNameKeepbalance   ServiceName = "keep-balance"
 619         ServiceNameKeepweb       ServiceName = "keep-web"
 620         ServiceNameKeepproxy     ServiceName = "keepproxy"
 621         ServiceNameKeepstore     ServiceName = "keepstore"
 622 )
 623
 624 // Map returns all services as a map, suitable for iterating over all
 625 // services or looking up a service by name.
 626 func (svcs Services) Map() map[ServiceName]Service {
 627         return map[ServiceName]Service{
 628                 ServiceNameRailsAPI:      svcs.RailsAPI,
 629                 ServiceNameController:    svcs.Controller,
 630                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 631                 ServiceNameDispatchLSF:   svcs.DispatchLSF,
 632                 ServiceNameHealth:        svcs.Health,
 633                 ServiceNameWorkbench1:    svcs.Workbench1,
 634                 ServiceNameWorkbench2:    svcs.Workbench2,
 635                 ServiceNameWebsocket:     svcs.Websocket,
 636                 ServiceNameKeepbalance:   svcs.Keepbalance,
 637                 ServiceNameKeepweb:       svcs.WebDAV,
 638                 ServiceNameKeepproxy:     svcs.Keepproxy,
 639                 ServiceNameKeepstore:     svcs.Keepstore,
 640         }
 641 }