sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.arvados.org/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         }
  21         return "/etc/arvados/config.yml"
  22 }()
  23
  24 type Config struct {
  25         Clusters         map[string]Cluster
  26         AutoReloadConfig bool
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         cc, ok := sc.Clusters[clusterID]
  53         if !ok {
  54                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  55         }
  56         cc.ClusterID = clusterID
  57         return &cc, nil
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxBlockEntries      int
  64         MaxCollectionEntries int
  65         MaxCollectionBytes   int64
  66         MaxPermissionEntries int
  67         MaxUUIDEntries       int
  68         MaxSessions          int
  69 }
  70
  71 type UploadDownloadPermission struct {
  72         Upload   bool
  73         Download bool
  74 }
  75
  76 type UploadDownloadRolePermissions struct {
  77         User  UploadDownloadPermission
  78         Admin UploadDownloadPermission
  79 }
  80
  81 type Cluster struct {
  82         ClusterID       string `json:"-"`
  83         ManagementToken string
  84         SystemRootToken string
  85         Services        Services
  86         InstanceTypes   InstanceTypeMap
  87         Containers      ContainersConfig
  88         RemoteClusters  map[string]RemoteCluster
  89         PostgreSQL      PostgreSQL
  90
  91         API struct {
  92                 AsyncPermissionsUpdateInterval Duration
  93                 DisabledAPIs                   StringSet
  94                 MaxIndexDatabaseRead           int
  95                 MaxItemsPerResponse            int
  96                 MaxConcurrentRequests          int
  97                 MaxKeepBlobBuffers             int
  98                 MaxRequestAmplification        int
  99                 MaxRequestSize                 int
 100                 MaxTokenLifetime               Duration
 101                 RequestTimeout                 Duration
 102                 SendTimeout                    Duration
 103                 WebsocketClientEventQueue      int
 104                 WebsocketServerEventQueue      int
 105                 KeepServiceRequestTimeout      Duration
 106         }
 107         AuditLogs struct {
 108                 MaxAge             Duration
 109                 MaxDeleteBatch     int
 110                 UnloggedAttributes StringSet
 111         }
 112         Collections struct {
 113                 BlobSigning              bool
 114                 BlobSigningKey           string
 115                 BlobSigningTTL           Duration
 116                 BlobTrash                bool
 117                 BlobTrashLifetime        Duration
 118                 BlobTrashCheckInterval   Duration
 119                 BlobTrashConcurrency     int
 120                 BlobDeleteConcurrency    int
 121                 BlobReplicateConcurrency int
 122                 CollectionVersioning     bool
 123                 DefaultTrashLifetime     Duration
 124                 DefaultReplication       int
 125                 ManagedProperties        map[string]struct {
 126                         Value     interface{}
 127                         Function  string
 128                         Protected bool
 129                 }
 130                 PreserveVersionIfIdle        Duration
 131                 TrashSweepInterval           Duration
 132                 TrustAllContent              bool
 133                 ForwardSlashNameSubstitution string
 134                 S3FolderObjects              bool
 135
 136                 BlobMissingReport        string
 137                 BalancePeriod            Duration
 138                 BalanceCollectionBatch   int
 139                 BalanceCollectionBuffers int
 140                 BalanceTimeout           Duration
 141                 BalanceUpdateLimit       int
 142
 143                 WebDAVCache WebDAVCacheConfig
 144
 145                 KeepproxyPermission UploadDownloadRolePermissions
 146                 WebDAVPermission    UploadDownloadRolePermissions
 147                 WebDAVLogEvents     bool
 148         }
 149         Git struct {
 150                 GitCommand   string
 151                 GitoliteHome string
 152                 Repositories string
 153         }
 154         Login struct {
 155                 LDAP struct {
 156                         Enable             bool
 157                         URL                URL
 158                         StartTLS           bool
 159                         InsecureTLS        bool
 160                         StripDomain        string
 161                         AppendDomain       string
 162                         SearchAttribute    string
 163                         SearchBindUser     string
 164                         SearchBindPassword string
 165                         SearchBase         string
 166                         SearchFilters      string
 167                         EmailAttribute     string
 168                         UsernameAttribute  string
 169                 }
 170                 Google struct {
 171                         Enable                          bool
 172                         ClientID                        string
 173                         ClientSecret                    string
 174                         AlternateEmailAddresses         bool
 175                         AuthenticationRequestParameters map[string]string
 176                 }
 177                 OpenIDConnect struct {
 178                         Enable                          bool
 179                         Issuer                          string
 180                         ClientID                        string
 181                         ClientSecret                    string
 182                         EmailClaim                      string
 183                         EmailVerifiedClaim              string
 184                         UsernameClaim                   string
 185                         AcceptAccessToken               bool
 186                         AcceptAccessTokenScope          string
 187                         AuthenticationRequestParameters map[string]string
 188                 }
 189                 PAM struct {
 190                         Enable             bool
 191                         Service            string
 192                         DefaultEmailDomain string
 193                 }
 194                 Test struct {
 195                         Enable bool
 196                         Users  map[string]TestUser
 197                 }
 198                 LoginCluster       string
 199                 RemoteTokenRefresh Duration
 200                 TokenLifetime      Duration
 201                 TrustedClients     map[string]struct{}
 202                 IssueTrustedTokens bool
 203         }
 204         Mail struct {
 205                 MailchimpAPIKey                string
 206                 MailchimpListID                string
 207                 SendUserSetupNotificationEmail bool
 208                 IssueReporterEmailFrom         string
 209                 IssueReporterEmailTo           string
 210                 SupportEmailAddress            string
 211                 EmailFrom                      string
 212         }
 213         SystemLogs struct {
 214                 LogLevel                string
 215                 Format                  string
 216                 MaxRequestLogParamsSize int
 217         }
 218         TLS struct {
 219                 Certificate string
 220                 Key         string
 221                 Insecure    bool
 222         }
 223         Users struct {
 224                 AnonymousUserToken                    string
 225                 AdminNotifierEmailFrom                string
 226                 AutoAdminFirstUser                    bool
 227                 AutoAdminUserWithEmail                string
 228                 AutoSetupNewUsers                     bool
 229                 AutoSetupNewUsersWithRepository       bool
 230                 AutoSetupNewUsersWithVmUUID           string
 231                 AutoSetupUsernameBlacklist            StringSet
 232                 EmailSubjectPrefix                    string
 233                 NewInactiveUserNotificationRecipients StringSet
 234                 NewUserNotificationRecipients         StringSet
 235                 NewUsersAreActive                     bool
 236                 UserNotifierEmailFrom                 string
 237                 UserProfileNotificationAddress        string
 238                 PreferDomainForUsername               string
 239                 UserSetupMailText                     string
 240         }
 241         Volumes   map[string]Volume
 242         Workbench struct {
 243                 ActivationContactLink            string
 244                 APIClientConnectTimeout          Duration
 245                 APIClientReceiveTimeout          Duration
 246                 APIResponseCompression           bool
 247                 ApplicationMimetypesWithViewIcon StringSet
 248                 ArvadosDocsite                   string
 249                 ArvadosPublicDataDocURL          string
 250                 DefaultOpenIdPrefix              string
 251                 EnableGettingStartedPopup        bool
 252                 EnablePublicProjectsPage         bool
 253                 FileViewersConfigURL             string
 254                 LogViewerMaxBytes                ByteSize
 255                 MultiSiteSearch                  string
 256                 ProfilingEnabled                 bool
 257                 Repositories                     bool
 258                 RepositoryCache                  string
 259                 RunningJobLogRecordsToFetch      int
 260                 SecretKeyBase                    string
 261                 ShowRecentCollectionsOnDashboard bool
 262                 ShowUserAgreementInline          bool
 263                 ShowUserNotifications            bool
 264                 SiteName                         string
 265                 Theme                            string
 266                 UserProfileFormFields            map[string]struct {
 267                         Type                 string
 268                         FormFieldTitle       string
 269                         FormFieldDescription string
 270                         Required             bool
 271                         Position             int
 272                         Options              map[string]struct{}
 273                 }
 274                 UserProfileFormMessage string
 275                 VocabularyURL          string
 276                 WelcomePageHTML        string
 277                 InactivePageHTML       string
 278                 SSHHelpPageHTML        string
 279                 SSHHelpHostSuffix      string
 280                 IdleTimeout            Duration
 281         }
 282 }
 283
 284 type Volume struct {
 285         AccessViaHosts   map[URL]VolumeAccess
 286         ReadOnly         bool
 287         Replication      int
 288         StorageClasses   map[string]bool
 289         Driver           string
 290         DriverParameters json.RawMessage
 291 }
 292
 293 type S3VolumeDriverParameters struct {
 294         IAMRole            string
 295         AccessKeyID        string
 296         SecretAccessKey    string
 297         Endpoint           string
 298         Region             string
 299         Bucket             string
 300         LocationConstraint bool
 301         V2Signature        bool
 302         UseAWSS3v2Driver   bool
 303         IndexPageSize      int
 304         ConnectTimeout     Duration
 305         ReadTimeout        Duration
 306         RaceWindow         Duration
 307         UnsafeDelete       bool
 308 }
 309
 310 type AzureVolumeDriverParameters struct {
 311         StorageAccountName   string
 312         StorageAccountKey    string
 313         StorageBaseURL       string
 314         ContainerName        string
 315         RequestTimeout       Duration
 316         ListBlobsRetryDelay  Duration
 317         ListBlobsMaxAttempts int
 318 }
 319
 320 type DirectoryVolumeDriverParameters struct {
 321         Root      string
 322         Serialize bool
 323 }
 324
 325 type VolumeAccess struct {
 326         ReadOnly bool
 327 }
 328
 329 type Services struct {
 330         Composer       Service
 331         Controller     Service
 332         DispatchCloud  Service
 333         DispatchLSF    Service
 334         GitHTTP        Service
 335         GitSSH         Service
 336         Health         Service
 337         Keepbalance    Service
 338         Keepproxy      Service
 339         Keepstore      Service
 340         RailsAPI       Service
 341         WebDAVDownload Service
 342         WebDAV         Service
 343         WebShell       Service
 344         Websocket      Service
 345         Workbench1     Service
 346         Workbench2     Service
 347 }
 348
 349 type Service struct {
 350         InternalURLs map[URL]ServiceInstance
 351         ExternalURL  URL
 352 }
 353
 354 type TestUser struct {
 355         Email    string
 356         Password string
 357 }
 358
 359 // URL is a url.URL that is also usable as a JSON key/value.
 360 type URL url.URL
 361
 362 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 363 // used as a JSON key/value.
 364 func (su *URL) UnmarshalText(text []byte) error {
 365         u, err := url.Parse(string(text))
 366         if err == nil {
 367                 *su = URL(*u)
 368                 if su.Path == "" && su.Host != "" {
 369                         // http://example really means http://example/
 370                         su.Path = "/"
 371                 }
 372         }
 373         return err
 374 }
 375
 376 func (su URL) MarshalText() ([]byte, error) {
 377         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 378 }
 379
 380 func (su URL) String() string {
 381         return (*url.URL)(&su).String()
 382 }
 383
 384 type ServiceInstance struct {
 385         Rendezvous string `json:",omitempty"`
 386 }
 387
 388 type PostgreSQL struct {
 389         Connection     PostgreSQLConnection
 390         ConnectionPool int
 391 }
 392
 393 type PostgreSQLConnection map[string]string
 394
 395 type RemoteCluster struct {
 396         Host          string
 397         Proxy         bool
 398         Scheme        string
 399         Insecure      bool
 400         ActivateUsers bool
 401 }
 402
 403 type InstanceType struct {
 404         Name            string
 405         ProviderType    string
 406         VCPUs           int
 407         RAM             ByteSize
 408         Scratch         ByteSize
 409         IncludedScratch ByteSize
 410         AddedScratch    ByteSize
 411         Price           float64
 412         Preemptible     bool
 413 }
 414
 415 type ContainersConfig struct {
 416         CloudVMs                    CloudVMsConfig
 417         CrunchRunCommand            string
 418         CrunchRunArgumentsList      []string
 419         DefaultKeepCacheRAM         ByteSize
 420         DispatchPrivateKey          string
 421         LogReuseDecisions           bool
 422         MaxComputeVMs               int
 423         MaxDispatchAttempts         int
 424         MaxRetryAttempts            int
 425         MinRetryPeriod              Duration
 426         ReserveExtraRAM             ByteSize
 427         StaleLockTimeout            Duration
 428         SupportedDockerImageFormats StringSet
 429         UsePreemptibleInstances     bool
 430         RuntimeEngine               string
 431
 432         JobsAPI struct {
 433                 Enable         string
 434                 GitInternalDir string
 435         }
 436         Logging struct {
 437                 MaxAge                       Duration
 438                 LogBytesPerEvent             int
 439                 LogSecondsBetweenEvents      Duration
 440                 LogThrottlePeriod            Duration
 441                 LogThrottleBytes             int
 442                 LogThrottleLines             int
 443                 LimitLogBytesPerJob          int
 444                 LogPartialLineThrottlePeriod Duration
 445                 LogUpdatePeriod              Duration
 446                 LogUpdateSize                ByteSize
 447         }
 448         ShellAccess struct {
 449                 Admin bool
 450                 User  bool
 451         }
 452         SLURM struct {
 453                 PrioritySpread             int64
 454                 SbatchArgumentsList        []string
 455                 SbatchEnvironmentVariables map[string]string
 456                 Managed                    struct {
 457                         DNSServerConfDir       string
 458                         DNSServerConfTemplate  string
 459                         DNSServerReloadCommand string
 460                         DNSServerUpdateCommand string
 461                         ComputeNodeDomain      string
 462                         ComputeNodeNameservers StringSet
 463                         AssignNodeHostname     string
 464                 }
 465         }
 466         LSF struct {
 467                 BsubSudoUser      string
 468                 BsubArgumentsList []string
 469         }
 470 }
 471
 472 type CloudVMsConfig struct {
 473         Enable bool
 474
 475         BootProbeCommand               string
 476         DeployRunnerBinary             string
 477         ImageID                        string
 478         MaxCloudOpsPerSecond           int
 479         MaxProbesPerSecond             int
 480         MaxConcurrentInstanceCreateOps int
 481         PollInterval                   Duration
 482         ProbeInterval                  Duration
 483         SSHPort                        string
 484         SyncInterval                   Duration
 485         TimeoutBooting                 Duration
 486         TimeoutIdle                    Duration
 487         TimeoutProbe                   Duration
 488         TimeoutShutdown                Duration
 489         TimeoutSignal                  Duration
 490         TimeoutStaleRunLock            Duration
 491         TimeoutTERM                    Duration
 492         ResourceTags                   map[string]string
 493         TagKeyPrefix                   string
 494
 495         Driver           string
 496         DriverParameters json.RawMessage
 497 }
 498
 499 type InstanceTypeMap map[string]InstanceType
 500
 501 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 502
 503 // UnmarshalJSON handles old config files that provide an array of
 504 // instance types instead of a hash.
 505 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 506         fixup := func(t InstanceType) (InstanceType, error) {
 507                 if t.ProviderType == "" {
 508                         t.ProviderType = t.Name
 509                 }
 510                 if t.Scratch == 0 {
 511                         t.Scratch = t.IncludedScratch + t.AddedScratch
 512                 } else if t.AddedScratch == 0 {
 513                         t.AddedScratch = t.Scratch - t.IncludedScratch
 514                 } else if t.IncludedScratch == 0 {
 515                         t.IncludedScratch = t.Scratch - t.AddedScratch
 516                 }
 517
 518                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 519                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 520                 }
 521                 return t, nil
 522         }
 523
 524         if len(data) > 0 && data[0] == '[' {
 525                 var arr []InstanceType
 526                 err := json.Unmarshal(data, &arr)
 527                 if err != nil {
 528                         return err
 529                 }
 530                 if len(arr) == 0 {
 531                         *it = nil
 532                         return nil
 533                 }
 534                 *it = make(map[string]InstanceType, len(arr))
 535                 for _, t := range arr {
 536                         if _, ok := (*it)[t.Name]; ok {
 537                                 return errDuplicateInstanceTypeName
 538                         }
 539                         t, err := fixup(t)
 540                         if err != nil {
 541                                 return err
 542                         }
 543                         (*it)[t.Name] = t
 544                 }
 545                 return nil
 546         }
 547         var hash map[string]InstanceType
 548         err := json.Unmarshal(data, &hash)
 549         if err != nil {
 550                 return err
 551         }
 552         // Fill in Name field (and ProviderType field, if not
 553         // specified) using hash key.
 554         *it = InstanceTypeMap(hash)
 555         for name, t := range *it {
 556                 t.Name = name
 557                 t, err := fixup(t)
 558                 if err != nil {
 559                         return err
 560                 }
 561                 (*it)[name] = t
 562         }
 563         return nil
 564 }
 565
 566 type StringSet map[string]struct{}
 567
 568 // UnmarshalJSON handles old config files that provide an array of
 569 // instance types instead of a hash.
 570 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 571         if len(data) > 0 && data[0] == '[' {
 572                 var arr []string
 573                 err := json.Unmarshal(data, &arr)
 574                 if err != nil {
 575                         return err
 576                 }
 577                 if len(arr) == 0 {
 578                         *ss = nil
 579                         return nil
 580                 }
 581                 *ss = make(map[string]struct{}, len(arr))
 582                 for _, t := range arr {
 583                         (*ss)[t] = struct{}{}
 584                 }
 585                 return nil
 586         }
 587         var hash map[string]struct{}
 588         err := json.Unmarshal(data, &hash)
 589         if err != nil {
 590                 return err
 591         }
 592         *ss = make(map[string]struct{}, len(hash))
 593         for t := range hash {
 594                 (*ss)[t] = struct{}{}
 595         }
 596
 597         return nil
 598 }
 599
 600 type ServiceName string
 601
 602 const (
 603         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 604         ServiceNameController    ServiceName = "arvados-controller"
 605         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 606         ServiceNameDispatchLSF   ServiceName = "arvados-dispatch-lsf"
 607         ServiceNameHealth        ServiceName = "arvados-health"
 608         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 609         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 610         ServiceNameWebsocket     ServiceName = "arvados-ws"
 611         ServiceNameKeepbalance   ServiceName = "keep-balance"
 612         ServiceNameKeepweb       ServiceName = "keep-web"
 613         ServiceNameKeepproxy     ServiceName = "keepproxy"
 614         ServiceNameKeepstore     ServiceName = "keepstore"
 615 )
 616
 617 // Map returns all services as a map, suitable for iterating over all
 618 // services or looking up a service by name.
 619 func (svcs Services) Map() map[ServiceName]Service {
 620         return map[ServiceName]Service{
 621                 ServiceNameRailsAPI:      svcs.RailsAPI,
 622                 ServiceNameController:    svcs.Controller,
 623                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 624                 ServiceNameDispatchLSF:   svcs.DispatchLSF,
 625                 ServiceNameHealth:        svcs.Health,
 626                 ServiceNameWorkbench1:    svcs.Workbench1,
 627                 ServiceNameWorkbench2:    svcs.Workbench2,
 628                 ServiceNameWebsocket:     svcs.Websocket,
 629                 ServiceNameKeepbalance:   svcs.Keepbalance,
 630                 ServiceNameKeepweb:       svcs.WebDAV,
 631                 ServiceNameKeepproxy:     svcs.Keepproxy,
 632                 ServiceNameKeepstore:     svcs.Keepstore,
 633         }
 634 }