sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.arvados.org/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         }
  21         return "/etc/arvados/config.yml"
  22 }()
  23
  24 type Config struct {
  25         Clusters         map[string]Cluster
  26         AutoReloadConfig bool
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         cc, ok := sc.Clusters[clusterID]
  53         if !ok {
  54                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  55         }
  56         cc.ClusterID = clusterID
  57         return &cc, nil
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxBlockEntries      int
  64         MaxCollectionEntries int
  65         MaxCollectionBytes   int64
  66         MaxUUIDEntries       int
  67         MaxSessions          int
  68 }
  69
  70 type UploadDownloadPermission struct {
  71         Upload   bool
  72         Download bool
  73 }
  74
  75 type UploadDownloadRolePermissions struct {
  76         User  UploadDownloadPermission
  77         Admin UploadDownloadPermission
  78 }
  79
  80 type Cluster struct {
  81         ClusterID       string `json:"-"`
  82         ManagementToken string
  83         SystemRootToken string
  84         Services        Services
  85         InstanceTypes   InstanceTypeMap
  86         Containers      ContainersConfig
  87         RemoteClusters  map[string]RemoteCluster
  88         PostgreSQL      PostgreSQL
  89
  90         API struct {
  91                 AsyncPermissionsUpdateInterval Duration
  92                 DisabledAPIs                   StringSet
  93                 MaxIndexDatabaseRead           int
  94                 MaxItemsPerResponse            int
  95                 MaxConcurrentRequests          int
  96                 MaxKeepBlobBuffers             int
  97                 MaxRequestAmplification        int
  98                 MaxRequestSize                 int
  99                 MaxTokenLifetime               Duration
 100                 RequestTimeout                 Duration
 101                 SendTimeout                    Duration
 102                 WebsocketClientEventQueue      int
 103                 WebsocketServerEventQueue      int
 104                 KeepServiceRequestTimeout      Duration
 105         }
 106         AuditLogs struct {
 107                 MaxAge             Duration
 108                 MaxDeleteBatch     int
 109                 UnloggedAttributes StringSet
 110         }
 111         Collections struct {
 112                 BlobSigning              bool
 113                 BlobSigningKey           string
 114                 BlobSigningTTL           Duration
 115                 BlobTrash                bool
 116                 BlobTrashLifetime        Duration
 117                 BlobTrashCheckInterval   Duration
 118                 BlobTrashConcurrency     int
 119                 BlobDeleteConcurrency    int
 120                 BlobReplicateConcurrency int
 121                 CollectionVersioning     bool
 122                 DefaultTrashLifetime     Duration
 123                 DefaultReplication       int
 124                 ManagedProperties        map[string]struct {
 125                         Value     interface{}
 126                         Function  string
 127                         Protected bool
 128                 }
 129                 PreserveVersionIfIdle        Duration
 130                 TrashSweepInterval           Duration
 131                 TrustAllContent              bool
 132                 ForwardSlashNameSubstitution string
 133                 S3FolderObjects              bool
 134
 135                 BlobMissingReport        string
 136                 BalancePeriod            Duration
 137                 BalanceCollectionBatch   int
 138                 BalanceCollectionBuffers int
 139                 BalanceTimeout           Duration
 140                 BalanceUpdateLimit       int
 141
 142                 WebDAVCache WebDAVCacheConfig
 143
 144                 KeepproxyPermission UploadDownloadRolePermissions
 145                 WebDAVPermission    UploadDownloadRolePermissions
 146                 WebDAVLogEvents     bool
 147         }
 148         Git struct {
 149                 GitCommand   string
 150                 GitoliteHome string
 151                 Repositories string
 152         }
 153         Login struct {
 154                 LDAP struct {
 155                         Enable             bool
 156                         URL                URL
 157                         StartTLS           bool
 158                         InsecureTLS        bool
 159                         StripDomain        string
 160                         AppendDomain       string
 161                         SearchAttribute    string
 162                         SearchBindUser     string
 163                         SearchBindPassword string
 164                         SearchBase         string
 165                         SearchFilters      string
 166                         EmailAttribute     string
 167                         UsernameAttribute  string
 168                 }
 169                 Google struct {
 170                         Enable                          bool
 171                         ClientID                        string
 172                         ClientSecret                    string
 173                         AlternateEmailAddresses         bool
 174                         AuthenticationRequestParameters map[string]string
 175                 }
 176                 OpenIDConnect struct {
 177                         Enable                          bool
 178                         Issuer                          string
 179                         ClientID                        string
 180                         ClientSecret                    string
 181                         EmailClaim                      string
 182                         EmailVerifiedClaim              string
 183                         UsernameClaim                   string
 184                         AcceptAccessToken               bool
 185                         AcceptAccessTokenScope          string
 186                         AuthenticationRequestParameters map[string]string
 187                 }
 188                 PAM struct {
 189                         Enable             bool
 190                         Service            string
 191                         DefaultEmailDomain string
 192                 }
 193                 Test struct {
 194                         Enable bool
 195                         Users  map[string]TestUser
 196                 }
 197                 LoginCluster       string
 198                 RemoteTokenRefresh Duration
 199                 TokenLifetime      Duration
 200                 TrustedClients     map[string]struct{}
 201                 IssueTrustedTokens bool
 202         }
 203         Mail struct {
 204                 MailchimpAPIKey                string
 205                 MailchimpListID                string
 206                 SendUserSetupNotificationEmail bool
 207                 IssueReporterEmailFrom         string
 208                 IssueReporterEmailTo           string
 209                 SupportEmailAddress            string
 210                 EmailFrom                      string
 211         }
 212         SystemLogs struct {
 213                 LogLevel                string
 214                 Format                  string
 215                 MaxRequestLogParamsSize int
 216         }
 217         TLS struct {
 218                 Certificate string
 219                 Key         string
 220                 Insecure    bool
 221         }
 222         Users struct {
 223                 ActivatedUsersAreVisibleToOthers      bool
 224                 AnonymousUserToken                    string
 225                 AdminNotifierEmailFrom                string
 226                 AutoAdminFirstUser                    bool
 227                 AutoAdminUserWithEmail                string
 228                 AutoSetupNewUsers                     bool
 229                 AutoSetupNewUsersWithRepository       bool
 230                 AutoSetupNewUsersWithVmUUID           string
 231                 AutoSetupUsernameBlacklist            StringSet
 232                 EmailSubjectPrefix                    string
 233                 NewInactiveUserNotificationRecipients StringSet
 234                 NewUserNotificationRecipients         StringSet
 235                 NewUsersAreActive                     bool
 236                 UserNotifierEmailFrom                 string
 237                 UserNotifierEmailBcc                  StringSet
 238                 UserProfileNotificationAddress        string
 239                 PreferDomainForUsername               string
 240                 UserSetupMailText                     string
 241         }
 242         StorageClasses map[string]StorageClassConfig
 243         Volumes        map[string]Volume
 244         Workbench      struct {
 245                 ActivationContactLink            string
 246                 APIClientConnectTimeout          Duration
 247                 APIClientReceiveTimeout          Duration
 248                 APIResponseCompression           bool
 249                 ApplicationMimetypesWithViewIcon StringSet
 250                 ArvadosDocsite                   string
 251                 ArvadosPublicDataDocURL          string
 252                 DefaultOpenIdPrefix              string
 253                 EnableGettingStartedPopup        bool
 254                 EnablePublicProjectsPage         bool
 255                 FileViewersConfigURL             string
 256                 LogViewerMaxBytes                ByteSize
 257                 MultiSiteSearch                  string
 258                 ProfilingEnabled                 bool
 259                 Repositories                     bool
 260                 RepositoryCache                  string
 261                 RunningJobLogRecordsToFetch      int
 262                 SecretKeyBase                    string
 263                 ShowRecentCollectionsOnDashboard bool
 264                 ShowUserAgreementInline          bool
 265                 ShowUserNotifications            bool
 266                 SiteName                         string
 267                 Theme                            string
 268                 UserProfileFormFields            map[string]struct {
 269                         Type                 string
 270                         FormFieldTitle       string
 271                         FormFieldDescription string
 272                         Required             bool
 273                         Position             int
 274                         Options              map[string]struct{}
 275                 }
 276                 UserProfileFormMessage string
 277                 VocabularyURL          string
 278                 WelcomePageHTML        string
 279                 InactivePageHTML       string
 280                 SSHHelpPageHTML        string
 281                 SSHHelpHostSuffix      string
 282                 IdleTimeout            Duration
 283         }
 284 }
 285
 286 type StorageClassConfig struct {
 287         Default  bool
 288         Priority int
 289 }
 290
 291 type Volume struct {
 292         AccessViaHosts   map[URL]VolumeAccess
 293         ReadOnly         bool
 294         Replication      int
 295         StorageClasses   map[string]bool
 296         Driver           string
 297         DriverParameters json.RawMessage
 298 }
 299
 300 type S3VolumeDriverParameters struct {
 301         IAMRole            string
 302         AccessKeyID        string
 303         SecretAccessKey    string
 304         Endpoint           string
 305         Region             string
 306         Bucket             string
 307         LocationConstraint bool
 308         V2Signature        bool
 309         UseAWSS3v2Driver   bool
 310         IndexPageSize      int
 311         ConnectTimeout     Duration
 312         ReadTimeout        Duration
 313         RaceWindow         Duration
 314         UnsafeDelete       bool
 315         PrefixLength       int
 316 }
 317
 318 type AzureVolumeDriverParameters struct {
 319         StorageAccountName   string
 320         StorageAccountKey    string
 321         StorageBaseURL       string
 322         ContainerName        string
 323         RequestTimeout       Duration
 324         ListBlobsRetryDelay  Duration
 325         ListBlobsMaxAttempts int
 326 }
 327
 328 type DirectoryVolumeDriverParameters struct {
 329         Root      string
 330         Serialize bool
 331 }
 332
 333 type VolumeAccess struct {
 334         ReadOnly bool
 335 }
 336
 337 type Services struct {
 338         Composer       Service
 339         Controller     Service
 340         DispatchCloud  Service
 341         DispatchLSF    Service
 342         GitHTTP        Service
 343         GitSSH         Service
 344         Health         Service
 345         Keepbalance    Service
 346         Keepproxy      Service
 347         Keepstore      Service
 348         RailsAPI       Service
 349         WebDAVDownload Service
 350         WebDAV         Service
 351         WebShell       Service
 352         Websocket      Service
 353         Workbench1     Service
 354         Workbench2     Service
 355 }
 356
 357 type Service struct {
 358         InternalURLs map[URL]ServiceInstance
 359         ExternalURL  URL
 360 }
 361
 362 type TestUser struct {
 363         Email    string
 364         Password string
 365 }
 366
 367 // URL is a url.URL that is also usable as a JSON key/value.
 368 type URL url.URL
 369
 370 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 371 // used as a JSON key/value.
 372 func (su *URL) UnmarshalText(text []byte) error {
 373         u, err := url.Parse(string(text))
 374         if err == nil {
 375                 *su = URL(*u)
 376                 if su.Path == "" && su.Host != "" {
 377                         // http://example really means http://example/
 378                         su.Path = "/"
 379                 }
 380         }
 381         return err
 382 }
 383
 384 func (su URL) MarshalText() ([]byte, error) {
 385         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 386 }
 387
 388 func (su URL) String() string {
 389         return (*url.URL)(&su).String()
 390 }
 391
 392 type ServiceInstance struct {
 393         Rendezvous string `json:",omitempty"`
 394 }
 395
 396 type PostgreSQL struct {
 397         Connection     PostgreSQLConnection
 398         ConnectionPool int
 399 }
 400
 401 type PostgreSQLConnection map[string]string
 402
 403 type RemoteCluster struct {
 404         Host          string
 405         Proxy         bool
 406         Scheme        string
 407         Insecure      bool
 408         ActivateUsers bool
 409 }
 410
 411 type InstanceType struct {
 412         Name            string
 413         ProviderType    string
 414         VCPUs           int
 415         RAM             ByteSize
 416         Scratch         ByteSize
 417         IncludedScratch ByteSize
 418         AddedScratch    ByteSize
 419         Price           float64
 420         Preemptible     bool
 421 }
 422
 423 type ContainersConfig struct {
 424         CloudVMs                    CloudVMsConfig
 425         CrunchRunCommand            string
 426         CrunchRunArgumentsList      []string
 427         DefaultKeepCacheRAM         ByteSize
 428         DispatchPrivateKey          string
 429         LogReuseDecisions           bool
 430         MaxComputeVMs               int
 431         MaxDispatchAttempts         int
 432         MaxRetryAttempts            int
 433         MinRetryPeriod              Duration
 434         ReserveExtraRAM             ByteSize
 435         StaleLockTimeout            Duration
 436         SupportedDockerImageFormats StringSet
 437         UsePreemptibleInstances     bool
 438         RuntimeEngine               string
 439         LocalKeepBlobBuffersPerVCPU int
 440         LocalKeepLogsToContainerLog string
 441
 442         JobsAPI struct {
 443                 Enable         string
 444                 GitInternalDir string
 445         }
 446         Logging struct {
 447                 MaxAge                       Duration
 448                 LogBytesPerEvent             int
 449                 LogSecondsBetweenEvents      Duration
 450                 LogThrottlePeriod            Duration
 451                 LogThrottleBytes             int
 452                 LogThrottleLines             int
 453                 LimitLogBytesPerJob          int
 454                 LogPartialLineThrottlePeriod Duration
 455                 LogUpdatePeriod              Duration
 456                 LogUpdateSize                ByteSize
 457         }
 458         ShellAccess struct {
 459                 Admin bool
 460                 User  bool
 461         }
 462         SLURM struct {
 463                 PrioritySpread             int64
 464                 SbatchArgumentsList        []string
 465                 SbatchEnvironmentVariables map[string]string
 466                 Managed                    struct {
 467                         DNSServerConfDir       string
 468                         DNSServerConfTemplate  string
 469                         DNSServerReloadCommand string
 470                         DNSServerUpdateCommand string
 471                         ComputeNodeDomain      string
 472                         ComputeNodeNameservers StringSet
 473                         AssignNodeHostname     string
 474                 }
 475         }
 476         LSF struct {
 477                 BsubSudoUser      string
 478                 BsubArgumentsList []string
 479         }
 480 }
 481
 482 type CloudVMsConfig struct {
 483         Enable bool
 484
 485         BootProbeCommand               string
 486         DeployRunnerBinary             string
 487         ImageID                        string
 488         MaxCloudOpsPerSecond           int
 489         MaxProbesPerSecond             int
 490         MaxConcurrentInstanceCreateOps int
 491         PollInterval                   Duration
 492         ProbeInterval                  Duration
 493         SSHPort                        string
 494         SyncInterval                   Duration
 495         TimeoutBooting                 Duration
 496         TimeoutIdle                    Duration
 497         TimeoutProbe                   Duration
 498         TimeoutShutdown                Duration
 499         TimeoutSignal                  Duration
 500         TimeoutStaleRunLock            Duration
 501         TimeoutTERM                    Duration
 502         ResourceTags                   map[string]string
 503         TagKeyPrefix                   string
 504
 505         Driver           string
 506         DriverParameters json.RawMessage
 507 }
 508
 509 type InstanceTypeMap map[string]InstanceType
 510
 511 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 512
 513 // UnmarshalJSON handles old config files that provide an array of
 514 // instance types instead of a hash.
 515 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 516         fixup := func(t InstanceType) (InstanceType, error) {
 517                 if t.ProviderType == "" {
 518                         t.ProviderType = t.Name
 519                 }
 520                 if t.Scratch == 0 {
 521                         t.Scratch = t.IncludedScratch + t.AddedScratch
 522                 } else if t.AddedScratch == 0 {
 523                         t.AddedScratch = t.Scratch - t.IncludedScratch
 524                 } else if t.IncludedScratch == 0 {
 525                         t.IncludedScratch = t.Scratch - t.AddedScratch
 526                 }
 527
 528                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 529                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 530                 }
 531                 return t, nil
 532         }
 533
 534         if len(data) > 0 && data[0] == '[' {
 535                 var arr []InstanceType
 536                 err := json.Unmarshal(data, &arr)
 537                 if err != nil {
 538                         return err
 539                 }
 540                 if len(arr) == 0 {
 541                         *it = nil
 542                         return nil
 543                 }
 544                 *it = make(map[string]InstanceType, len(arr))
 545                 for _, t := range arr {
 546                         if _, ok := (*it)[t.Name]; ok {
 547                                 return errDuplicateInstanceTypeName
 548                         }
 549                         t, err := fixup(t)
 550                         if err != nil {
 551                                 return err
 552                         }
 553                         (*it)[t.Name] = t
 554                 }
 555                 return nil
 556         }
 557         var hash map[string]InstanceType
 558         err := json.Unmarshal(data, &hash)
 559         if err != nil {
 560                 return err
 561         }
 562         // Fill in Name field (and ProviderType field, if not
 563         // specified) using hash key.
 564         *it = InstanceTypeMap(hash)
 565         for name, t := range *it {
 566                 t.Name = name
 567                 t, err := fixup(t)
 568                 if err != nil {
 569                         return err
 570                 }
 571                 (*it)[name] = t
 572         }
 573         return nil
 574 }
 575
 576 type StringSet map[string]struct{}
 577
 578 // UnmarshalJSON handles old config files that provide an array of
 579 // instance types instead of a hash.
 580 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 581         if len(data) > 0 && data[0] == '[' {
 582                 var arr []string
 583                 err := json.Unmarshal(data, &arr)
 584                 if err != nil {
 585                         return err
 586                 }
 587                 if len(arr) == 0 {
 588                         *ss = nil
 589                         return nil
 590                 }
 591                 *ss = make(map[string]struct{}, len(arr))
 592                 for _, t := range arr {
 593                         (*ss)[t] = struct{}{}
 594                 }
 595                 return nil
 596         }
 597         var hash map[string]struct{}
 598         err := json.Unmarshal(data, &hash)
 599         if err != nil {
 600                 return err
 601         }
 602         *ss = make(map[string]struct{}, len(hash))
 603         for t := range hash {
 604                 (*ss)[t] = struct{}{}
 605         }
 606
 607         return nil
 608 }
 609
 610 type ServiceName string
 611
 612 const (
 613         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 614         ServiceNameController    ServiceName = "arvados-controller"
 615         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 616         ServiceNameDispatchLSF   ServiceName = "arvados-dispatch-lsf"
 617         ServiceNameHealth        ServiceName = "arvados-health"
 618         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 619         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 620         ServiceNameWebsocket     ServiceName = "arvados-ws"
 621         ServiceNameKeepbalance   ServiceName = "keep-balance"
 622         ServiceNameKeepweb       ServiceName = "keep-web"
 623         ServiceNameKeepproxy     ServiceName = "keepproxy"
 624         ServiceNameKeepstore     ServiceName = "keepstore"
 625 )
 626
 627 // Map returns all services as a map, suitable for iterating over all
 628 // services or looking up a service by name.
 629 func (svcs Services) Map() map[ServiceName]Service {
 630         return map[ServiceName]Service{
 631                 ServiceNameRailsAPI:      svcs.RailsAPI,
 632                 ServiceNameController:    svcs.Controller,
 633                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 634                 ServiceNameDispatchLSF:   svcs.DispatchLSF,
 635                 ServiceNameHealth:        svcs.Health,
 636                 ServiceNameWorkbench1:    svcs.Workbench1,
 637                 ServiceNameWorkbench2:    svcs.Workbench2,
 638                 ServiceNameWebsocket:     svcs.Websocket,
 639                 ServiceNameKeepbalance:   svcs.Keepbalance,
 640                 ServiceNameKeepweb:       svcs.WebDAV,
 641                 ServiceNameKeepproxy:     svcs.Keepproxy,
 642                 ServiceNameKeepstore:     svcs.Keepstore,
 643         }
 644 }