sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.arvados.org/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         }
  21         return "/etc/arvados/config.yml"
  22 }()
  23
  24 type Config struct {
  25         Clusters         map[string]Cluster
  26         AutoReloadConfig bool
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         cc, ok := sc.Clusters[clusterID]
  53         if !ok {
  54                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  55         }
  56         cc.ClusterID = clusterID
  57         return &cc, nil
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxBlockEntries      int
  64         MaxCollectionEntries int
  65         MaxCollectionBytes   int64
  66         MaxUUIDEntries       int
  67         MaxSessions          int
  68 }
  69
  70 type UploadDownloadPermission struct {
  71         Upload   bool
  72         Download bool
  73 }
  74
  75 type UploadDownloadRolePermissions struct {
  76         User  UploadDownloadPermission
  77         Admin UploadDownloadPermission
  78 }
  79
  80 type Cluster struct {
  81         ClusterID       string `json:"-"`
  82         ManagementToken string
  83         SystemRootToken string
  84         Services        Services
  85         InstanceTypes   InstanceTypeMap
  86         Containers      ContainersConfig
  87         RemoteClusters  map[string]RemoteCluster
  88         PostgreSQL      PostgreSQL
  89
  90         API struct {
  91                 AsyncPermissionsUpdateInterval Duration
  92                 DisabledAPIs                   StringSet
  93                 MaxIndexDatabaseRead           int
  94                 MaxItemsPerResponse            int
  95                 MaxConcurrentRequests          int
  96                 MaxKeepBlobBuffers             int
  97                 MaxRequestAmplification        int
  98                 MaxRequestSize                 int
  99                 MaxTokenLifetime               Duration
 100                 RequestTimeout                 Duration
 101                 SendTimeout                    Duration
 102                 WebsocketClientEventQueue      int
 103                 WebsocketServerEventQueue      int
 104                 KeepServiceRequestTimeout      Duration
 105         }
 106         AuditLogs struct {
 107                 MaxAge             Duration
 108                 MaxDeleteBatch     int
 109                 UnloggedAttributes StringSet
 110         }
 111         Collections struct {
 112                 BlobSigning              bool
 113                 BlobSigningKey           string
 114                 BlobSigningTTL           Duration
 115                 BlobTrash                bool
 116                 BlobTrashLifetime        Duration
 117                 BlobTrashCheckInterval   Duration
 118                 BlobTrashConcurrency     int
 119                 BlobDeleteConcurrency    int
 120                 BlobReplicateConcurrency int
 121                 CollectionVersioning     bool
 122                 DefaultTrashLifetime     Duration
 123                 DefaultReplication       int
 124                 ManagedProperties        map[string]struct {
 125                         Value     interface{}
 126                         Function  string
 127                         Protected bool
 128                 }
 129                 PreserveVersionIfIdle        Duration
 130                 TrashSweepInterval           Duration
 131                 TrustAllContent              bool
 132                 ForwardSlashNameSubstitution string
 133                 S3FolderObjects              bool
 134
 135                 BlobMissingReport        string
 136                 BalancePeriod            Duration
 137                 BalanceCollectionBatch   int
 138                 BalanceCollectionBuffers int
 139                 BalanceTimeout           Duration
 140                 BalanceUpdateLimit       int
 141
 142                 WebDAVCache WebDAVCacheConfig
 143
 144                 KeepproxyPermission UploadDownloadRolePermissions
 145                 WebDAVPermission    UploadDownloadRolePermissions
 146                 WebDAVLogEvents     bool
 147         }
 148         Git struct {
 149                 GitCommand   string
 150                 GitoliteHome string
 151                 Repositories string
 152         }
 153         Login struct {
 154                 LDAP struct {
 155                         Enable             bool
 156                         URL                URL
 157                         StartTLS           bool
 158                         InsecureTLS        bool
 159                         StripDomain        string
 160                         AppendDomain       string
 161                         SearchAttribute    string
 162                         SearchBindUser     string
 163                         SearchBindPassword string
 164                         SearchBase         string
 165                         SearchFilters      string
 166                         EmailAttribute     string
 167                         UsernameAttribute  string
 168                 }
 169                 Google struct {
 170                         Enable                          bool
 171                         ClientID                        string
 172                         ClientSecret                    string
 173                         AlternateEmailAddresses         bool
 174                         AuthenticationRequestParameters map[string]string
 175                 }
 176                 OpenIDConnect struct {
 177                         Enable                          bool
 178                         Issuer                          string
 179                         ClientID                        string
 180                         ClientSecret                    string
 181                         EmailClaim                      string
 182                         EmailVerifiedClaim              string
 183                         UsernameClaim                   string
 184                         AcceptAccessToken               bool
 185                         AcceptAccessTokenScope          string
 186                         AuthenticationRequestParameters map[string]string
 187                 }
 188                 PAM struct {
 189                         Enable             bool
 190                         Service            string
 191                         DefaultEmailDomain string
 192                 }
 193                 Test struct {
 194                         Enable bool
 195                         Users  map[string]TestUser
 196                 }
 197                 LoginCluster       string
 198                 RemoteTokenRefresh Duration
 199                 TokenLifetime      Duration
 200                 TrustedClients     map[string]struct{}
 201                 IssueTrustedTokens bool
 202         }
 203         Mail struct {
 204                 MailchimpAPIKey                string
 205                 MailchimpListID                string
 206                 SendUserSetupNotificationEmail bool
 207                 IssueReporterEmailFrom         string
 208                 IssueReporterEmailTo           string
 209                 SupportEmailAddress            string
 210                 EmailFrom                      string
 211         }
 212         SystemLogs struct {
 213                 LogLevel                string
 214                 Format                  string
 215                 MaxRequestLogParamsSize int
 216         }
 217         TLS struct {
 218                 Certificate string
 219                 Key         string
 220                 Insecure    bool
 221         }
 222         Users struct {
 223                 AnonymousUserToken                    string
 224                 AdminNotifierEmailFrom                string
 225                 AutoAdminFirstUser                    bool
 226                 AutoAdminUserWithEmail                string
 227                 AutoSetupNewUsers                     bool
 228                 AutoSetupNewUsersWithRepository       bool
 229                 AutoSetupNewUsersWithVmUUID           string
 230                 AutoSetupUsernameBlacklist            StringSet
 231                 EmailSubjectPrefix                    string
 232                 NewInactiveUserNotificationRecipients StringSet
 233                 NewUserNotificationRecipients         StringSet
 234                 NewUsersAreActive                     bool
 235                 UserNotifierEmailFrom                 string
 236                 UserNotifierEmailBcc                  StringSet
 237                 UserProfileNotificationAddress        string
 238                 PreferDomainForUsername               string
 239                 UserSetupMailText                     string
 240         }
 241         StorageClasses map[string]StorageClassConfig
 242         Volumes        map[string]Volume
 243         Workbench      struct {
 244                 ActivationContactLink            string
 245                 APIClientConnectTimeout          Duration
 246                 APIClientReceiveTimeout          Duration
 247                 APIResponseCompression           bool
 248                 ApplicationMimetypesWithViewIcon StringSet
 249                 ArvadosDocsite                   string
 250                 ArvadosPublicDataDocURL          string
 251                 DefaultOpenIdPrefix              string
 252                 EnableGettingStartedPopup        bool
 253                 EnablePublicProjectsPage         bool
 254                 FileViewersConfigURL             string
 255                 LogViewerMaxBytes                ByteSize
 256                 MultiSiteSearch                  string
 257                 ProfilingEnabled                 bool
 258                 Repositories                     bool
 259                 RepositoryCache                  string
 260                 RunningJobLogRecordsToFetch      int
 261                 SecretKeyBase                    string
 262                 ShowRecentCollectionsOnDashboard bool
 263                 ShowUserAgreementInline          bool
 264                 ShowUserNotifications            bool
 265                 SiteName                         string
 266                 Theme                            string
 267                 UserProfileFormFields            map[string]struct {
 268                         Type                 string
 269                         FormFieldTitle       string
 270                         FormFieldDescription string
 271                         Required             bool
 272                         Position             int
 273                         Options              map[string]struct{}
 274                 }
 275                 UserProfileFormMessage string
 276                 VocabularyURL          string
 277                 WelcomePageHTML        string
 278                 InactivePageHTML       string
 279                 SSHHelpPageHTML        string
 280                 SSHHelpHostSuffix      string
 281                 IdleTimeout            Duration
 282         }
 283 }
 284
 285 type StorageClassConfig struct {
 286         Default  bool
 287         Priority int
 288 }
 289
 290 type Volume struct {
 291         AccessViaHosts   map[URL]VolumeAccess
 292         ReadOnly         bool
 293         Replication      int
 294         StorageClasses   map[string]bool
 295         Driver           string
 296         DriverParameters json.RawMessage
 297 }
 298
 299 type S3VolumeDriverParameters struct {
 300         IAMRole            string
 301         AccessKeyID        string
 302         SecretAccessKey    string
 303         Endpoint           string
 304         Region             string
 305         Bucket             string
 306         LocationConstraint bool
 307         V2Signature        bool
 308         UseAWSS3v2Driver   bool
 309         IndexPageSize      int
 310         ConnectTimeout     Duration
 311         ReadTimeout        Duration
 312         RaceWindow         Duration
 313         UnsafeDelete       bool
 314         PrefixLength       int
 315 }
 316
 317 type AzureVolumeDriverParameters struct {
 318         StorageAccountName   string
 319         StorageAccountKey    string
 320         StorageBaseURL       string
 321         ContainerName        string
 322         RequestTimeout       Duration
 323         ListBlobsRetryDelay  Duration
 324         ListBlobsMaxAttempts int
 325 }
 326
 327 type DirectoryVolumeDriverParameters struct {
 328         Root      string
 329         Serialize bool
 330 }
 331
 332 type VolumeAccess struct {
 333         ReadOnly bool
 334 }
 335
 336 type Services struct {
 337         Composer       Service
 338         Controller     Service
 339         DispatchCloud  Service
 340         DispatchLSF    Service
 341         GitHTTP        Service
 342         GitSSH         Service
 343         Health         Service
 344         Keepbalance    Service
 345         Keepproxy      Service
 346         Keepstore      Service
 347         RailsAPI       Service
 348         WebDAVDownload Service
 349         WebDAV         Service
 350         WebShell       Service
 351         Websocket      Service
 352         Workbench1     Service
 353         Workbench2     Service
 354 }
 355
 356 type Service struct {
 357         InternalURLs map[URL]ServiceInstance
 358         ExternalURL  URL
 359 }
 360
 361 type TestUser struct {
 362         Email    string
 363         Password string
 364 }
 365
 366 // URL is a url.URL that is also usable as a JSON key/value.
 367 type URL url.URL
 368
 369 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 370 // used as a JSON key/value.
 371 func (su *URL) UnmarshalText(text []byte) error {
 372         u, err := url.Parse(string(text))
 373         if err == nil {
 374                 *su = URL(*u)
 375                 if su.Path == "" && su.Host != "" {
 376                         // http://example really means http://example/
 377                         su.Path = "/"
 378                 }
 379         }
 380         return err
 381 }
 382
 383 func (su URL) MarshalText() ([]byte, error) {
 384         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 385 }
 386
 387 func (su URL) String() string {
 388         return (*url.URL)(&su).String()
 389 }
 390
 391 type ServiceInstance struct {
 392         Rendezvous string `json:",omitempty"`
 393 }
 394
 395 type PostgreSQL struct {
 396         Connection     PostgreSQLConnection
 397         ConnectionPool int
 398 }
 399
 400 type PostgreSQLConnection map[string]string
 401
 402 type RemoteCluster struct {
 403         Host          string
 404         Proxy         bool
 405         Scheme        string
 406         Insecure      bool
 407         ActivateUsers bool
 408 }
 409
 410 type InstanceType struct {
 411         Name            string
 412         ProviderType    string
 413         VCPUs           int
 414         RAM             ByteSize
 415         Scratch         ByteSize
 416         IncludedScratch ByteSize
 417         AddedScratch    ByteSize
 418         Price           float64
 419         Preemptible     bool
 420 }
 421
 422 type ContainersConfig struct {
 423         CloudVMs                    CloudVMsConfig
 424         CrunchRunCommand            string
 425         CrunchRunArgumentsList      []string
 426         DefaultKeepCacheRAM         ByteSize
 427         DispatchPrivateKey          string
 428         LogReuseDecisions           bool
 429         MaxComputeVMs               int
 430         MaxDispatchAttempts         int
 431         MaxRetryAttempts            int
 432         MinRetryPeriod              Duration
 433         ReserveExtraRAM             ByteSize
 434         StaleLockTimeout            Duration
 435         SupportedDockerImageFormats StringSet
 436         UsePreemptibleInstances     bool
 437         RuntimeEngine               string
 438         LocalKeepBlobBuffersPerVCPU int
 439         LocalKeepLogsToContainerLog string
 440
 441         JobsAPI struct {
 442                 Enable         string
 443                 GitInternalDir string
 444         }
 445         Logging struct {
 446                 MaxAge                       Duration
 447                 LogBytesPerEvent             int
 448                 LogSecondsBetweenEvents      Duration
 449                 LogThrottlePeriod            Duration
 450                 LogThrottleBytes             int
 451                 LogThrottleLines             int
 452                 LimitLogBytesPerJob          int
 453                 LogPartialLineThrottlePeriod Duration
 454                 LogUpdatePeriod              Duration
 455                 LogUpdateSize                ByteSize
 456         }
 457         ShellAccess struct {
 458                 Admin bool
 459                 User  bool
 460         }
 461         SLURM struct {
 462                 PrioritySpread             int64
 463                 SbatchArgumentsList        []string
 464                 SbatchEnvironmentVariables map[string]string
 465                 Managed                    struct {
 466                         DNSServerConfDir       string
 467                         DNSServerConfTemplate  string
 468                         DNSServerReloadCommand string
 469                         DNSServerUpdateCommand string
 470                         ComputeNodeDomain      string
 471                         ComputeNodeNameservers StringSet
 472                         AssignNodeHostname     string
 473                 }
 474         }
 475         LSF struct {
 476                 BsubSudoUser      string
 477                 BsubArgumentsList []string
 478         }
 479 }
 480
 481 type CloudVMsConfig struct {
 482         Enable bool
 483
 484         BootProbeCommand               string
 485         DeployRunnerBinary             string
 486         ImageID                        string
 487         MaxCloudOpsPerSecond           int
 488         MaxProbesPerSecond             int
 489         MaxConcurrentInstanceCreateOps int
 490         PollInterval                   Duration
 491         ProbeInterval                  Duration
 492         SSHPort                        string
 493         SyncInterval                   Duration
 494         TimeoutBooting                 Duration
 495         TimeoutIdle                    Duration
 496         TimeoutProbe                   Duration
 497         TimeoutShutdown                Duration
 498         TimeoutSignal                  Duration
 499         TimeoutStaleRunLock            Duration
 500         TimeoutTERM                    Duration
 501         ResourceTags                   map[string]string
 502         TagKeyPrefix                   string
 503
 504         Driver           string
 505         DriverParameters json.RawMessage
 506 }
 507
 508 type InstanceTypeMap map[string]InstanceType
 509
 510 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 511
 512 // UnmarshalJSON handles old config files that provide an array of
 513 // instance types instead of a hash.
 514 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 515         fixup := func(t InstanceType) (InstanceType, error) {
 516                 if t.ProviderType == "" {
 517                         t.ProviderType = t.Name
 518                 }
 519                 if t.Scratch == 0 {
 520                         t.Scratch = t.IncludedScratch + t.AddedScratch
 521                 } else if t.AddedScratch == 0 {
 522                         t.AddedScratch = t.Scratch - t.IncludedScratch
 523                 } else if t.IncludedScratch == 0 {
 524                         t.IncludedScratch = t.Scratch - t.AddedScratch
 525                 }
 526
 527                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 528                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 529                 }
 530                 return t, nil
 531         }
 532
 533         if len(data) > 0 && data[0] == '[' {
 534                 var arr []InstanceType
 535                 err := json.Unmarshal(data, &arr)
 536                 if err != nil {
 537                         return err
 538                 }
 539                 if len(arr) == 0 {
 540                         *it = nil
 541                         return nil
 542                 }
 543                 *it = make(map[string]InstanceType, len(arr))
 544                 for _, t := range arr {
 545                         if _, ok := (*it)[t.Name]; ok {
 546                                 return errDuplicateInstanceTypeName
 547                         }
 548                         t, err := fixup(t)
 549                         if err != nil {
 550                                 return err
 551                         }
 552                         (*it)[t.Name] = t
 553                 }
 554                 return nil
 555         }
 556         var hash map[string]InstanceType
 557         err := json.Unmarshal(data, &hash)
 558         if err != nil {
 559                 return err
 560         }
 561         // Fill in Name field (and ProviderType field, if not
 562         // specified) using hash key.
 563         *it = InstanceTypeMap(hash)
 564         for name, t := range *it {
 565                 t.Name = name
 566                 t, err := fixup(t)
 567                 if err != nil {
 568                         return err
 569                 }
 570                 (*it)[name] = t
 571         }
 572         return nil
 573 }
 574
 575 type StringSet map[string]struct{}
 576
 577 // UnmarshalJSON handles old config files that provide an array of
 578 // instance types instead of a hash.
 579 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 580         if len(data) > 0 && data[0] == '[' {
 581                 var arr []string
 582                 err := json.Unmarshal(data, &arr)
 583                 if err != nil {
 584                         return err
 585                 }
 586                 if len(arr) == 0 {
 587                         *ss = nil
 588                         return nil
 589                 }
 590                 *ss = make(map[string]struct{}, len(arr))
 591                 for _, t := range arr {
 592                         (*ss)[t] = struct{}{}
 593                 }
 594                 return nil
 595         }
 596         var hash map[string]struct{}
 597         err := json.Unmarshal(data, &hash)
 598         if err != nil {
 599                 return err
 600         }
 601         *ss = make(map[string]struct{}, len(hash))
 602         for t := range hash {
 603                 (*ss)[t] = struct{}{}
 604         }
 605
 606         return nil
 607 }
 608
 609 type ServiceName string
 610
 611 const (
 612         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 613         ServiceNameController    ServiceName = "arvados-controller"
 614         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 615         ServiceNameDispatchLSF   ServiceName = "arvados-dispatch-lsf"
 616         ServiceNameHealth        ServiceName = "arvados-health"
 617         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 618         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 619         ServiceNameWebsocket     ServiceName = "arvados-ws"
 620         ServiceNameKeepbalance   ServiceName = "keep-balance"
 621         ServiceNameKeepweb       ServiceName = "keep-web"
 622         ServiceNameKeepproxy     ServiceName = "keepproxy"
 623         ServiceNameKeepstore     ServiceName = "keepstore"
 624 )
 625
 626 // Map returns all services as a map, suitable for iterating over all
 627 // services or looking up a service by name.
 628 func (svcs Services) Map() map[ServiceName]Service {
 629         return map[ServiceName]Service{
 630                 ServiceNameRailsAPI:      svcs.RailsAPI,
 631                 ServiceNameController:    svcs.Controller,
 632                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 633                 ServiceNameDispatchLSF:   svcs.DispatchLSF,
 634                 ServiceNameHealth:        svcs.Health,
 635                 ServiceNameWorkbench1:    svcs.Workbench1,
 636                 ServiceNameWorkbench2:    svcs.Workbench2,
 637                 ServiceNameWebsocket:     svcs.Websocket,
 638                 ServiceNameKeepbalance:   svcs.Keepbalance,
 639                 ServiceNameKeepweb:       svcs.WebDAV,
 640                 ServiceNameKeepproxy:     svcs.Keepproxy,
 641                 ServiceNameKeepstore:     svcs.Keepstore,
 642         }
 643 }