sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.arvados.org/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         }
  21         return "/etc/arvados/config.yml"
  22 }()
  23
  24 type Config struct {
  25         Clusters         map[string]Cluster
  26         AutoReloadConfig bool
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         cc, ok := sc.Clusters[clusterID]
  53         if !ok {
  54                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  55         }
  56         cc.ClusterID = clusterID
  57         return &cc, nil
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxBlockEntries      int
  64         MaxCollectionEntries int
  65         MaxCollectionBytes   int64
  66         MaxPermissionEntries int
  67         MaxUUIDEntries       int
  68         MaxSessions          int
  69 }
  70
  71 type UploadDownloadPermission struct {
  72         Upload   bool
  73         Download bool
  74 }
  75
  76 type UploadDownloadRolePermissions struct {
  77         User  UploadDownloadPermission
  78         Admin UploadDownloadPermission
  79 }
  80
  81 type Cluster struct {
  82         ClusterID       string `json:"-"`
  83         ManagementToken string
  84         SystemRootToken string
  85         Services        Services
  86         InstanceTypes   InstanceTypeMap
  87         Containers      ContainersConfig
  88         RemoteClusters  map[string]RemoteCluster
  89         PostgreSQL      PostgreSQL
  90
  91         API struct {
  92                 AsyncPermissionsUpdateInterval Duration
  93                 DisabledAPIs                   StringSet
  94                 MaxIndexDatabaseRead           int
  95                 MaxItemsPerResponse            int
  96                 MaxConcurrentRequests          int
  97                 MaxKeepBlobBuffers             int
  98                 MaxRequestAmplification        int
  99                 MaxRequestSize                 int
 100                 MaxTokenLifetime               Duration
 101                 RequestTimeout                 Duration
 102                 SendTimeout                    Duration
 103                 WebsocketClientEventQueue      int
 104                 WebsocketServerEventQueue      int
 105                 KeepServiceRequestTimeout      Duration
 106         }
 107         AuditLogs struct {
 108                 MaxAge             Duration
 109                 MaxDeleteBatch     int
 110                 UnloggedAttributes StringSet
 111         }
 112         Collections struct {
 113                 BlobSigning              bool
 114                 BlobSigningKey           string
 115                 BlobSigningTTL           Duration
 116                 BlobTrash                bool
 117                 BlobTrashLifetime        Duration
 118                 BlobTrashCheckInterval   Duration
 119                 BlobTrashConcurrency     int
 120                 BlobDeleteConcurrency    int
 121                 BlobReplicateConcurrency int
 122                 CollectionVersioning     bool
 123                 DefaultTrashLifetime     Duration
 124                 DefaultReplication       int
 125                 ManagedProperties        map[string]struct {
 126                         Value     interface{}
 127                         Function  string
 128                         Protected bool
 129                 }
 130                 PreserveVersionIfIdle        Duration
 131                 TrashSweepInterval           Duration
 132                 TrustAllContent              bool
 133                 ForwardSlashNameSubstitution string
 134                 S3FolderObjects              bool
 135
 136                 BlobMissingReport        string
 137                 BalancePeriod            Duration
 138                 BalanceCollectionBatch   int
 139                 BalanceCollectionBuffers int
 140                 BalanceTimeout           Duration
 141                 BalanceUpdateLimit       int
 142
 143                 WebDAVCache WebDAVCacheConfig
 144
 145                 KeepproxyPermission UploadDownloadRolePermissions
 146                 WebDAVPermission    UploadDownloadRolePermissions
 147                 WebDAVLogEvents     bool
 148         }
 149         Git struct {
 150                 GitCommand   string
 151                 GitoliteHome string
 152                 Repositories string
 153         }
 154         Login struct {
 155                 LDAP struct {
 156                         Enable             bool
 157                         URL                URL
 158                         StartTLS           bool
 159                         InsecureTLS        bool
 160                         StripDomain        string
 161                         AppendDomain       string
 162                         SearchAttribute    string
 163                         SearchBindUser     string
 164                         SearchBindPassword string
 165                         SearchBase         string
 166                         SearchFilters      string
 167                         EmailAttribute     string
 168                         UsernameAttribute  string
 169                 }
 170                 Google struct {
 171                         Enable                          bool
 172                         ClientID                        string
 173                         ClientSecret                    string
 174                         AlternateEmailAddresses         bool
 175                         AuthenticationRequestParameters map[string]string
 176                 }
 177                 OpenIDConnect struct {
 178                         Enable                          bool
 179                         Issuer                          string
 180                         ClientID                        string
 181                         ClientSecret                    string
 182                         EmailClaim                      string
 183                         EmailVerifiedClaim              string
 184                         UsernameClaim                   string
 185                         AcceptAccessToken               bool
 186                         AcceptAccessTokenScope          string
 187                         AuthenticationRequestParameters map[string]string
 188                 }
 189                 PAM struct {
 190                         Enable             bool
 191                         Service            string
 192                         DefaultEmailDomain string
 193                 }
 194                 Test struct {
 195                         Enable bool
 196                         Users  map[string]TestUser
 197                 }
 198                 LoginCluster       string
 199                 RemoteTokenRefresh Duration
 200                 TokenLifetime      Duration
 201                 TrustedClients     map[string]struct{}
 202                 IssueTrustedTokens bool
 203         }
 204         Mail struct {
 205                 MailchimpAPIKey                string
 206                 MailchimpListID                string
 207                 SendUserSetupNotificationEmail bool
 208                 IssueReporterEmailFrom         string
 209                 IssueReporterEmailTo           string
 210                 SupportEmailAddress            string
 211                 EmailFrom                      string
 212         }
 213         SystemLogs struct {
 214                 LogLevel                string
 215                 Format                  string
 216                 MaxRequestLogParamsSize int
 217         }
 218         TLS struct {
 219                 Certificate string
 220                 Key         string
 221                 Insecure    bool
 222         }
 223         Users struct {
 224                 AnonymousUserToken                    string
 225                 AdminNotifierEmailFrom                string
 226                 AutoAdminFirstUser                    bool
 227                 AutoAdminUserWithEmail                string
 228                 AutoSetupNewUsers                     bool
 229                 AutoSetupNewUsersWithRepository       bool
 230                 AutoSetupNewUsersWithVmUUID           string
 231                 AutoSetupUsernameBlacklist            StringSet
 232                 EmailSubjectPrefix                    string
 233                 NewInactiveUserNotificationRecipients StringSet
 234                 NewUserNotificationRecipients         StringSet
 235                 NewUsersAreActive                     bool
 236                 UserNotifierEmailFrom                 string
 237                 UserProfileNotificationAddress        string
 238                 PreferDomainForUsername               string
 239                 UserSetupMailText                     string
 240         }
 241         StorageClasses map[string]StorageClassConfig
 242         Volumes        map[string]Volume
 243         Workbench      struct {
 244                 ActivationContactLink            string
 245                 APIClientConnectTimeout          Duration
 246                 APIClientReceiveTimeout          Duration
 247                 APIResponseCompression           bool
 248                 ApplicationMimetypesWithViewIcon StringSet
 249                 ArvadosDocsite                   string
 250                 ArvadosPublicDataDocURL          string
 251                 DefaultOpenIdPrefix              string
 252                 EnableGettingStartedPopup        bool
 253                 EnablePublicProjectsPage         bool
 254                 FileViewersConfigURL             string
 255                 LogViewerMaxBytes                ByteSize
 256                 MultiSiteSearch                  string
 257                 ProfilingEnabled                 bool
 258                 Repositories                     bool
 259                 RepositoryCache                  string
 260                 RunningJobLogRecordsToFetch      int
 261                 SecretKeyBase                    string
 262                 ShowRecentCollectionsOnDashboard bool
 263                 ShowUserAgreementInline          bool
 264                 ShowUserNotifications            bool
 265                 SiteName                         string
 266                 Theme                            string
 267                 UserProfileFormFields            map[string]struct {
 268                         Type                 string
 269                         FormFieldTitle       string
 270                         FormFieldDescription string
 271                         Required             bool
 272                         Position             int
 273                         Options              map[string]struct{}
 274                 }
 275                 UserProfileFormMessage string
 276                 VocabularyURL          string
 277                 WelcomePageHTML        string
 278                 InactivePageHTML       string
 279                 SSHHelpPageHTML        string
 280                 SSHHelpHostSuffix      string
 281                 IdleTimeout            Duration
 282         }
 283 }
 284
 285 type StorageClassConfig struct {
 286         Default  bool
 287         Priority int
 288 }
 289
 290 type Volume struct {
 291         AccessViaHosts   map[URL]VolumeAccess
 292         ReadOnly         bool
 293         Replication      int
 294         StorageClasses   map[string]bool
 295         Driver           string
 296         DriverParameters json.RawMessage
 297 }
 298
 299 type S3VolumeDriverParameters struct {
 300         IAMRole            string
 301         AccessKeyID        string
 302         SecretAccessKey    string
 303         Endpoint           string
 304         Region             string
 305         Bucket             string
 306         LocationConstraint bool
 307         V2Signature        bool
 308         UseAWSS3v2Driver   bool
 309         IndexPageSize      int
 310         ConnectTimeout     Duration
 311         ReadTimeout        Duration
 312         RaceWindow         Duration
 313         UnsafeDelete       bool
 314 }
 315
 316 type AzureVolumeDriverParameters struct {
 317         StorageAccountName   string
 318         StorageAccountKey    string
 319         StorageBaseURL       string
 320         ContainerName        string
 321         RequestTimeout       Duration
 322         ListBlobsRetryDelay  Duration
 323         ListBlobsMaxAttempts int
 324 }
 325
 326 type DirectoryVolumeDriverParameters struct {
 327         Root      string
 328         Serialize bool
 329 }
 330
 331 type VolumeAccess struct {
 332         ReadOnly bool
 333 }
 334
 335 type Services struct {
 336         Composer       Service
 337         Controller     Service
 338         DispatchCloud  Service
 339         DispatchLSF    Service
 340         GitHTTP        Service
 341         GitSSH         Service
 342         Health         Service
 343         Keepbalance    Service
 344         Keepproxy      Service
 345         Keepstore      Service
 346         RailsAPI       Service
 347         WebDAVDownload Service
 348         WebDAV         Service
 349         WebShell       Service
 350         Websocket      Service
 351         Workbench1     Service
 352         Workbench2     Service
 353 }
 354
 355 type Service struct {
 356         InternalURLs map[URL]ServiceInstance
 357         ExternalURL  URL
 358 }
 359
 360 type TestUser struct {
 361         Email    string
 362         Password string
 363 }
 364
 365 // URL is a url.URL that is also usable as a JSON key/value.
 366 type URL url.URL
 367
 368 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 369 // used as a JSON key/value.
 370 func (su *URL) UnmarshalText(text []byte) error {
 371         u, err := url.Parse(string(text))
 372         if err == nil {
 373                 *su = URL(*u)
 374                 if su.Path == "" && su.Host != "" {
 375                         // http://example really means http://example/
 376                         su.Path = "/"
 377                 }
 378         }
 379         return err
 380 }
 381
 382 func (su URL) MarshalText() ([]byte, error) {
 383         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 384 }
 385
 386 func (su URL) String() string {
 387         return (*url.URL)(&su).String()
 388 }
 389
 390 type ServiceInstance struct {
 391         Rendezvous string `json:",omitempty"`
 392 }
 393
 394 type PostgreSQL struct {
 395         Connection     PostgreSQLConnection
 396         ConnectionPool int
 397 }
 398
 399 type PostgreSQLConnection map[string]string
 400
 401 type RemoteCluster struct {
 402         Host          string
 403         Proxy         bool
 404         Scheme        string
 405         Insecure      bool
 406         ActivateUsers bool
 407 }
 408
 409 type InstanceType struct {
 410         Name            string
 411         ProviderType    string
 412         VCPUs           int
 413         RAM             ByteSize
 414         Scratch         ByteSize
 415         IncludedScratch ByteSize
 416         AddedScratch    ByteSize
 417         Price           float64
 418         Preemptible     bool
 419 }
 420
 421 type ContainersConfig struct {
 422         CloudVMs                    CloudVMsConfig
 423         CrunchRunCommand            string
 424         CrunchRunArgumentsList      []string
 425         DefaultKeepCacheRAM         ByteSize
 426         DispatchPrivateKey          string
 427         LogReuseDecisions           bool
 428         MaxComputeVMs               int
 429         MaxDispatchAttempts         int
 430         MaxRetryAttempts            int
 431         MinRetryPeriod              Duration
 432         ReserveExtraRAM             ByteSize
 433         StaleLockTimeout            Duration
 434         SupportedDockerImageFormats StringSet
 435         UsePreemptibleInstances     bool
 436         RuntimeEngine               string
 437
 438         JobsAPI struct {
 439                 Enable         string
 440                 GitInternalDir string
 441         }
 442         Logging struct {
 443                 MaxAge                       Duration
 444                 LogBytesPerEvent             int
 445                 LogSecondsBetweenEvents      Duration
 446                 LogThrottlePeriod            Duration
 447                 LogThrottleBytes             int
 448                 LogThrottleLines             int
 449                 LimitLogBytesPerJob          int
 450                 LogPartialLineThrottlePeriod Duration
 451                 LogUpdatePeriod              Duration
 452                 LogUpdateSize                ByteSize
 453         }
 454         ShellAccess struct {
 455                 Admin bool
 456                 User  bool
 457         }
 458         SLURM struct {
 459                 PrioritySpread             int64
 460                 SbatchArgumentsList        []string
 461                 SbatchEnvironmentVariables map[string]string
 462                 Managed                    struct {
 463                         DNSServerConfDir       string
 464                         DNSServerConfTemplate  string
 465                         DNSServerReloadCommand string
 466                         DNSServerUpdateCommand string
 467                         ComputeNodeDomain      string
 468                         ComputeNodeNameservers StringSet
 469                         AssignNodeHostname     string
 470                 }
 471         }
 472         LSF struct {
 473                 BsubSudoUser      string
 474                 BsubArgumentsList []string
 475         }
 476 }
 477
 478 type CloudVMsConfig struct {
 479         Enable bool
 480
 481         BootProbeCommand               string
 482         DeployRunnerBinary             string
 483         ImageID                        string
 484         MaxCloudOpsPerSecond           int
 485         MaxProbesPerSecond             int
 486         MaxConcurrentInstanceCreateOps int
 487         PollInterval                   Duration
 488         ProbeInterval                  Duration
 489         SSHPort                        string
 490         SyncInterval                   Duration
 491         TimeoutBooting                 Duration
 492         TimeoutIdle                    Duration
 493         TimeoutProbe                   Duration
 494         TimeoutShutdown                Duration
 495         TimeoutSignal                  Duration
 496         TimeoutStaleRunLock            Duration
 497         TimeoutTERM                    Duration
 498         ResourceTags                   map[string]string
 499         TagKeyPrefix                   string
 500
 501         Driver           string
 502         DriverParameters json.RawMessage
 503 }
 504
 505 type InstanceTypeMap map[string]InstanceType
 506
 507 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 508
 509 // UnmarshalJSON handles old config files that provide an array of
 510 // instance types instead of a hash.
 511 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 512         fixup := func(t InstanceType) (InstanceType, error) {
 513                 if t.ProviderType == "" {
 514                         t.ProviderType = t.Name
 515                 }
 516                 if t.Scratch == 0 {
 517                         t.Scratch = t.IncludedScratch + t.AddedScratch
 518                 } else if t.AddedScratch == 0 {
 519                         t.AddedScratch = t.Scratch - t.IncludedScratch
 520                 } else if t.IncludedScratch == 0 {
 521                         t.IncludedScratch = t.Scratch - t.AddedScratch
 522                 }
 523
 524                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 525                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 526                 }
 527                 return t, nil
 528         }
 529
 530         if len(data) > 0 && data[0] == '[' {
 531                 var arr []InstanceType
 532                 err := json.Unmarshal(data, &arr)
 533                 if err != nil {
 534                         return err
 535                 }
 536                 if len(arr) == 0 {
 537                         *it = nil
 538                         return nil
 539                 }
 540                 *it = make(map[string]InstanceType, len(arr))
 541                 for _, t := range arr {
 542                         if _, ok := (*it)[t.Name]; ok {
 543                                 return errDuplicateInstanceTypeName
 544                         }
 545                         t, err := fixup(t)
 546                         if err != nil {
 547                                 return err
 548                         }
 549                         (*it)[t.Name] = t
 550                 }
 551                 return nil
 552         }
 553         var hash map[string]InstanceType
 554         err := json.Unmarshal(data, &hash)
 555         if err != nil {
 556                 return err
 557         }
 558         // Fill in Name field (and ProviderType field, if not
 559         // specified) using hash key.
 560         *it = InstanceTypeMap(hash)
 561         for name, t := range *it {
 562                 t.Name = name
 563                 t, err := fixup(t)
 564                 if err != nil {
 565                         return err
 566                 }
 567                 (*it)[name] = t
 568         }
 569         return nil
 570 }
 571
 572 type StringSet map[string]struct{}
 573
 574 // UnmarshalJSON handles old config files that provide an array of
 575 // instance types instead of a hash.
 576 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 577         if len(data) > 0 && data[0] == '[' {
 578                 var arr []string
 579                 err := json.Unmarshal(data, &arr)
 580                 if err != nil {
 581                         return err
 582                 }
 583                 if len(arr) == 0 {
 584                         *ss = nil
 585                         return nil
 586                 }
 587                 *ss = make(map[string]struct{}, len(arr))
 588                 for _, t := range arr {
 589                         (*ss)[t] = struct{}{}
 590                 }
 591                 return nil
 592         }
 593         var hash map[string]struct{}
 594         err := json.Unmarshal(data, &hash)
 595         if err != nil {
 596                 return err
 597         }
 598         *ss = make(map[string]struct{}, len(hash))
 599         for t := range hash {
 600                 (*ss)[t] = struct{}{}
 601         }
 602
 603         return nil
 604 }
 605
 606 type ServiceName string
 607
 608 const (
 609         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 610         ServiceNameController    ServiceName = "arvados-controller"
 611         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 612         ServiceNameDispatchLSF   ServiceName = "arvados-dispatch-lsf"
 613         ServiceNameHealth        ServiceName = "arvados-health"
 614         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 615         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 616         ServiceNameWebsocket     ServiceName = "arvados-ws"
 617         ServiceNameKeepbalance   ServiceName = "keep-balance"
 618         ServiceNameKeepweb       ServiceName = "keep-web"
 619         ServiceNameKeepproxy     ServiceName = "keepproxy"
 620         ServiceNameKeepstore     ServiceName = "keepstore"
 621 )
 622
 623 // Map returns all services as a map, suitable for iterating over all
 624 // services or looking up a service by name.
 625 func (svcs Services) Map() map[ServiceName]Service {
 626         return map[ServiceName]Service{
 627                 ServiceNameRailsAPI:      svcs.RailsAPI,
 628                 ServiceNameController:    svcs.Controller,
 629                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 630                 ServiceNameDispatchLSF:   svcs.DispatchLSF,
 631                 ServiceNameHealth:        svcs.Health,
 632                 ServiceNameWorkbench1:    svcs.Workbench1,
 633                 ServiceNameWorkbench2:    svcs.Workbench2,
 634                 ServiceNameWebsocket:     svcs.Websocket,
 635                 ServiceNameKeepbalance:   svcs.Keepbalance,
 636                 ServiceNameKeepweb:       svcs.WebDAV,
 637                 ServiceNameKeepproxy:     svcs.Keepproxy,
 638                 ServiceNameKeepstore:     svcs.Keepstore,
 639         }
 640 }