sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.arvados.org/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         }
  21         return "/etc/arvados/config.yml"
  22 }()
  23
  24 type Config struct {
  25         Clusters         map[string]Cluster
  26         AutoReloadConfig bool
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         cc, ok := sc.Clusters[clusterID]
  53         if !ok {
  54                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  55         }
  56         cc.ClusterID = clusterID
  57         return &cc, nil
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxBlockEntries      int
  64         MaxCollectionEntries int
  65         MaxCollectionBytes   int64
  66         MaxUUIDEntries       int
  67         MaxSessions          int
  68 }
  69
  70 type UploadDownloadPermission struct {
  71         Upload   bool
  72         Download bool
  73 }
  74
  75 type UploadDownloadRolePermissions struct {
  76         User  UploadDownloadPermission
  77         Admin UploadDownloadPermission
  78 }
  79
  80 type ManagedProperties map[string]struct {
  81         Value     interface{}
  82         Function  string
  83         Protected bool
  84 }
  85
  86 type Cluster struct {
  87         ClusterID       string `json:"-"`
  88         ManagementToken string
  89         SystemRootToken string
  90         Services        Services
  91         InstanceTypes   InstanceTypeMap
  92         Containers      ContainersConfig
  93         RemoteClusters  map[string]RemoteCluster
  94         PostgreSQL      PostgreSQL
  95
  96         API struct {
  97                 AsyncPermissionsUpdateInterval Duration
  98                 DisabledAPIs                   StringSet
  99                 MaxIndexDatabaseRead           int
 100                 MaxItemsPerResponse            int
 101                 MaxConcurrentRequests          int
 102                 MaxKeepBlobBuffers             int
 103                 MaxRequestAmplification        int
 104                 MaxRequestSize                 int
 105                 MaxTokenLifetime               Duration
 106                 RequestTimeout                 Duration
 107                 SendTimeout                    Duration
 108                 WebsocketClientEventQueue      int
 109                 WebsocketServerEventQueue      int
 110                 KeepServiceRequestTimeout      Duration
 111                 VocabularyPath                 string
 112         }
 113         AuditLogs struct {
 114                 MaxAge             Duration
 115                 MaxDeleteBatch     int
 116                 UnloggedAttributes StringSet
 117         }
 118         Collections struct {
 119                 BlobSigning                  bool
 120                 BlobSigningKey               string
 121                 BlobSigningTTL               Duration
 122                 BlobTrash                    bool
 123                 BlobTrashLifetime            Duration
 124                 BlobTrashCheckInterval       Duration
 125                 BlobTrashConcurrency         int
 126                 BlobDeleteConcurrency        int
 127                 BlobReplicateConcurrency     int
 128                 CollectionVersioning         bool
 129                 DefaultTrashLifetime         Duration
 130                 DefaultReplication           int
 131                 ManagedProperties            ManagedProperties
 132                 PreserveVersionIfIdle        Duration
 133                 TrashSweepInterval           Duration
 134                 TrustAllContent              bool
 135                 ForwardSlashNameSubstitution string
 136                 S3FolderObjects              bool
 137
 138                 BlobMissingReport        string
 139                 BalancePeriod            Duration
 140                 BalanceCollectionBatch   int
 141                 BalanceCollectionBuffers int
 142                 BalanceTimeout           Duration
 143                 BalanceUpdateLimit       int
 144
 145                 WebDAVCache WebDAVCacheConfig
 146
 147                 KeepproxyPermission UploadDownloadRolePermissions
 148                 WebDAVPermission    UploadDownloadRolePermissions
 149                 WebDAVLogEvents     bool
 150         }
 151         Git struct {
 152                 GitCommand   string
 153                 GitoliteHome string
 154                 Repositories string
 155         }
 156         Login struct {
 157                 LDAP struct {
 158                         Enable             bool
 159                         URL                URL
 160                         StartTLS           bool
 161                         InsecureTLS        bool
 162                         StripDomain        string
 163                         AppendDomain       string
 164                         SearchAttribute    string
 165                         SearchBindUser     string
 166                         SearchBindPassword string
 167                         SearchBase         string
 168                         SearchFilters      string
 169                         EmailAttribute     string
 170                         UsernameAttribute  string
 171                 }
 172                 Google struct {
 173                         Enable                          bool
 174                         ClientID                        string
 175                         ClientSecret                    string
 176                         AlternateEmailAddresses         bool
 177                         AuthenticationRequestParameters map[string]string
 178                 }
 179                 OpenIDConnect struct {
 180                         Enable                          bool
 181                         Issuer                          string
 182                         ClientID                        string
 183                         ClientSecret                    string
 184                         EmailClaim                      string
 185                         EmailVerifiedClaim              string
 186                         UsernameClaim                   string
 187                         AcceptAccessToken               bool
 188                         AcceptAccessTokenScope          string
 189                         AuthenticationRequestParameters map[string]string
 190                 }
 191                 PAM struct {
 192                         Enable             bool
 193                         Service            string
 194                         DefaultEmailDomain string
 195                 }
 196                 Test struct {
 197                         Enable bool
 198                         Users  map[string]TestUser
 199                 }
 200                 LoginCluster       string
 201                 RemoteTokenRefresh Duration
 202                 TokenLifetime      Duration
 203                 TrustedClients     map[string]struct{}
 204                 IssueTrustedTokens bool
 205         }
 206         Mail struct {
 207                 MailchimpAPIKey                string
 208                 MailchimpListID                string
 209                 SendUserSetupNotificationEmail bool
 210                 IssueReporterEmailFrom         string
 211                 IssueReporterEmailTo           string
 212                 SupportEmailAddress            string
 213                 EmailFrom                      string
 214         }
 215         SystemLogs struct {
 216                 LogLevel                string
 217                 Format                  string
 218                 MaxRequestLogParamsSize int
 219         }
 220         TLS struct {
 221                 Certificate string
 222                 Key         string
 223                 Insecure    bool
 224         }
 225         Users struct {
 226                 AnonymousUserToken                    string
 227                 AdminNotifierEmailFrom                string
 228                 AutoAdminFirstUser                    bool
 229                 AutoAdminUserWithEmail                string
 230                 AutoSetupNewUsers                     bool
 231                 AutoSetupNewUsersWithRepository       bool
 232                 AutoSetupNewUsersWithVmUUID           string
 233                 AutoSetupUsernameBlacklist            StringSet
 234                 EmailSubjectPrefix                    string
 235                 NewInactiveUserNotificationRecipients StringSet
 236                 NewUserNotificationRecipients         StringSet
 237                 NewUsersAreActive                     bool
 238                 UserNotifierEmailFrom                 string
 239                 UserNotifierEmailBcc                  StringSet
 240                 UserProfileNotificationAddress        string
 241                 PreferDomainForUsername               string
 242                 UserSetupMailText                     string
 243         }
 244         StorageClasses map[string]StorageClassConfig
 245         Volumes        map[string]Volume
 246         Workbench      struct {
 247                 ActivationContactLink            string
 248                 APIClientConnectTimeout          Duration
 249                 APIClientReceiveTimeout          Duration
 250                 APIResponseCompression           bool
 251                 ApplicationMimetypesWithViewIcon StringSet
 252                 ArvadosDocsite                   string
 253                 ArvadosPublicDataDocURL          string
 254                 DefaultOpenIdPrefix              string
 255                 EnableGettingStartedPopup        bool
 256                 EnablePublicProjectsPage         bool
 257                 FileViewersConfigURL             string
 258                 LogViewerMaxBytes                ByteSize
 259                 MultiSiteSearch                  string
 260                 ProfilingEnabled                 bool
 261                 Repositories                     bool
 262                 RepositoryCache                  string
 263                 RunningJobLogRecordsToFetch      int
 264                 SecretKeyBase                    string
 265                 ShowRecentCollectionsOnDashboard bool
 266                 ShowUserAgreementInline          bool
 267                 ShowUserNotifications            bool
 268                 SiteName                         string
 269                 Theme                            string
 270                 UserProfileFormFields            map[string]struct {
 271                         Type                 string
 272                         FormFieldTitle       string
 273                         FormFieldDescription string
 274                         Required             bool
 275                         Position             int
 276                         Options              map[string]struct{}
 277                 }
 278                 UserProfileFormMessage string
 279                 WelcomePageHTML        string
 280                 InactivePageHTML       string
 281                 SSHHelpPageHTML        string
 282                 SSHHelpHostSuffix      string
 283                 IdleTimeout            Duration
 284         }
 285 }
 286
 287 type StorageClassConfig struct {
 288         Default  bool
 289         Priority int
 290 }
 291
 292 type Volume struct {
 293         AccessViaHosts   map[URL]VolumeAccess
 294         ReadOnly         bool
 295         Replication      int
 296         StorageClasses   map[string]bool
 297         Driver           string
 298         DriverParameters json.RawMessage
 299 }
 300
 301 type S3VolumeDriverParameters struct {
 302         IAMRole            string
 303         AccessKeyID        string
 304         SecretAccessKey    string
 305         Endpoint           string
 306         Region             string
 307         Bucket             string
 308         LocationConstraint bool
 309         V2Signature        bool
 310         UseAWSS3v2Driver   bool
 311         IndexPageSize      int
 312         ConnectTimeout     Duration
 313         ReadTimeout        Duration
 314         RaceWindow         Duration
 315         UnsafeDelete       bool
 316         PrefixLength       int
 317 }
 318
 319 type AzureVolumeDriverParameters struct {
 320         StorageAccountName   string
 321         StorageAccountKey    string
 322         StorageBaseURL       string
 323         ContainerName        string
 324         RequestTimeout       Duration
 325         ListBlobsRetryDelay  Duration
 326         ListBlobsMaxAttempts int
 327 }
 328
 329 type DirectoryVolumeDriverParameters struct {
 330         Root      string
 331         Serialize bool
 332 }
 333
 334 type VolumeAccess struct {
 335         ReadOnly bool
 336 }
 337
 338 type Services struct {
 339         Composer       Service
 340         Controller     Service
 341         DispatchCloud  Service
 342         DispatchLSF    Service
 343         GitHTTP        Service
 344         GitSSH         Service
 345         Health         Service
 346         Keepbalance    Service
 347         Keepproxy      Service
 348         Keepstore      Service
 349         RailsAPI       Service
 350         WebDAVDownload Service
 351         WebDAV         Service
 352         WebShell       Service
 353         Websocket      Service
 354         Workbench1     Service
 355         Workbench2     Service
 356 }
 357
 358 type Service struct {
 359         InternalURLs map[URL]ServiceInstance
 360         ExternalURL  URL
 361 }
 362
 363 type TestUser struct {
 364         Email    string
 365         Password string
 366 }
 367
 368 // URL is a url.URL that is also usable as a JSON key/value.
 369 type URL url.URL
 370
 371 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 372 // used as a JSON key/value.
 373 func (su *URL) UnmarshalText(text []byte) error {
 374         u, err := url.Parse(string(text))
 375         if err == nil {
 376                 *su = URL(*u)
 377                 if su.Path == "" && su.Host != "" {
 378                         // http://example really means http://example/
 379                         su.Path = "/"
 380                 }
 381         }
 382         return err
 383 }
 384
 385 func (su URL) MarshalText() ([]byte, error) {
 386         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 387 }
 388
 389 func (su URL) String() string {
 390         return (*url.URL)(&su).String()
 391 }
 392
 393 type ServiceInstance struct {
 394         Rendezvous string `json:",omitempty"`
 395 }
 396
 397 type PostgreSQL struct {
 398         Connection     PostgreSQLConnection
 399         ConnectionPool int
 400 }
 401
 402 type PostgreSQLConnection map[string]string
 403
 404 type RemoteCluster struct {
 405         Host          string
 406         Proxy         bool
 407         Scheme        string
 408         Insecure      bool
 409         ActivateUsers bool
 410 }
 411
 412 type InstanceType struct {
 413         Name            string
 414         ProviderType    string
 415         VCPUs           int
 416         RAM             ByteSize
 417         Scratch         ByteSize
 418         IncludedScratch ByteSize
 419         AddedScratch    ByteSize
 420         Price           float64
 421         Preemptible     bool
 422 }
 423
 424 type ContainersConfig struct {
 425         CloudVMs                    CloudVMsConfig
 426         CrunchRunCommand            string
 427         CrunchRunArgumentsList      []string
 428         DefaultKeepCacheRAM         ByteSize
 429         DispatchPrivateKey          string
 430         LogReuseDecisions           bool
 431         MaxComputeVMs               int
 432         MaxDispatchAttempts         int
 433         MaxRetryAttempts            int
 434         MinRetryPeriod              Duration
 435         ReserveExtraRAM             ByteSize
 436         StaleLockTimeout            Duration
 437         SupportedDockerImageFormats StringSet
 438         UsePreemptibleInstances     bool
 439         RuntimeEngine               string
 440         LocalKeepBlobBuffersPerVCPU int
 441         LocalKeepLogsToContainerLog string
 442
 443         JobsAPI struct {
 444                 Enable         string
 445                 GitInternalDir string
 446         }
 447         Logging struct {
 448                 MaxAge                       Duration
 449                 LogBytesPerEvent             int
 450                 LogSecondsBetweenEvents      Duration
 451                 LogThrottlePeriod            Duration
 452                 LogThrottleBytes             int
 453                 LogThrottleLines             int
 454                 LimitLogBytesPerJob          int
 455                 LogPartialLineThrottlePeriod Duration
 456                 LogUpdatePeriod              Duration
 457                 LogUpdateSize                ByteSize
 458         }
 459         ShellAccess struct {
 460                 Admin bool
 461                 User  bool
 462         }
 463         SLURM struct {
 464                 PrioritySpread             int64
 465                 SbatchArgumentsList        []string
 466                 SbatchEnvironmentVariables map[string]string
 467                 Managed                    struct {
 468                         DNSServerConfDir       string
 469                         DNSServerConfTemplate  string
 470                         DNSServerReloadCommand string
 471                         DNSServerUpdateCommand string
 472                         ComputeNodeDomain      string
 473                         ComputeNodeNameservers StringSet
 474                         AssignNodeHostname     string
 475                 }
 476         }
 477         LSF struct {
 478                 BsubSudoUser      string
 479                 BsubArgumentsList []string
 480         }
 481 }
 482
 483 type CloudVMsConfig struct {
 484         Enable bool
 485
 486         BootProbeCommand               string
 487         DeployRunnerBinary             string
 488         ImageID                        string
 489         MaxCloudOpsPerSecond           int
 490         MaxProbesPerSecond             int
 491         MaxConcurrentInstanceCreateOps int
 492         PollInterval                   Duration
 493         ProbeInterval                  Duration
 494         SSHPort                        string
 495         SyncInterval                   Duration
 496         TimeoutBooting                 Duration
 497         TimeoutIdle                    Duration
 498         TimeoutProbe                   Duration
 499         TimeoutShutdown                Duration
 500         TimeoutSignal                  Duration
 501         TimeoutStaleRunLock            Duration
 502         TimeoutTERM                    Duration
 503         ResourceTags                   map[string]string
 504         TagKeyPrefix                   string
 505
 506         Driver           string
 507         DriverParameters json.RawMessage
 508 }
 509
 510 type InstanceTypeMap map[string]InstanceType
 511
 512 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 513
 514 // UnmarshalJSON handles old config files that provide an array of
 515 // instance types instead of a hash.
 516 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 517         fixup := func(t InstanceType) (InstanceType, error) {
 518                 if t.ProviderType == "" {
 519                         t.ProviderType = t.Name
 520                 }
 521                 if t.Scratch == 0 {
 522                         t.Scratch = t.IncludedScratch + t.AddedScratch
 523                 } else if t.AddedScratch == 0 {
 524                         t.AddedScratch = t.Scratch - t.IncludedScratch
 525                 } else if t.IncludedScratch == 0 {
 526                         t.IncludedScratch = t.Scratch - t.AddedScratch
 527                 }
 528
 529                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 530                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 531                 }
 532                 return t, nil
 533         }
 534
 535         if len(data) > 0 && data[0] == '[' {
 536                 var arr []InstanceType
 537                 err := json.Unmarshal(data, &arr)
 538                 if err != nil {
 539                         return err
 540                 }
 541                 if len(arr) == 0 {
 542                         *it = nil
 543                         return nil
 544                 }
 545                 *it = make(map[string]InstanceType, len(arr))
 546                 for _, t := range arr {
 547                         if _, ok := (*it)[t.Name]; ok {
 548                                 return errDuplicateInstanceTypeName
 549                         }
 550                         t, err := fixup(t)
 551                         if err != nil {
 552                                 return err
 553                         }
 554                         (*it)[t.Name] = t
 555                 }
 556                 return nil
 557         }
 558         var hash map[string]InstanceType
 559         err := json.Unmarshal(data, &hash)
 560         if err != nil {
 561                 return err
 562         }
 563         // Fill in Name field (and ProviderType field, if not
 564         // specified) using hash key.
 565         *it = InstanceTypeMap(hash)
 566         for name, t := range *it {
 567                 t.Name = name
 568                 t, err := fixup(t)
 569                 if err != nil {
 570                         return err
 571                 }
 572                 (*it)[name] = t
 573         }
 574         return nil
 575 }
 576
 577 type StringSet map[string]struct{}
 578
 579 // UnmarshalJSON handles old config files that provide an array of
 580 // instance types instead of a hash.
 581 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 582         if len(data) > 0 && data[0] == '[' {
 583                 var arr []string
 584                 err := json.Unmarshal(data, &arr)
 585                 if err != nil {
 586                         return err
 587                 }
 588                 if len(arr) == 0 {
 589                         *ss = nil
 590                         return nil
 591                 }
 592                 *ss = make(map[string]struct{}, len(arr))
 593                 for _, t := range arr {
 594                         (*ss)[t] = struct{}{}
 595                 }
 596                 return nil
 597         }
 598         var hash map[string]struct{}
 599         err := json.Unmarshal(data, &hash)
 600         if err != nil {
 601                 return err
 602         }
 603         *ss = make(map[string]struct{}, len(hash))
 604         for t := range hash {
 605                 (*ss)[t] = struct{}{}
 606         }
 607
 608         return nil
 609 }
 610
 611 type ServiceName string
 612
 613 const (
 614         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 615         ServiceNameController    ServiceName = "arvados-controller"
 616         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 617         ServiceNameDispatchLSF   ServiceName = "arvados-dispatch-lsf"
 618         ServiceNameHealth        ServiceName = "arvados-health"
 619         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 620         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 621         ServiceNameWebsocket     ServiceName = "arvados-ws"
 622         ServiceNameKeepbalance   ServiceName = "keep-balance"
 623         ServiceNameKeepweb       ServiceName = "keep-web"
 624         ServiceNameKeepproxy     ServiceName = "keepproxy"
 625         ServiceNameKeepstore     ServiceName = "keepstore"
 626 )
 627
 628 // Map returns all services as a map, suitable for iterating over all
 629 // services or looking up a service by name.
 630 func (svcs Services) Map() map[ServiceName]Service {
 631         return map[ServiceName]Service{
 632                 ServiceNameRailsAPI:      svcs.RailsAPI,
 633                 ServiceNameController:    svcs.Controller,
 634                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 635                 ServiceNameDispatchLSF:   svcs.DispatchLSF,
 636                 ServiceNameHealth:        svcs.Health,
 637                 ServiceNameWorkbench1:    svcs.Workbench1,
 638                 ServiceNameWorkbench2:    svcs.Workbench2,
 639                 ServiceNameWebsocket:     svcs.Websocket,
 640                 ServiceNameKeepbalance:   svcs.Keepbalance,
 641                 ServiceNameKeepweb:       svcs.WebDAV,
 642                 ServiceNameKeepproxy:     svcs.Keepproxy,
 643                 ServiceNameKeepstore:     svcs.Keepstore,
 644         }
 645 }