sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.arvados.org/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         }
  21         return "/etc/arvados/config.yml"
  22 }()
  23
  24 type Config struct {
  25         Clusters         map[string]Cluster
  26         AutoReloadConfig bool
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         cc, ok := sc.Clusters[clusterID]
  53         if !ok {
  54                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  55         }
  56         cc.ClusterID = clusterID
  57         return &cc, nil
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxBlockEntries      int
  64         MaxCollectionEntries int
  65         MaxCollectionBytes   int64
  66         MaxPermissionEntries int
  67         MaxUUIDEntries       int
  68         MaxSessions          int
  69 }
  70
  71 type UploadDownloadPermission struct {
  72         Upload   bool
  73         Download bool
  74 }
  75
  76 type UploadDownloadRolePermissions struct {
  77         User  UploadDownloadPermission
  78         Admin UploadDownloadPermission
  79 }
  80
  81 type Cluster struct {
  82         ClusterID       string `json:"-"`
  83         ManagementToken string
  84         SystemRootToken string
  85         Services        Services
  86         InstanceTypes   InstanceTypeMap
  87         Containers      ContainersConfig
  88         RemoteClusters  map[string]RemoteCluster
  89         PostgreSQL      PostgreSQL
  90
  91         API struct {
  92                 AsyncPermissionsUpdateInterval Duration
  93                 DisabledAPIs                   StringSet
  94                 MaxIndexDatabaseRead           int
  95                 MaxItemsPerResponse            int
  96                 MaxConcurrentRequests          int
  97                 MaxKeepBlobBuffers             int
  98                 MaxRequestAmplification        int
  99                 MaxRequestSize                 int
 100                 MaxTokenLifetime               Duration
 101                 RequestTimeout                 Duration
 102                 SendTimeout                    Duration
 103                 WebsocketClientEventQueue      int
 104                 WebsocketServerEventQueue      int
 105                 KeepServiceRequestTimeout      Duration
 106         }
 107         AuditLogs struct {
 108                 MaxAge             Duration
 109                 MaxDeleteBatch     int
 110                 UnloggedAttributes StringSet
 111         }
 112         Collections struct {
 113                 BlobSigning              bool
 114                 BlobSigningKey           string
 115                 BlobSigningTTL           Duration
 116                 BlobTrash                bool
 117                 BlobTrashLifetime        Duration
 118                 BlobTrashCheckInterval   Duration
 119                 BlobTrashConcurrency     int
 120                 BlobDeleteConcurrency    int
 121                 BlobReplicateConcurrency int
 122                 CollectionVersioning     bool
 123                 DefaultTrashLifetime     Duration
 124                 DefaultReplication       int
 125                 ManagedProperties        map[string]struct {
 126                         Value     interface{}
 127                         Function  string
 128                         Protected bool
 129                 }
 130                 PreserveVersionIfIdle        Duration
 131                 TrashSweepInterval           Duration
 132                 TrustAllContent              bool
 133                 ForwardSlashNameSubstitution string
 134                 S3FolderObjects              bool
 135
 136                 BlobMissingReport        string
 137                 BalancePeriod            Duration
 138                 BalanceCollectionBatch   int
 139                 BalanceCollectionBuffers int
 140                 BalanceTimeout           Duration
 141
 142                 WebDAVCache WebDAVCacheConfig
 143
 144                 KeepproxyPermission UploadDownloadRolePermissions
 145                 WebDAVPermission    UploadDownloadRolePermissions
 146                 WebDAVLogEvents     bool
 147         }
 148         Git struct {
 149                 GitCommand   string
 150                 GitoliteHome string
 151                 Repositories string
 152         }
 153         Login struct {
 154                 LDAP struct {
 155                         Enable             bool
 156                         URL                URL
 157                         StartTLS           bool
 158                         InsecureTLS        bool
 159                         StripDomain        string
 160                         AppendDomain       string
 161                         SearchAttribute    string
 162                         SearchBindUser     string
 163                         SearchBindPassword string
 164                         SearchBase         string
 165                         SearchFilters      string
 166                         EmailAttribute     string
 167                         UsernameAttribute  string
 168                 }
 169                 Google struct {
 170                         Enable                          bool
 171                         ClientID                        string
 172                         ClientSecret                    string
 173                         AlternateEmailAddresses         bool
 174                         AuthenticationRequestParameters map[string]string
 175                 }
 176                 OpenIDConnect struct {
 177                         Enable                          bool
 178                         Issuer                          string
 179                         ClientID                        string
 180                         ClientSecret                    string
 181                         EmailClaim                      string
 182                         EmailVerifiedClaim              string
 183                         UsernameClaim                   string
 184                         AcceptAccessToken               bool
 185                         AcceptAccessTokenScope          string
 186                         AuthenticationRequestParameters map[string]string
 187                 }
 188                 PAM struct {
 189                         Enable             bool
 190                         Service            string
 191                         DefaultEmailDomain string
 192                 }
 193                 SSO struct {
 194                         Enable            bool
 195                         ProviderAppID     string
 196                         ProviderAppSecret string
 197                 }
 198                 Test struct {
 199                         Enable bool
 200                         Users  map[string]TestUser
 201                 }
 202                 LoginCluster       string
 203                 RemoteTokenRefresh Duration
 204                 TokenLifetime      Duration
 205                 TrustedClients     map[string]struct{}
 206                 IssueTrustedTokens bool
 207         }
 208         Mail struct {
 209                 MailchimpAPIKey                string
 210                 MailchimpListID                string
 211                 SendUserSetupNotificationEmail bool
 212                 IssueReporterEmailFrom         string
 213                 IssueReporterEmailTo           string
 214                 SupportEmailAddress            string
 215                 EmailFrom                      string
 216         }
 217         SystemLogs struct {
 218                 LogLevel                string
 219                 Format                  string
 220                 MaxRequestLogParamsSize int
 221         }
 222         TLS struct {
 223                 Certificate string
 224                 Key         string
 225                 Insecure    bool
 226         }
 227         Users struct {
 228                 AnonymousUserToken                    string
 229                 AdminNotifierEmailFrom                string
 230                 AutoAdminFirstUser                    bool
 231                 AutoAdminUserWithEmail                string
 232                 AutoSetupNewUsers                     bool
 233                 AutoSetupNewUsersWithRepository       bool
 234                 AutoSetupNewUsersWithVmUUID           string
 235                 AutoSetupUsernameBlacklist            StringSet
 236                 EmailSubjectPrefix                    string
 237                 NewInactiveUserNotificationRecipients StringSet
 238                 NewUserNotificationRecipients         StringSet
 239                 NewUsersAreActive                     bool
 240                 UserNotifierEmailFrom                 string
 241                 UserProfileNotificationAddress        string
 242                 PreferDomainForUsername               string
 243                 UserSetupMailText                     string
 244         }
 245         Volumes   map[string]Volume
 246         Workbench struct {
 247                 ActivationContactLink            string
 248                 APIClientConnectTimeout          Duration
 249                 APIClientReceiveTimeout          Duration
 250                 APIResponseCompression           bool
 251                 ApplicationMimetypesWithViewIcon StringSet
 252                 ArvadosDocsite                   string
 253                 ArvadosPublicDataDocURL          string
 254                 DefaultOpenIdPrefix              string
 255                 EnableGettingStartedPopup        bool
 256                 EnablePublicProjectsPage         bool
 257                 FileViewersConfigURL             string
 258                 LogViewerMaxBytes                ByteSize
 259                 MultiSiteSearch                  string
 260                 ProfilingEnabled                 bool
 261                 Repositories                     bool
 262                 RepositoryCache                  string
 263                 RunningJobLogRecordsToFetch      int
 264                 SecretKeyBase                    string
 265                 ShowRecentCollectionsOnDashboard bool
 266                 ShowUserAgreementInline          bool
 267                 ShowUserNotifications            bool
 268                 SiteName                         string
 269                 Theme                            string
 270                 UserProfileFormFields            map[string]struct {
 271                         Type                 string
 272                         FormFieldTitle       string
 273                         FormFieldDescription string
 274                         Required             bool
 275                         Position             int
 276                         Options              map[string]struct{}
 277                 }
 278                 UserProfileFormMessage string
 279                 VocabularyURL          string
 280                 WelcomePageHTML        string
 281                 InactivePageHTML       string
 282                 SSHHelpPageHTML        string
 283                 SSHHelpHostSuffix      string
 284                 IdleTimeout            Duration
 285         }
 286 }
 287
 288 type Volume struct {
 289         AccessViaHosts   map[URL]VolumeAccess
 290         ReadOnly         bool
 291         Replication      int
 292         StorageClasses   map[string]bool
 293         Driver           string
 294         DriverParameters json.RawMessage
 295 }
 296
 297 type S3VolumeDriverParameters struct {
 298         IAMRole            string
 299         AccessKeyID        string
 300         SecretAccessKey    string
 301         Endpoint           string
 302         Region             string
 303         Bucket             string
 304         LocationConstraint bool
 305         V2Signature        bool
 306         UseAWSS3v2Driver   bool
 307         IndexPageSize      int
 308         ConnectTimeout     Duration
 309         ReadTimeout        Duration
 310         RaceWindow         Duration
 311         UnsafeDelete       bool
 312 }
 313
 314 type AzureVolumeDriverParameters struct {
 315         StorageAccountName   string
 316         StorageAccountKey    string
 317         StorageBaseURL       string
 318         ContainerName        string
 319         RequestTimeout       Duration
 320         ListBlobsRetryDelay  Duration
 321         ListBlobsMaxAttempts int
 322 }
 323
 324 type DirectoryVolumeDriverParameters struct {
 325         Root      string
 326         Serialize bool
 327 }
 328
 329 type VolumeAccess struct {
 330         ReadOnly bool
 331 }
 332
 333 type Services struct {
 334         Composer       Service
 335         Controller     Service
 336         DispatchCloud  Service
 337         GitHTTP        Service
 338         GitSSH         Service
 339         Health         Service
 340         Keepbalance    Service
 341         Keepproxy      Service
 342         Keepstore      Service
 343         RailsAPI       Service
 344         SSO            Service
 345         WebDAVDownload Service
 346         WebDAV         Service
 347         WebShell       Service
 348         Websocket      Service
 349         Workbench1     Service
 350         Workbench2     Service
 351 }
 352
 353 type Service struct {
 354         InternalURLs map[URL]ServiceInstance
 355         ExternalURL  URL
 356 }
 357
 358 type TestUser struct {
 359         Email    string
 360         Password string
 361 }
 362
 363 // URL is a url.URL that is also usable as a JSON key/value.
 364 type URL url.URL
 365
 366 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 367 // used as a JSON key/value.
 368 func (su *URL) UnmarshalText(text []byte) error {
 369         u, err := url.Parse(string(text))
 370         if err == nil {
 371                 *su = URL(*u)
 372                 if su.Path == "" && su.Host != "" {
 373                         // http://example really means http://example/
 374                         su.Path = "/"
 375                 }
 376         }
 377         return err
 378 }
 379
 380 func (su URL) MarshalText() ([]byte, error) {
 381         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 382 }
 383
 384 func (su URL) String() string {
 385         return (*url.URL)(&su).String()
 386 }
 387
 388 type ServiceInstance struct {
 389         Rendezvous string `json:",omitempty"`
 390 }
 391
 392 type PostgreSQL struct {
 393         Connection     PostgreSQLConnection
 394         ConnectionPool int
 395 }
 396
 397 type PostgreSQLConnection map[string]string
 398
 399 type RemoteCluster struct {
 400         Host          string
 401         Proxy         bool
 402         Scheme        string
 403         Insecure      bool
 404         ActivateUsers bool
 405 }
 406
 407 type InstanceType struct {
 408         Name            string
 409         ProviderType    string
 410         VCPUs           int
 411         RAM             ByteSize
 412         Scratch         ByteSize
 413         IncludedScratch ByteSize
 414         AddedScratch    ByteSize
 415         Price           float64
 416         Preemptible     bool
 417 }
 418
 419 type ContainersConfig struct {
 420         CloudVMs                    CloudVMsConfig
 421         CrunchRunCommand            string
 422         CrunchRunArgumentsList      []string
 423         DefaultKeepCacheRAM         ByteSize
 424         DispatchPrivateKey          string
 425         LogReuseDecisions           bool
 426         MaxComputeVMs               int
 427         MaxDispatchAttempts         int
 428         MaxRetryAttempts            int
 429         MinRetryPeriod              Duration
 430         ReserveExtraRAM             ByteSize
 431         StaleLockTimeout            Duration
 432         SupportedDockerImageFormats StringSet
 433         UsePreemptibleInstances     bool
 434
 435         JobsAPI struct {
 436                 Enable         string
 437                 GitInternalDir string
 438         }
 439         Logging struct {
 440                 MaxAge                       Duration
 441                 LogBytesPerEvent             int
 442                 LogSecondsBetweenEvents      Duration
 443                 LogThrottlePeriod            Duration
 444                 LogThrottleBytes             int
 445                 LogThrottleLines             int
 446                 LimitLogBytesPerJob          int
 447                 LogPartialLineThrottlePeriod Duration
 448                 LogUpdatePeriod              Duration
 449                 LogUpdateSize                ByteSize
 450         }
 451         ShellAccess struct {
 452                 Admin bool
 453                 User  bool
 454         }
 455         SLURM struct {
 456                 PrioritySpread             int64
 457                 SbatchArgumentsList        []string
 458                 SbatchEnvironmentVariables map[string]string
 459                 Managed                    struct {
 460                         DNSServerConfDir       string
 461                         DNSServerConfTemplate  string
 462                         DNSServerReloadCommand string
 463                         DNSServerUpdateCommand string
 464                         ComputeNodeDomain      string
 465                         ComputeNodeNameservers StringSet
 466                         AssignNodeHostname     string
 467                 }
 468         }
 469 }
 470
 471 type CloudVMsConfig struct {
 472         Enable bool
 473
 474         BootProbeCommand               string
 475         DeployRunnerBinary             string
 476         ImageID                        string
 477         MaxCloudOpsPerSecond           int
 478         MaxProbesPerSecond             int
 479         MaxConcurrentInstanceCreateOps int
 480         PollInterval                   Duration
 481         ProbeInterval                  Duration
 482         SSHPort                        string
 483         SyncInterval                   Duration
 484         TimeoutBooting                 Duration
 485         TimeoutIdle                    Duration
 486         TimeoutProbe                   Duration
 487         TimeoutShutdown                Duration
 488         TimeoutSignal                  Duration
 489         TimeoutStaleRunLock            Duration
 490         TimeoutTERM                    Duration
 491         ResourceTags                   map[string]string
 492         TagKeyPrefix                   string
 493
 494         Driver           string
 495         DriverParameters json.RawMessage
 496 }
 497
 498 type InstanceTypeMap map[string]InstanceType
 499
 500 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 501
 502 // UnmarshalJSON handles old config files that provide an array of
 503 // instance types instead of a hash.
 504 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 505         fixup := func(t InstanceType) (InstanceType, error) {
 506                 if t.ProviderType == "" {
 507                         t.ProviderType = t.Name
 508                 }
 509                 if t.Scratch == 0 {
 510                         t.Scratch = t.IncludedScratch + t.AddedScratch
 511                 } else if t.AddedScratch == 0 {
 512                         t.AddedScratch = t.Scratch - t.IncludedScratch
 513                 } else if t.IncludedScratch == 0 {
 514                         t.IncludedScratch = t.Scratch - t.AddedScratch
 515                 }
 516
 517                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 518                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 519                 }
 520                 return t, nil
 521         }
 522
 523         if len(data) > 0 && data[0] == '[' {
 524                 var arr []InstanceType
 525                 err := json.Unmarshal(data, &arr)
 526                 if err != nil {
 527                         return err
 528                 }
 529                 if len(arr) == 0 {
 530                         *it = nil
 531                         return nil
 532                 }
 533                 *it = make(map[string]InstanceType, len(arr))
 534                 for _, t := range arr {
 535                         if _, ok := (*it)[t.Name]; ok {
 536                                 return errDuplicateInstanceTypeName
 537                         }
 538                         t, err := fixup(t)
 539                         if err != nil {
 540                                 return err
 541                         }
 542                         (*it)[t.Name] = t
 543                 }
 544                 return nil
 545         }
 546         var hash map[string]InstanceType
 547         err := json.Unmarshal(data, &hash)
 548         if err != nil {
 549                 return err
 550         }
 551         // Fill in Name field (and ProviderType field, if not
 552         // specified) using hash key.
 553         *it = InstanceTypeMap(hash)
 554         for name, t := range *it {
 555                 t.Name = name
 556                 t, err := fixup(t)
 557                 if err != nil {
 558                         return err
 559                 }
 560                 (*it)[name] = t
 561         }
 562         return nil
 563 }
 564
 565 type StringSet map[string]struct{}
 566
 567 // UnmarshalJSON handles old config files that provide an array of
 568 // instance types instead of a hash.
 569 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 570         if len(data) > 0 && data[0] == '[' {
 571                 var arr []string
 572                 err := json.Unmarshal(data, &arr)
 573                 if err != nil {
 574                         return err
 575                 }
 576                 if len(arr) == 0 {
 577                         *ss = nil
 578                         return nil
 579                 }
 580                 *ss = make(map[string]struct{}, len(arr))
 581                 for _, t := range arr {
 582                         (*ss)[t] = struct{}{}
 583                 }
 584                 return nil
 585         }
 586         var hash map[string]struct{}
 587         err := json.Unmarshal(data, &hash)
 588         if err != nil {
 589                 return err
 590         }
 591         *ss = make(map[string]struct{}, len(hash))
 592         for t := range hash {
 593                 (*ss)[t] = struct{}{}
 594         }
 595
 596         return nil
 597 }
 598
 599 type ServiceName string
 600
 601 const (
 602         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 603         ServiceNameController    ServiceName = "arvados-controller"
 604         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 605         ServiceNameHealth        ServiceName = "arvados-health"
 606         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 607         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 608         ServiceNameWebsocket     ServiceName = "arvados-ws"
 609         ServiceNameKeepbalance   ServiceName = "keep-balance"
 610         ServiceNameKeepweb       ServiceName = "keep-web"
 611         ServiceNameKeepproxy     ServiceName = "keepproxy"
 612         ServiceNameKeepstore     ServiceName = "keepstore"
 613 )
 614
 615 // Map returns all services as a map, suitable for iterating over all
 616 // services or looking up a service by name.
 617 func (svcs Services) Map() map[ServiceName]Service {
 618         return map[ServiceName]Service{
 619                 ServiceNameRailsAPI:      svcs.RailsAPI,
 620                 ServiceNameController:    svcs.Controller,
 621                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 622                 ServiceNameHealth:        svcs.Health,
 623                 ServiceNameWorkbench1:    svcs.Workbench1,
 624                 ServiceNameWorkbench2:    svcs.Workbench2,
 625                 ServiceNameWebsocket:     svcs.Websocket,
 626                 ServiceNameKeepbalance:   svcs.Keepbalance,
 627                 ServiceNameKeepweb:       svcs.WebDAV,
 628                 ServiceNameKeepproxy:     svcs.Keepproxy,
 629                 ServiceNameKeepstore:     svcs.Keepstore,
 630         }
 631 }