sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.arvados.org/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         }
  21         return "/etc/arvados/config.yml"
  22 }()
  23
  24 type Config struct {
  25         Clusters         map[string]Cluster
  26         AutoReloadConfig bool
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         cc, ok := sc.Clusters[clusterID]
  53         if !ok {
  54                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  55         }
  56         cc.ClusterID = clusterID
  57         return &cc, nil
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxBlockEntries      int
  64         MaxCollectionEntries int
  65         MaxCollectionBytes   int64
  66         MaxUUIDEntries       int
  67         MaxSessions          int
  68 }
  69
  70 type UploadDownloadPermission struct {
  71         Upload   bool
  72         Download bool
  73 }
  74
  75 type UploadDownloadRolePermissions struct {
  76         User  UploadDownloadPermission
  77         Admin UploadDownloadPermission
  78 }
  79
  80 type ManagedProperties map[string]struct {
  81         Value     interface{}
  82         Function  string
  83         Protected bool
  84 }
  85
  86 type Cluster struct {
  87         ClusterID       string `json:"-"`
  88         ManagementToken string
  89         SystemRootToken string
  90         Services        Services
  91         InstanceTypes   InstanceTypeMap
  92         Containers      ContainersConfig
  93         RemoteClusters  map[string]RemoteCluster
  94         PostgreSQL      PostgreSQL
  95
  96         API struct {
  97                 AsyncPermissionsUpdateInterval   Duration
  98                 DisabledAPIs                     StringSet
  99                 MaxIndexDatabaseRead             int
 100                 MaxItemsPerResponse              int
 101                 MaxConcurrentRequests            int
 102                 MaxKeepBlobBuffers               int
 103                 MaxRequestAmplification          int
 104                 MaxRequestSize                   int
 105                 MaxTokenLifetime                 Duration
 106                 RequestTimeout                   Duration
 107                 SendTimeout                      Duration
 108                 WebsocketClientEventQueue        int
 109                 WebsocketServerEventQueue        int
 110                 KeepServiceRequestTimeout        Duration
 111                 VocabularyPath                   string
 112                 FreezeProjectRequiresDescription bool
 113                 FreezeProjectRequiresProperties  StringSet
 114                 UnfreezeProjectRequiresAdmin     bool
 115         }
 116         AuditLogs struct {
 117                 MaxAge             Duration
 118                 MaxDeleteBatch     int
 119                 UnloggedAttributes StringSet
 120         }
 121         Collections struct {
 122                 BlobSigning                  bool
 123                 BlobSigningKey               string
 124                 BlobSigningTTL               Duration
 125                 BlobTrash                    bool
 126                 BlobTrashLifetime            Duration
 127                 BlobTrashCheckInterval       Duration
 128                 BlobTrashConcurrency         int
 129                 BlobDeleteConcurrency        int
 130                 BlobReplicateConcurrency     int
 131                 CollectionVersioning         bool
 132                 DefaultTrashLifetime         Duration
 133                 DefaultReplication           int
 134                 ManagedProperties            ManagedProperties
 135                 PreserveVersionIfIdle        Duration
 136                 TrashSweepInterval           Duration
 137                 TrustAllContent              bool
 138                 ForwardSlashNameSubstitution string
 139                 S3FolderObjects              bool
 140
 141                 BlobMissingReport        string
 142                 BalancePeriod            Duration
 143                 BalanceCollectionBatch   int
 144                 BalanceCollectionBuffers int
 145                 BalanceTimeout           Duration
 146                 BalanceUpdateLimit       int
 147
 148                 WebDAVCache WebDAVCacheConfig
 149
 150                 KeepproxyPermission UploadDownloadRolePermissions
 151                 WebDAVPermission    UploadDownloadRolePermissions
 152                 WebDAVLogEvents     bool
 153         }
 154         Git struct {
 155                 GitCommand   string
 156                 GitoliteHome string
 157                 Repositories string
 158         }
 159         Login struct {
 160                 LDAP struct {
 161                         Enable             bool
 162                         URL                URL
 163                         StartTLS           bool
 164                         InsecureTLS        bool
 165                         StripDomain        string
 166                         AppendDomain       string
 167                         SearchAttribute    string
 168                         SearchBindUser     string
 169                         SearchBindPassword string
 170                         SearchBase         string
 171                         SearchFilters      string
 172                         EmailAttribute     string
 173                         UsernameAttribute  string
 174                 }
 175                 Google struct {
 176                         Enable                          bool
 177                         ClientID                        string
 178                         ClientSecret                    string
 179                         AlternateEmailAddresses         bool
 180                         AuthenticationRequestParameters map[string]string
 181                 }
 182                 OpenIDConnect struct {
 183                         Enable                          bool
 184                         Issuer                          string
 185                         ClientID                        string
 186                         ClientSecret                    string
 187                         EmailClaim                      string
 188                         EmailVerifiedClaim              string
 189                         UsernameClaim                   string
 190                         AcceptAccessToken               bool
 191                         AcceptAccessTokenScope          string
 192                         AuthenticationRequestParameters map[string]string
 193                 }
 194                 PAM struct {
 195                         Enable             bool
 196                         Service            string
 197                         DefaultEmailDomain string
 198                 }
 199                 Test struct {
 200                         Enable bool
 201                         Users  map[string]TestUser
 202                 }
 203                 LoginCluster       string
 204                 RemoteTokenRefresh Duration
 205                 TokenLifetime      Duration
 206                 TrustedClients     map[string]struct{}
 207                 IssueTrustedTokens bool
 208         }
 209         Mail struct {
 210                 MailchimpAPIKey                string
 211                 MailchimpListID                string
 212                 SendUserSetupNotificationEmail bool
 213                 IssueReporterEmailFrom         string
 214                 IssueReporterEmailTo           string
 215                 SupportEmailAddress            string
 216                 EmailFrom                      string
 217         }
 218         SystemLogs struct {
 219                 LogLevel                string
 220                 Format                  string
 221                 MaxRequestLogParamsSize int
 222         }
 223         TLS struct {
 224                 Certificate string
 225                 Key         string
 226                 Insecure    bool
 227         }
 228         Users struct {
 229                 ActivatedUsersAreVisibleToOthers      bool
 230                 AnonymousUserToken                    string
 231                 AdminNotifierEmailFrom                string
 232                 AutoAdminFirstUser                    bool
 233                 AutoAdminUserWithEmail                string
 234                 AutoSetupNewUsers                     bool
 235                 AutoSetupNewUsersWithRepository       bool
 236                 AutoSetupNewUsersWithVmUUID           string
 237                 AutoSetupUsernameBlacklist            StringSet
 238                 EmailSubjectPrefix                    string
 239                 NewInactiveUserNotificationRecipients StringSet
 240                 NewUserNotificationRecipients         StringSet
 241                 NewUsersAreActive                     bool
 242                 UserNotifierEmailFrom                 string
 243                 UserNotifierEmailBcc                  StringSet
 244                 UserProfileNotificationAddress        string
 245                 PreferDomainForUsername               string
 246                 UserSetupMailText                     string
 247                 RoleGroupsVisibleToAll                bool
 248         }
 249         StorageClasses map[string]StorageClassConfig
 250         Volumes        map[string]Volume
 251         Workbench      struct {
 252                 ActivationContactLink            string
 253                 APIClientConnectTimeout          Duration
 254                 APIClientReceiveTimeout          Duration
 255                 APIResponseCompression           bool
 256                 ApplicationMimetypesWithViewIcon StringSet
 257                 ArvadosDocsite                   string
 258                 ArvadosPublicDataDocURL          string
 259                 DefaultOpenIdPrefix              string
 260                 EnableGettingStartedPopup        bool
 261                 EnablePublicProjectsPage         bool
 262                 FileViewersConfigURL             string
 263                 LogViewerMaxBytes                ByteSize
 264                 MultiSiteSearch                  string
 265                 ProfilingEnabled                 bool
 266                 Repositories                     bool
 267                 RepositoryCache                  string
 268                 RunningJobLogRecordsToFetch      int
 269                 SecretKeyBase                    string
 270                 ShowRecentCollectionsOnDashboard bool
 271                 ShowUserAgreementInline          bool
 272                 ShowUserNotifications            bool
 273                 SiteName                         string
 274                 Theme                            string
 275                 UserProfileFormFields            map[string]struct {
 276                         Type                 string
 277                         FormFieldTitle       string
 278                         FormFieldDescription string
 279                         Required             bool
 280                         Position             int
 281                         Options              map[string]struct{}
 282                 }
 283                 UserProfileFormMessage string
 284                 WelcomePageHTML        string
 285                 InactivePageHTML       string
 286                 SSHHelpPageHTML        string
 287                 SSHHelpHostSuffix      string
 288                 IdleTimeout            Duration
 289         }
 290 }
 291
 292 type StorageClassConfig struct {
 293         Default  bool
 294         Priority int
 295 }
 296
 297 type Volume struct {
 298         AccessViaHosts   map[URL]VolumeAccess
 299         ReadOnly         bool
 300         Replication      int
 301         StorageClasses   map[string]bool
 302         Driver           string
 303         DriverParameters json.RawMessage
 304 }
 305
 306 type S3VolumeDriverParameters struct {
 307         IAMRole            string
 308         AccessKeyID        string
 309         SecretAccessKey    string
 310         Endpoint           string
 311         Region             string
 312         Bucket             string
 313         LocationConstraint bool
 314         V2Signature        bool
 315         UseAWSS3v2Driver   bool
 316         IndexPageSize      int
 317         ConnectTimeout     Duration
 318         ReadTimeout        Duration
 319         RaceWindow         Duration
 320         UnsafeDelete       bool
 321         PrefixLength       int
 322 }
 323
 324 type AzureVolumeDriverParameters struct {
 325         StorageAccountName   string
 326         StorageAccountKey    string
 327         StorageBaseURL       string
 328         ContainerName        string
 329         RequestTimeout       Duration
 330         ListBlobsRetryDelay  Duration
 331         ListBlobsMaxAttempts int
 332 }
 333
 334 type DirectoryVolumeDriverParameters struct {
 335         Root      string
 336         Serialize bool
 337 }
 338
 339 type VolumeAccess struct {
 340         ReadOnly bool
 341 }
 342
 343 type Services struct {
 344         Composer       Service
 345         Controller     Service
 346         DispatchCloud  Service
 347         DispatchLSF    Service
 348         GitHTTP        Service
 349         GitSSH         Service
 350         Health         Service
 351         Keepbalance    Service
 352         Keepproxy      Service
 353         Keepstore      Service
 354         RailsAPI       Service
 355         WebDAVDownload Service
 356         WebDAV         Service
 357         WebShell       Service
 358         Websocket      Service
 359         Workbench1     Service
 360         Workbench2     Service
 361 }
 362
 363 type Service struct {
 364         InternalURLs map[URL]ServiceInstance
 365         ExternalURL  URL
 366 }
 367
 368 type TestUser struct {
 369         Email    string
 370         Password string
 371 }
 372
 373 // URL is a url.URL that is also usable as a JSON key/value.
 374 type URL url.URL
 375
 376 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 377 // used as a JSON key/value.
 378 func (su *URL) UnmarshalText(text []byte) error {
 379         u, err := url.Parse(string(text))
 380         if err == nil {
 381                 *su = URL(*u)
 382                 if su.Path == "" && su.Host != "" {
 383                         // http://example really means http://example/
 384                         su.Path = "/"
 385                 }
 386         }
 387         return err
 388 }
 389
 390 func (su URL) MarshalText() ([]byte, error) {
 391         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 392 }
 393
 394 func (su URL) String() string {
 395         return (*url.URL)(&su).String()
 396 }
 397
 398 type ServiceInstance struct {
 399         Rendezvous string `json:",omitempty"`
 400 }
 401
 402 type PostgreSQL struct {
 403         Connection     PostgreSQLConnection
 404         ConnectionPool int
 405 }
 406
 407 type PostgreSQLConnection map[string]string
 408
 409 type RemoteCluster struct {
 410         Host          string
 411         Proxy         bool
 412         Scheme        string
 413         Insecure      bool
 414         ActivateUsers bool
 415 }
 416
 417 type CUDAFeatures struct {
 418         DriverVersion      string
 419         HardwareCapability string
 420         DeviceCount        int
 421 }
 422
 423 type InstanceType struct {
 424         Name            string
 425         ProviderType    string
 426         VCPUs           int
 427         RAM             ByteSize
 428         Scratch         ByteSize
 429         IncludedScratch ByteSize
 430         AddedScratch    ByteSize
 431         Price           float64
 432         Preemptible     bool
 433         CUDA            CUDAFeatures
 434 }
 435
 436 type ContainersConfig struct {
 437         CloudVMs                      CloudVMsConfig
 438         CrunchRunCommand              string
 439         CrunchRunArgumentsList        []string
 440         DefaultKeepCacheRAM           ByteSize
 441         DispatchPrivateKey            string
 442         LogReuseDecisions             bool
 443         MaxComputeVMs                 int
 444         MaxDispatchAttempts           int
 445         MaxRetryAttempts              int
 446         MinRetryPeriod                Duration
 447         ReserveExtraRAM               ByteSize
 448         StaleLockTimeout              Duration
 449         SupportedDockerImageFormats   StringSet
 450         AlwaysUsePreemptibleInstances bool
 451         PreemptiblePriceFactor        float64
 452         RuntimeEngine                 string
 453         LocalKeepBlobBuffersPerVCPU   int
 454         LocalKeepLogsToContainerLog   string
 455
 456         JobsAPI struct {
 457                 Enable         string
 458                 GitInternalDir string
 459         }
 460         Logging struct {
 461                 MaxAge                       Duration
 462                 LogBytesPerEvent             int
 463                 LogSecondsBetweenEvents      Duration
 464                 LogThrottlePeriod            Duration
 465                 LogThrottleBytes             int
 466                 LogThrottleLines             int
 467                 LimitLogBytesPerJob          int
 468                 LogPartialLineThrottlePeriod Duration
 469                 LogUpdatePeriod              Duration
 470                 LogUpdateSize                ByteSize
 471         }
 472         ShellAccess struct {
 473                 Admin bool
 474                 User  bool
 475         }
 476         SLURM struct {
 477                 PrioritySpread             int64
 478                 SbatchArgumentsList        []string
 479                 SbatchEnvironmentVariables map[string]string
 480                 Managed                    struct {
 481                         DNSServerConfDir       string
 482                         DNSServerConfTemplate  string
 483                         DNSServerReloadCommand string
 484                         DNSServerUpdateCommand string
 485                         ComputeNodeDomain      string
 486                         ComputeNodeNameservers StringSet
 487                         AssignNodeHostname     string
 488                 }
 489         }
 490         LSF struct {
 491                 BsubSudoUser      string
 492                 BsubArgumentsList []string
 493                 BsubCUDAArguments []string
 494         }
 495 }
 496
 497 type CloudVMsConfig struct {
 498         Enable bool
 499
 500         BootProbeCommand               string
 501         DeployRunnerBinary             string
 502         ImageID                        string
 503         MaxCloudOpsPerSecond           int
 504         MaxProbesPerSecond             int
 505         MaxConcurrentInstanceCreateOps int
 506         PollInterval                   Duration
 507         ProbeInterval                  Duration
 508         SSHPort                        string
 509         SyncInterval                   Duration
 510         TimeoutBooting                 Duration
 511         TimeoutIdle                    Duration
 512         TimeoutProbe                   Duration
 513         TimeoutShutdown                Duration
 514         TimeoutSignal                  Duration
 515         TimeoutStaleRunLock            Duration
 516         TimeoutTERM                    Duration
 517         ResourceTags                   map[string]string
 518         TagKeyPrefix                   string
 519
 520         Driver           string
 521         DriverParameters json.RawMessage
 522 }
 523
 524 type InstanceTypeMap map[string]InstanceType
 525
 526 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 527
 528 // UnmarshalJSON handles old config files that provide an array of
 529 // instance types instead of a hash.
 530 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 531         fixup := func(t InstanceType) (InstanceType, error) {
 532                 if t.ProviderType == "" {
 533                         t.ProviderType = t.Name
 534                 }
 535                 if t.Scratch == 0 {
 536                         t.Scratch = t.IncludedScratch + t.AddedScratch
 537                 } else if t.AddedScratch == 0 {
 538                         t.AddedScratch = t.Scratch - t.IncludedScratch
 539                 } else if t.IncludedScratch == 0 {
 540                         t.IncludedScratch = t.Scratch - t.AddedScratch
 541                 }
 542
 543                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 544                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 545                 }
 546                 return t, nil
 547         }
 548
 549         if len(data) > 0 && data[0] == '[' {
 550                 var arr []InstanceType
 551                 err := json.Unmarshal(data, &arr)
 552                 if err != nil {
 553                         return err
 554                 }
 555                 if len(arr) == 0 {
 556                         *it = nil
 557                         return nil
 558                 }
 559                 *it = make(map[string]InstanceType, len(arr))
 560                 for _, t := range arr {
 561                         if _, ok := (*it)[t.Name]; ok {
 562                                 return errDuplicateInstanceTypeName
 563                         }
 564                         t, err := fixup(t)
 565                         if err != nil {
 566                                 return err
 567                         }
 568                         (*it)[t.Name] = t
 569                 }
 570                 return nil
 571         }
 572         var hash map[string]InstanceType
 573         err := json.Unmarshal(data, &hash)
 574         if err != nil {
 575                 return err
 576         }
 577         // Fill in Name field (and ProviderType field, if not
 578         // specified) using hash key.
 579         *it = InstanceTypeMap(hash)
 580         for name, t := range *it {
 581                 t.Name = name
 582                 t, err := fixup(t)
 583                 if err != nil {
 584                         return err
 585                 }
 586                 (*it)[name] = t
 587         }
 588         return nil
 589 }
 590
 591 type StringSet map[string]struct{}
 592
 593 // UnmarshalJSON handles old config files that provide an array of
 594 // instance types instead of a hash.
 595 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 596         if len(data) > 0 && data[0] == '[' {
 597                 var arr []string
 598                 err := json.Unmarshal(data, &arr)
 599                 if err != nil {
 600                         return err
 601                 }
 602                 if len(arr) == 0 {
 603                         *ss = nil
 604                         return nil
 605                 }
 606                 *ss = make(map[string]struct{}, len(arr))
 607                 for _, t := range arr {
 608                         (*ss)[t] = struct{}{}
 609                 }
 610                 return nil
 611         }
 612         var hash map[string]struct{}
 613         err := json.Unmarshal(data, &hash)
 614         if err != nil {
 615                 return err
 616         }
 617         *ss = make(map[string]struct{}, len(hash))
 618         for t := range hash {
 619                 (*ss)[t] = struct{}{}
 620         }
 621
 622         return nil
 623 }
 624
 625 type ServiceName string
 626
 627 const (
 628         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 629         ServiceNameController    ServiceName = "arvados-controller"
 630         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 631         ServiceNameDispatchLSF   ServiceName = "arvados-dispatch-lsf"
 632         ServiceNameHealth        ServiceName = "arvados-health"
 633         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 634         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 635         ServiceNameWebsocket     ServiceName = "arvados-ws"
 636         ServiceNameKeepbalance   ServiceName = "keep-balance"
 637         ServiceNameKeepweb       ServiceName = "keep-web"
 638         ServiceNameKeepproxy     ServiceName = "keepproxy"
 639         ServiceNameKeepstore     ServiceName = "keepstore"
 640 )
 641
 642 // Map returns all services as a map, suitable for iterating over all
 643 // services or looking up a service by name.
 644 func (svcs Services) Map() map[ServiceName]Service {
 645         return map[ServiceName]Service{
 646                 ServiceNameRailsAPI:      svcs.RailsAPI,
 647                 ServiceNameController:    svcs.Controller,
 648                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 649                 ServiceNameDispatchLSF:   svcs.DispatchLSF,
 650                 ServiceNameHealth:        svcs.Health,
 651                 ServiceNameWorkbench1:    svcs.Workbench1,
 652                 ServiceNameWorkbench2:    svcs.Workbench2,
 653                 ServiceNameWebsocket:     svcs.Websocket,
 654                 ServiceNameKeepbalance:   svcs.Keepbalance,
 655                 ServiceNameKeepweb:       svcs.WebDAV,
 656                 ServiceNameKeepproxy:     svcs.Keepproxy,
 657                 ServiceNameKeepstore:     svcs.Keepstore,
 658         }
 659 }