sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.arvados.org/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         }
  21         return "/etc/arvados/config.yml"
  22 }()
  23
  24 type Config struct {
  25         Clusters         map[string]Cluster
  26         AutoReloadConfig bool
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         cc, ok := sc.Clusters[clusterID]
  53         if !ok {
  54                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  55         }
  56         cc.ClusterID = clusterID
  57         return &cc, nil
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxBlockEntries      int
  64         MaxCollectionEntries int
  65         MaxCollectionBytes   int64
  66         MaxUUIDEntries       int
  67         MaxSessions          int
  68 }
  69
  70 type UploadDownloadPermission struct {
  71         Upload   bool
  72         Download bool
  73 }
  74
  75 type UploadDownloadRolePermissions struct {
  76         User  UploadDownloadPermission
  77         Admin UploadDownloadPermission
  78 }
  79
  80 type ManagedProperties map[string]struct {
  81         Value     interface{}
  82         Function  string
  83         Protected bool
  84 }
  85
  86 type Cluster struct {
  87         ClusterID       string `json:"-"`
  88         ManagementToken string
  89         SystemRootToken string
  90         Services        Services
  91         InstanceTypes   InstanceTypeMap
  92         Containers      ContainersConfig
  93         RemoteClusters  map[string]RemoteCluster
  94         PostgreSQL      PostgreSQL
  95
  96         API struct {
  97                 AsyncPermissionsUpdateInterval Duration
  98                 DisabledAPIs                   StringSet
  99                 MaxIndexDatabaseRead           int
 100                 MaxItemsPerResponse            int
 101                 MaxConcurrentRequests          int
 102                 MaxKeepBlobBuffers             int
 103                 MaxRequestAmplification        int
 104                 MaxRequestSize                 int
 105                 MaxTokenLifetime               Duration
 106                 RequestTimeout                 Duration
 107                 SendTimeout                    Duration
 108                 WebsocketClientEventQueue      int
 109                 WebsocketServerEventQueue      int
 110                 KeepServiceRequestTimeout      Duration
 111                 VocabularyPath                 string
 112         }
 113         AuditLogs struct {
 114                 MaxAge             Duration
 115                 MaxDeleteBatch     int
 116                 UnloggedAttributes StringSet
 117         }
 118         Collections struct {
 119                 BlobSigning                  bool
 120                 BlobSigningKey               string
 121                 BlobSigningTTL               Duration
 122                 BlobTrash                    bool
 123                 BlobTrashLifetime            Duration
 124                 BlobTrashCheckInterval       Duration
 125                 BlobTrashConcurrency         int
 126                 BlobDeleteConcurrency        int
 127                 BlobReplicateConcurrency     int
 128                 CollectionVersioning         bool
 129                 DefaultTrashLifetime         Duration
 130                 DefaultReplication           int
 131                 ManagedProperties            ManagedProperties
 132                 PreserveVersionIfIdle        Duration
 133                 TrashSweepInterval           Duration
 134                 TrustAllContent              bool
 135                 ForwardSlashNameSubstitution string
 136                 S3FolderObjects              bool
 137
 138                 BlobMissingReport        string
 139                 BalancePeriod            Duration
 140                 BalanceCollectionBatch   int
 141                 BalanceCollectionBuffers int
 142                 BalanceTimeout           Duration
 143                 BalanceUpdateLimit       int
 144
 145                 WebDAVCache WebDAVCacheConfig
 146
 147                 KeepproxyPermission UploadDownloadRolePermissions
 148                 WebDAVPermission    UploadDownloadRolePermissions
 149                 WebDAVLogEvents     bool
 150         }
 151         Git struct {
 152                 GitCommand   string
 153                 GitoliteHome string
 154                 Repositories string
 155         }
 156         Login struct {
 157                 LDAP struct {
 158                         Enable             bool
 159                         URL                URL
 160                         StartTLS           bool
 161                         InsecureTLS        bool
 162                         StripDomain        string
 163                         AppendDomain       string
 164                         SearchAttribute    string
 165                         SearchBindUser     string
 166                         SearchBindPassword string
 167                         SearchBase         string
 168                         SearchFilters      string
 169                         EmailAttribute     string
 170                         UsernameAttribute  string
 171                 }
 172                 Google struct {
 173                         Enable                          bool
 174                         ClientID                        string
 175                         ClientSecret                    string
 176                         AlternateEmailAddresses         bool
 177                         AuthenticationRequestParameters map[string]string
 178                 }
 179                 OpenIDConnect struct {
 180                         Enable                          bool
 181                         Issuer                          string
 182                         ClientID                        string
 183                         ClientSecret                    string
 184                         EmailClaim                      string
 185                         EmailVerifiedClaim              string
 186                         UsernameClaim                   string
 187                         AcceptAccessToken               bool
 188                         AcceptAccessTokenScope          string
 189                         AuthenticationRequestParameters map[string]string
 190                 }
 191                 PAM struct {
 192                         Enable             bool
 193                         Service            string
 194                         DefaultEmailDomain string
 195                 }
 196                 Test struct {
 197                         Enable bool
 198                         Users  map[string]TestUser
 199                 }
 200                 LoginCluster       string
 201                 RemoteTokenRefresh Duration
 202                 TokenLifetime      Duration
 203                 TrustedClients     map[string]struct{}
 204                 IssueTrustedTokens bool
 205         }
 206         Mail struct {
 207                 MailchimpAPIKey                string
 208                 MailchimpListID                string
 209                 SendUserSetupNotificationEmail bool
 210                 IssueReporterEmailFrom         string
 211                 IssueReporterEmailTo           string
 212                 SupportEmailAddress            string
 213                 EmailFrom                      string
 214         }
 215         SystemLogs struct {
 216                 LogLevel                string
 217                 Format                  string
 218                 MaxRequestLogParamsSize int
 219         }
 220         TLS struct {
 221                 Certificate string
 222                 Key         string
 223                 Insecure    bool
 224         }
 225         Users struct {
 226                 ActivatedUsersAreVisibleToOthers      bool
 227                 AnonymousUserToken                    string
 228                 AdminNotifierEmailFrom                string
 229                 AutoAdminFirstUser                    bool
 230                 AutoAdminUserWithEmail                string
 231                 AutoSetupNewUsers                     bool
 232                 AutoSetupNewUsersWithRepository       bool
 233                 AutoSetupNewUsersWithVmUUID           string
 234                 AutoSetupUsernameBlacklist            StringSet
 235                 EmailSubjectPrefix                    string
 236                 NewInactiveUserNotificationRecipients StringSet
 237                 NewUserNotificationRecipients         StringSet
 238                 NewUsersAreActive                     bool
 239                 UserNotifierEmailFrom                 string
 240                 UserNotifierEmailBcc                  StringSet
 241                 UserProfileNotificationAddress        string
 242                 PreferDomainForUsername               string
 243                 UserSetupMailText                     string
 244                 RoleGroupsVisibleToAll                bool
 245         }
 246         StorageClasses map[string]StorageClassConfig
 247         Volumes        map[string]Volume
 248         Workbench      struct {
 249                 ActivationContactLink            string
 250                 APIClientConnectTimeout          Duration
 251                 APIClientReceiveTimeout          Duration
 252                 APIResponseCompression           bool
 253                 ApplicationMimetypesWithViewIcon StringSet
 254                 ArvadosDocsite                   string
 255                 ArvadosPublicDataDocURL          string
 256                 DefaultOpenIdPrefix              string
 257                 EnableGettingStartedPopup        bool
 258                 EnablePublicProjectsPage         bool
 259                 FileViewersConfigURL             string
 260                 LogViewerMaxBytes                ByteSize
 261                 MultiSiteSearch                  string
 262                 ProfilingEnabled                 bool
 263                 Repositories                     bool
 264                 RepositoryCache                  string
 265                 RunningJobLogRecordsToFetch      int
 266                 SecretKeyBase                    string
 267                 ShowRecentCollectionsOnDashboard bool
 268                 ShowUserAgreementInline          bool
 269                 ShowUserNotifications            bool
 270                 SiteName                         string
 271                 Theme                            string
 272                 UserProfileFormFields            map[string]struct {
 273                         Type                 string
 274                         FormFieldTitle       string
 275                         FormFieldDescription string
 276                         Required             bool
 277                         Position             int
 278                         Options              map[string]struct{}
 279                 }
 280                 UserProfileFormMessage string
 281                 WelcomePageHTML        string
 282                 InactivePageHTML       string
 283                 SSHHelpPageHTML        string
 284                 SSHHelpHostSuffix      string
 285                 IdleTimeout            Duration
 286         }
 287 }
 288
 289 type StorageClassConfig struct {
 290         Default  bool
 291         Priority int
 292 }
 293
 294 type Volume struct {
 295         AccessViaHosts   map[URL]VolumeAccess
 296         ReadOnly         bool
 297         Replication      int
 298         StorageClasses   map[string]bool
 299         Driver           string
 300         DriverParameters json.RawMessage
 301 }
 302
 303 type S3VolumeDriverParameters struct {
 304         IAMRole            string
 305         AccessKeyID        string
 306         SecretAccessKey    string
 307         Endpoint           string
 308         Region             string
 309         Bucket             string
 310         LocationConstraint bool
 311         V2Signature        bool
 312         UseAWSS3v2Driver   bool
 313         IndexPageSize      int
 314         ConnectTimeout     Duration
 315         ReadTimeout        Duration
 316         RaceWindow         Duration
 317         UnsafeDelete       bool
 318         PrefixLength       int
 319 }
 320
 321 type AzureVolumeDriverParameters struct {
 322         StorageAccountName   string
 323         StorageAccountKey    string
 324         StorageBaseURL       string
 325         ContainerName        string
 326         RequestTimeout       Duration
 327         ListBlobsRetryDelay  Duration
 328         ListBlobsMaxAttempts int
 329 }
 330
 331 type DirectoryVolumeDriverParameters struct {
 332         Root      string
 333         Serialize bool
 334 }
 335
 336 type VolumeAccess struct {
 337         ReadOnly bool
 338 }
 339
 340 type Services struct {
 341         Composer       Service
 342         Controller     Service
 343         DispatchCloud  Service
 344         DispatchLSF    Service
 345         GitHTTP        Service
 346         GitSSH         Service
 347         Health         Service
 348         Keepbalance    Service
 349         Keepproxy      Service
 350         Keepstore      Service
 351         RailsAPI       Service
 352         WebDAVDownload Service
 353         WebDAV         Service
 354         WebShell       Service
 355         Websocket      Service
 356         Workbench1     Service
 357         Workbench2     Service
 358 }
 359
 360 type Service struct {
 361         InternalURLs map[URL]ServiceInstance
 362         ExternalURL  URL
 363 }
 364
 365 type TestUser struct {
 366         Email    string
 367         Password string
 368 }
 369
 370 // URL is a url.URL that is also usable as a JSON key/value.
 371 type URL url.URL
 372
 373 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 374 // used as a JSON key/value.
 375 func (su *URL) UnmarshalText(text []byte) error {
 376         u, err := url.Parse(string(text))
 377         if err == nil {
 378                 *su = URL(*u)
 379                 if su.Path == "" && su.Host != "" {
 380                         // http://example really means http://example/
 381                         su.Path = "/"
 382                 }
 383         }
 384         return err
 385 }
 386
 387 func (su URL) MarshalText() ([]byte, error) {
 388         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 389 }
 390
 391 func (su URL) String() string {
 392         return (*url.URL)(&su).String()
 393 }
 394
 395 type ServiceInstance struct {
 396         Rendezvous string `json:",omitempty"`
 397 }
 398
 399 type PostgreSQL struct {
 400         Connection     PostgreSQLConnection
 401         ConnectionPool int
 402 }
 403
 404 type PostgreSQLConnection map[string]string
 405
 406 type RemoteCluster struct {
 407         Host          string
 408         Proxy         bool
 409         Scheme        string
 410         Insecure      bool
 411         ActivateUsers bool
 412 }
 413
 414 type CUDAFeatures struct {
 415         DriverVersion      string
 416         HardwareCapability string
 417         DeviceCount        int
 418 }
 419
 420 type InstanceType struct {
 421         Name            string
 422         ProviderType    string
 423         VCPUs           int
 424         RAM             ByteSize
 425         Scratch         ByteSize
 426         IncludedScratch ByteSize
 427         AddedScratch    ByteSize
 428         Price           float64
 429         Preemptible     bool
 430         CUDA            CUDAFeatures
 431 }
 432
 433 type ContainersConfig struct {
 434         CloudVMs                      CloudVMsConfig
 435         CrunchRunCommand              string
 436         CrunchRunArgumentsList        []string
 437         DefaultKeepCacheRAM           ByteSize
 438         DispatchPrivateKey            string
 439         LogReuseDecisions             bool
 440         MaxComputeVMs                 int
 441         MaxDispatchAttempts           int
 442         MaxRetryAttempts              int
 443         MinRetryPeriod                Duration
 444         ReserveExtraRAM               ByteSize
 445         StaleLockTimeout              Duration
 446         SupportedDockerImageFormats   StringSet
 447         AlwaysUsePreemptibleInstances bool
 448         PreemptiblePriceFactor        float64
 449         RuntimeEngine                 string
 450         LocalKeepBlobBuffersPerVCPU   int
 451         LocalKeepLogsToContainerLog   string
 452
 453         JobsAPI struct {
 454                 Enable         string
 455                 GitInternalDir string
 456         }
 457         Logging struct {
 458                 MaxAge                       Duration
 459                 LogBytesPerEvent             int
 460                 LogSecondsBetweenEvents      Duration
 461                 LogThrottlePeriod            Duration
 462                 LogThrottleBytes             int
 463                 LogThrottleLines             int
 464                 LimitLogBytesPerJob          int
 465                 LogPartialLineThrottlePeriod Duration
 466                 LogUpdatePeriod              Duration
 467                 LogUpdateSize                ByteSize
 468         }
 469         ShellAccess struct {
 470                 Admin bool
 471                 User  bool
 472         }
 473         SLURM struct {
 474                 PrioritySpread             int64
 475                 SbatchArgumentsList        []string
 476                 SbatchEnvironmentVariables map[string]string
 477                 Managed                    struct {
 478                         DNSServerConfDir       string
 479                         DNSServerConfTemplate  string
 480                         DNSServerReloadCommand string
 481                         DNSServerUpdateCommand string
 482                         ComputeNodeDomain      string
 483                         ComputeNodeNameservers StringSet
 484                         AssignNodeHostname     string
 485                 }
 486         }
 487         LSF struct {
 488                 BsubSudoUser      string
 489                 BsubArgumentsList []string
 490                 BsubCUDAArguments []string
 491         }
 492 }
 493
 494 type CloudVMsConfig struct {
 495         Enable bool
 496
 497         BootProbeCommand               string
 498         DeployRunnerBinary             string
 499         ImageID                        string
 500         MaxCloudOpsPerSecond           int
 501         MaxProbesPerSecond             int
 502         MaxConcurrentInstanceCreateOps int
 503         PollInterval                   Duration
 504         ProbeInterval                  Duration
 505         SSHPort                        string
 506         SyncInterval                   Duration
 507         TimeoutBooting                 Duration
 508         TimeoutIdle                    Duration
 509         TimeoutProbe                   Duration
 510         TimeoutShutdown                Duration
 511         TimeoutSignal                  Duration
 512         TimeoutStaleRunLock            Duration
 513         TimeoutTERM                    Duration
 514         ResourceTags                   map[string]string
 515         TagKeyPrefix                   string
 516
 517         Driver           string
 518         DriverParameters json.RawMessage
 519 }
 520
 521 type InstanceTypeMap map[string]InstanceType
 522
 523 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 524
 525 // UnmarshalJSON handles old config files that provide an array of
 526 // instance types instead of a hash.
 527 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 528         fixup := func(t InstanceType) (InstanceType, error) {
 529                 if t.ProviderType == "" {
 530                         t.ProviderType = t.Name
 531                 }
 532                 if t.Scratch == 0 {
 533                         t.Scratch = t.IncludedScratch + t.AddedScratch
 534                 } else if t.AddedScratch == 0 {
 535                         t.AddedScratch = t.Scratch - t.IncludedScratch
 536                 } else if t.IncludedScratch == 0 {
 537                         t.IncludedScratch = t.Scratch - t.AddedScratch
 538                 }
 539
 540                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 541                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 542                 }
 543                 return t, nil
 544         }
 545
 546         if len(data) > 0 && data[0] == '[' {
 547                 var arr []InstanceType
 548                 err := json.Unmarshal(data, &arr)
 549                 if err != nil {
 550                         return err
 551                 }
 552                 if len(arr) == 0 {
 553                         *it = nil
 554                         return nil
 555                 }
 556                 *it = make(map[string]InstanceType, len(arr))
 557                 for _, t := range arr {
 558                         if _, ok := (*it)[t.Name]; ok {
 559                                 return errDuplicateInstanceTypeName
 560                         }
 561                         t, err := fixup(t)
 562                         if err != nil {
 563                                 return err
 564                         }
 565                         (*it)[t.Name] = t
 566                 }
 567                 return nil
 568         }
 569         var hash map[string]InstanceType
 570         err := json.Unmarshal(data, &hash)
 571         if err != nil {
 572                 return err
 573         }
 574         // Fill in Name field (and ProviderType field, if not
 575         // specified) using hash key.
 576         *it = InstanceTypeMap(hash)
 577         for name, t := range *it {
 578                 t.Name = name
 579                 t, err := fixup(t)
 580                 if err != nil {
 581                         return err
 582                 }
 583                 (*it)[name] = t
 584         }
 585         return nil
 586 }
 587
 588 type StringSet map[string]struct{}
 589
 590 // UnmarshalJSON handles old config files that provide an array of
 591 // instance types instead of a hash.
 592 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 593         if len(data) > 0 && data[0] == '[' {
 594                 var arr []string
 595                 err := json.Unmarshal(data, &arr)
 596                 if err != nil {
 597                         return err
 598                 }
 599                 if len(arr) == 0 {
 600                         *ss = nil
 601                         return nil
 602                 }
 603                 *ss = make(map[string]struct{}, len(arr))
 604                 for _, t := range arr {
 605                         (*ss)[t] = struct{}{}
 606                 }
 607                 return nil
 608         }
 609         var hash map[string]struct{}
 610         err := json.Unmarshal(data, &hash)
 611         if err != nil {
 612                 return err
 613         }
 614         *ss = make(map[string]struct{}, len(hash))
 615         for t := range hash {
 616                 (*ss)[t] = struct{}{}
 617         }
 618
 619         return nil
 620 }
 621
 622 type ServiceName string
 623
 624 const (
 625         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 626         ServiceNameController    ServiceName = "arvados-controller"
 627         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 628         ServiceNameDispatchLSF   ServiceName = "arvados-dispatch-lsf"
 629         ServiceNameHealth        ServiceName = "arvados-health"
 630         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 631         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 632         ServiceNameWebsocket     ServiceName = "arvados-ws"
 633         ServiceNameKeepbalance   ServiceName = "keep-balance"
 634         ServiceNameKeepweb       ServiceName = "keep-web"
 635         ServiceNameKeepproxy     ServiceName = "keepproxy"
 636         ServiceNameKeepstore     ServiceName = "keepstore"
 637 )
 638
 639 // Map returns all services as a map, suitable for iterating over all
 640 // services or looking up a service by name.
 641 func (svcs Services) Map() map[ServiceName]Service {
 642         return map[ServiceName]Service{
 643                 ServiceNameRailsAPI:      svcs.RailsAPI,
 644                 ServiceNameController:    svcs.Controller,
 645                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 646                 ServiceNameDispatchLSF:   svcs.DispatchLSF,
 647                 ServiceNameHealth:        svcs.Health,
 648                 ServiceNameWorkbench1:    svcs.Workbench1,
 649                 ServiceNameWorkbench2:    svcs.Workbench2,
 650                 ServiceNameWebsocket:     svcs.Websocket,
 651                 ServiceNameKeepbalance:   svcs.Keepbalance,
 652                 ServiceNameKeepweb:       svcs.WebDAV,
 653                 ServiceNameKeepproxy:     svcs.Keepproxy,
 654                 ServiceNameKeepstore:     svcs.Keepstore,
 655         }
 656 }