sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.arvados.org/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         }
  21         return "/etc/arvados/config.yml"
  22 }()
  23
  24 type Config struct {
  25         Clusters         map[string]Cluster
  26         AutoReloadConfig bool
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         cc, ok := sc.Clusters[clusterID]
  53         if !ok {
  54                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  55         }
  56         cc.ClusterID = clusterID
  57         return &cc, nil
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxBlockEntries      int
  64         MaxCollectionEntries int
  65         MaxCollectionBytes   int64
  66         MaxUUIDEntries       int
  67         MaxSessions          int
  68 }
  69
  70 type UploadDownloadPermission struct {
  71         Upload   bool
  72         Download bool
  73 }
  74
  75 type UploadDownloadRolePermissions struct {
  76         User  UploadDownloadPermission
  77         Admin UploadDownloadPermission
  78 }
  79
  80 type ManagedProperties map[string]struct {
  81         Value     interface{}
  82         Function  string
  83         Protected bool
  84 }
  85
  86 type Cluster struct {
  87         ClusterID       string `json:"-"`
  88         ManagementToken string
  89         SystemRootToken string
  90         Services        Services
  91         InstanceTypes   InstanceTypeMap
  92         Containers      ContainersConfig
  93         RemoteClusters  map[string]RemoteCluster
  94         PostgreSQL      PostgreSQL
  95
  96         API struct {
  97                 AsyncPermissionsUpdateInterval Duration
  98                 DisabledAPIs                   StringSet
  99                 MaxIndexDatabaseRead           int
 100                 MaxItemsPerResponse            int
 101                 MaxConcurrentRequests          int
 102                 MaxKeepBlobBuffers             int
 103                 MaxRequestAmplification        int
 104                 MaxRequestSize                 int
 105                 MaxTokenLifetime               Duration
 106                 RequestTimeout                 Duration
 107                 SendTimeout                    Duration
 108                 WebsocketClientEventQueue      int
 109                 WebsocketServerEventQueue      int
 110                 KeepServiceRequestTimeout      Duration
 111                 VocabularyPath                 string
 112         }
 113         AuditLogs struct {
 114                 MaxAge             Duration
 115                 MaxDeleteBatch     int
 116                 UnloggedAttributes StringSet
 117         }
 118         Collections struct {
 119                 BlobSigning                  bool
 120                 BlobSigningKey               string
 121                 BlobSigningTTL               Duration
 122                 BlobTrash                    bool
 123                 BlobTrashLifetime            Duration
 124                 BlobTrashCheckInterval       Duration
 125                 BlobTrashConcurrency         int
 126                 BlobDeleteConcurrency        int
 127                 BlobReplicateConcurrency     int
 128                 CollectionVersioning         bool
 129                 DefaultTrashLifetime         Duration
 130                 DefaultReplication           int
 131                 ManagedProperties            ManagedProperties
 132                 PreserveVersionIfIdle        Duration
 133                 TrashSweepInterval           Duration
 134                 TrustAllContent              bool
 135                 ForwardSlashNameSubstitution string
 136                 S3FolderObjects              bool
 137
 138                 BlobMissingReport        string
 139                 BalancePeriod            Duration
 140                 BalanceCollectionBatch   int
 141                 BalanceCollectionBuffers int
 142                 BalanceTimeout           Duration
 143                 BalanceUpdateLimit       int
 144
 145                 WebDAVCache WebDAVCacheConfig
 146
 147                 KeepproxyPermission UploadDownloadRolePermissions
 148                 WebDAVPermission    UploadDownloadRolePermissions
 149                 WebDAVLogEvents     bool
 150         }
 151         Git struct {
 152                 GitCommand   string
 153                 GitoliteHome string
 154                 Repositories string
 155         }
 156         Login struct {
 157                 LDAP struct {
 158                         Enable             bool
 159                         URL                URL
 160                         StartTLS           bool
 161                         InsecureTLS        bool
 162                         StripDomain        string
 163                         AppendDomain       string
 164                         SearchAttribute    string
 165                         SearchBindUser     string
 166                         SearchBindPassword string
 167                         SearchBase         string
 168                         SearchFilters      string
 169                         EmailAttribute     string
 170                         UsernameAttribute  string
 171                 }
 172                 Google struct {
 173                         Enable                          bool
 174                         ClientID                        string
 175                         ClientSecret                    string
 176                         AlternateEmailAddresses         bool
 177                         AuthenticationRequestParameters map[string]string
 178                 }
 179                 OpenIDConnect struct {
 180                         Enable                          bool
 181                         Issuer                          string
 182                         ClientID                        string
 183                         ClientSecret                    string
 184                         EmailClaim                      string
 185                         EmailVerifiedClaim              string
 186                         UsernameClaim                   string
 187                         AcceptAccessToken               bool
 188                         AcceptAccessTokenScope          string
 189                         AuthenticationRequestParameters map[string]string
 190                 }
 191                 PAM struct {
 192                         Enable             bool
 193                         Service            string
 194                         DefaultEmailDomain string
 195                 }
 196                 Test struct {
 197                         Enable bool
 198                         Users  map[string]TestUser
 199                 }
 200                 LoginCluster       string
 201                 RemoteTokenRefresh Duration
 202                 TokenLifetime      Duration
 203                 TrustedClients     map[string]struct{}
 204                 IssueTrustedTokens bool
 205         }
 206         Mail struct {
 207                 MailchimpAPIKey                string
 208                 MailchimpListID                string
 209                 SendUserSetupNotificationEmail bool
 210                 IssueReporterEmailFrom         string
 211                 IssueReporterEmailTo           string
 212                 SupportEmailAddress            string
 213                 EmailFrom                      string
 214         }
 215         SystemLogs struct {
 216                 LogLevel                string
 217                 Format                  string
 218                 MaxRequestLogParamsSize int
 219         }
 220         TLS struct {
 221                 Certificate string
 222                 Key         string
 223                 Insecure    bool
 224         }
 225         Users struct {
 226                 ActivatedUsersAreVisibleToOthers      bool
 227                 AnonymousUserToken                    string
 228                 AdminNotifierEmailFrom                string
 229                 AutoAdminFirstUser                    bool
 230                 AutoAdminUserWithEmail                string
 231                 AutoSetupNewUsers                     bool
 232                 AutoSetupNewUsersWithRepository       bool
 233                 AutoSetupNewUsersWithVmUUID           string
 234                 AutoSetupUsernameBlacklist            StringSet
 235                 EmailSubjectPrefix                    string
 236                 NewInactiveUserNotificationRecipients StringSet
 237                 NewUserNotificationRecipients         StringSet
 238                 NewUsersAreActive                     bool
 239                 UserNotifierEmailFrom                 string
 240                 UserNotifierEmailBcc                  StringSet
 241                 UserProfileNotificationAddress        string
 242                 PreferDomainForUsername               string
 243                 UserSetupMailText                     string
 244         }
 245         StorageClasses map[string]StorageClassConfig
 246         Volumes        map[string]Volume
 247         Workbench      struct {
 248                 ActivationContactLink            string
 249                 APIClientConnectTimeout          Duration
 250                 APIClientReceiveTimeout          Duration
 251                 APIResponseCompression           bool
 252                 ApplicationMimetypesWithViewIcon StringSet
 253                 ArvadosDocsite                   string
 254                 ArvadosPublicDataDocURL          string
 255                 DefaultOpenIdPrefix              string
 256                 EnableGettingStartedPopup        bool
 257                 EnablePublicProjectsPage         bool
 258                 FileViewersConfigURL             string
 259                 LogViewerMaxBytes                ByteSize
 260                 MultiSiteSearch                  string
 261                 ProfilingEnabled                 bool
 262                 Repositories                     bool
 263                 RepositoryCache                  string
 264                 RunningJobLogRecordsToFetch      int
 265                 SecretKeyBase                    string
 266                 ShowRecentCollectionsOnDashboard bool
 267                 ShowUserAgreementInline          bool
 268                 ShowUserNotifications            bool
 269                 SiteName                         string
 270                 Theme                            string
 271                 UserProfileFormFields            map[string]struct {
 272                         Type                 string
 273                         FormFieldTitle       string
 274                         FormFieldDescription string
 275                         Required             bool
 276                         Position             int
 277                         Options              map[string]struct{}
 278                 }
 279                 UserProfileFormMessage string
 280                 WelcomePageHTML        string
 281                 InactivePageHTML       string
 282                 SSHHelpPageHTML        string
 283                 SSHHelpHostSuffix      string
 284                 IdleTimeout            Duration
 285         }
 286 }
 287
 288 type StorageClassConfig struct {
 289         Default  bool
 290         Priority int
 291 }
 292
 293 type Volume struct {
 294         AccessViaHosts   map[URL]VolumeAccess
 295         ReadOnly         bool
 296         Replication      int
 297         StorageClasses   map[string]bool
 298         Driver           string
 299         DriverParameters json.RawMessage
 300 }
 301
 302 type S3VolumeDriverParameters struct {
 303         IAMRole            string
 304         AccessKeyID        string
 305         SecretAccessKey    string
 306         Endpoint           string
 307         Region             string
 308         Bucket             string
 309         LocationConstraint bool
 310         V2Signature        bool
 311         UseAWSS3v2Driver   bool
 312         IndexPageSize      int
 313         ConnectTimeout     Duration
 314         ReadTimeout        Duration
 315         RaceWindow         Duration
 316         UnsafeDelete       bool
 317         PrefixLength       int
 318 }
 319
 320 type AzureVolumeDriverParameters struct {
 321         StorageAccountName   string
 322         StorageAccountKey    string
 323         StorageBaseURL       string
 324         ContainerName        string
 325         RequestTimeout       Duration
 326         ListBlobsRetryDelay  Duration
 327         ListBlobsMaxAttempts int
 328 }
 329
 330 type DirectoryVolumeDriverParameters struct {
 331         Root      string
 332         Serialize bool
 333 }
 334
 335 type VolumeAccess struct {
 336         ReadOnly bool
 337 }
 338
 339 type Services struct {
 340         Composer       Service
 341         Controller     Service
 342         DispatchCloud  Service
 343         DispatchLSF    Service
 344         GitHTTP        Service
 345         GitSSH         Service
 346         Health         Service
 347         Keepbalance    Service
 348         Keepproxy      Service
 349         Keepstore      Service
 350         RailsAPI       Service
 351         WebDAVDownload Service
 352         WebDAV         Service
 353         WebShell       Service
 354         Websocket      Service
 355         Workbench1     Service
 356         Workbench2     Service
 357 }
 358
 359 type Service struct {
 360         InternalURLs map[URL]ServiceInstance
 361         ExternalURL  URL
 362 }
 363
 364 type TestUser struct {
 365         Email    string
 366         Password string
 367 }
 368
 369 // URL is a url.URL that is also usable as a JSON key/value.
 370 type URL url.URL
 371
 372 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 373 // used as a JSON key/value.
 374 func (su *URL) UnmarshalText(text []byte) error {
 375         u, err := url.Parse(string(text))
 376         if err == nil {
 377                 *su = URL(*u)
 378                 if su.Path == "" && su.Host != "" {
 379                         // http://example really means http://example/
 380                         su.Path = "/"
 381                 }
 382         }
 383         return err
 384 }
 385
 386 func (su URL) MarshalText() ([]byte, error) {
 387         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 388 }
 389
 390 func (su URL) String() string {
 391         return (*url.URL)(&su).String()
 392 }
 393
 394 type ServiceInstance struct {
 395         Rendezvous string `json:",omitempty"`
 396 }
 397
 398 type PostgreSQL struct {
 399         Connection     PostgreSQLConnection
 400         ConnectionPool int
 401 }
 402
 403 type PostgreSQLConnection map[string]string
 404
 405 type RemoteCluster struct {
 406         Host          string
 407         Proxy         bool
 408         Scheme        string
 409         Insecure      bool
 410         ActivateUsers bool
 411 }
 412
 413 type CUDAFeatures struct {
 414         DriverVersion      string
 415         HardwareCapability string
 416         DeviceCount        int
 417 }
 418
 419 type InstanceType struct {
 420         Name            string
 421         ProviderType    string
 422         VCPUs           int
 423         RAM             ByteSize
 424         Scratch         ByteSize
 425         IncludedScratch ByteSize
 426         AddedScratch    ByteSize
 427         Price           float64
 428         Preemptible     bool
 429         CUDA            CUDAFeatures
 430 }
 431
 432 type ContainersConfig struct {
 433         CloudVMs                    CloudVMsConfig
 434         CrunchRunCommand            string
 435         CrunchRunArgumentsList      []string
 436         DefaultKeepCacheRAM         ByteSize
 437         DispatchPrivateKey          string
 438         LogReuseDecisions           bool
 439         MaxComputeVMs               int
 440         MaxDispatchAttempts         int
 441         MaxRetryAttempts            int
 442         MinRetryPeriod              Duration
 443         ReserveExtraRAM             ByteSize
 444         StaleLockTimeout            Duration
 445         SupportedDockerImageFormats StringSet
 446         UsePreemptibleInstances     bool
 447         RuntimeEngine               string
 448         LocalKeepBlobBuffersPerVCPU int
 449         LocalKeepLogsToContainerLog string
 450
 451         JobsAPI struct {
 452                 Enable         string
 453                 GitInternalDir string
 454         }
 455         Logging struct {
 456                 MaxAge                       Duration
 457                 LogBytesPerEvent             int
 458                 LogSecondsBetweenEvents      Duration
 459                 LogThrottlePeriod            Duration
 460                 LogThrottleBytes             int
 461                 LogThrottleLines             int
 462                 LimitLogBytesPerJob          int
 463                 LogPartialLineThrottlePeriod Duration
 464                 LogUpdatePeriod              Duration
 465                 LogUpdateSize                ByteSize
 466         }
 467         ShellAccess struct {
 468                 Admin bool
 469                 User  bool
 470         }
 471         SLURM struct {
 472                 PrioritySpread             int64
 473                 SbatchArgumentsList        []string
 474                 SbatchEnvironmentVariables map[string]string
 475                 Managed                    struct {
 476                         DNSServerConfDir       string
 477                         DNSServerConfTemplate  string
 478                         DNSServerReloadCommand string
 479                         DNSServerUpdateCommand string
 480                         ComputeNodeDomain      string
 481                         ComputeNodeNameservers StringSet
 482                         AssignNodeHostname     string
 483                 }
 484         }
 485         LSF struct {
 486                 BsubSudoUser      string
 487                 BsubArgumentsList []string
 488         }
 489 }
 490
 491 type CloudVMsConfig struct {
 492         Enable bool
 493
 494         BootProbeCommand               string
 495         DeployRunnerBinary             string
 496         ImageID                        string
 497         MaxCloudOpsPerSecond           int
 498         MaxProbesPerSecond             int
 499         MaxConcurrentInstanceCreateOps int
 500         PollInterval                   Duration
 501         ProbeInterval                  Duration
 502         SSHPort                        string
 503         SyncInterval                   Duration
 504         TimeoutBooting                 Duration
 505         TimeoutIdle                    Duration
 506         TimeoutProbe                   Duration
 507         TimeoutShutdown                Duration
 508         TimeoutSignal                  Duration
 509         TimeoutStaleRunLock            Duration
 510         TimeoutTERM                    Duration
 511         ResourceTags                   map[string]string
 512         TagKeyPrefix                   string
 513
 514         Driver           string
 515         DriverParameters json.RawMessage
 516 }
 517
 518 type InstanceTypeMap map[string]InstanceType
 519
 520 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 521
 522 // UnmarshalJSON handles old config files that provide an array of
 523 // instance types instead of a hash.
 524 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 525         fixup := func(t InstanceType) (InstanceType, error) {
 526                 if t.ProviderType == "" {
 527                         t.ProviderType = t.Name
 528                 }
 529                 if t.Scratch == 0 {
 530                         t.Scratch = t.IncludedScratch + t.AddedScratch
 531                 } else if t.AddedScratch == 0 {
 532                         t.AddedScratch = t.Scratch - t.IncludedScratch
 533                 } else if t.IncludedScratch == 0 {
 534                         t.IncludedScratch = t.Scratch - t.AddedScratch
 535                 }
 536
 537                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 538                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 539                 }
 540                 return t, nil
 541         }
 542
 543         if len(data) > 0 && data[0] == '[' {
 544                 var arr []InstanceType
 545                 err := json.Unmarshal(data, &arr)
 546                 if err != nil {
 547                         return err
 548                 }
 549                 if len(arr) == 0 {
 550                         *it = nil
 551                         return nil
 552                 }
 553                 *it = make(map[string]InstanceType, len(arr))
 554                 for _, t := range arr {
 555                         if _, ok := (*it)[t.Name]; ok {
 556                                 return errDuplicateInstanceTypeName
 557                         }
 558                         t, err := fixup(t)
 559                         if err != nil {
 560                                 return err
 561                         }
 562                         (*it)[t.Name] = t
 563                 }
 564                 return nil
 565         }
 566         var hash map[string]InstanceType
 567         err := json.Unmarshal(data, &hash)
 568         if err != nil {
 569                 return err
 570         }
 571         // Fill in Name field (and ProviderType field, if not
 572         // specified) using hash key.
 573         *it = InstanceTypeMap(hash)
 574         for name, t := range *it {
 575                 t.Name = name
 576                 t, err := fixup(t)
 577                 if err != nil {
 578                         return err
 579                 }
 580                 (*it)[name] = t
 581         }
 582         return nil
 583 }
 584
 585 type StringSet map[string]struct{}
 586
 587 // UnmarshalJSON handles old config files that provide an array of
 588 // instance types instead of a hash.
 589 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 590         if len(data) > 0 && data[0] == '[' {
 591                 var arr []string
 592                 err := json.Unmarshal(data, &arr)
 593                 if err != nil {
 594                         return err
 595                 }
 596                 if len(arr) == 0 {
 597                         *ss = nil
 598                         return nil
 599                 }
 600                 *ss = make(map[string]struct{}, len(arr))
 601                 for _, t := range arr {
 602                         (*ss)[t] = struct{}{}
 603                 }
 604                 return nil
 605         }
 606         var hash map[string]struct{}
 607         err := json.Unmarshal(data, &hash)
 608         if err != nil {
 609                 return err
 610         }
 611         *ss = make(map[string]struct{}, len(hash))
 612         for t := range hash {
 613                 (*ss)[t] = struct{}{}
 614         }
 615
 616         return nil
 617 }
 618
 619 type ServiceName string
 620
 621 const (
 622         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 623         ServiceNameController    ServiceName = "arvados-controller"
 624         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 625         ServiceNameDispatchLSF   ServiceName = "arvados-dispatch-lsf"
 626         ServiceNameHealth        ServiceName = "arvados-health"
 627         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 628         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 629         ServiceNameWebsocket     ServiceName = "arvados-ws"
 630         ServiceNameKeepbalance   ServiceName = "keep-balance"
 631         ServiceNameKeepweb       ServiceName = "keep-web"
 632         ServiceNameKeepproxy     ServiceName = "keepproxy"
 633         ServiceNameKeepstore     ServiceName = "keepstore"
 634 )
 635
 636 // Map returns all services as a map, suitable for iterating over all
 637 // services or looking up a service by name.
 638 func (svcs Services) Map() map[ServiceName]Service {
 639         return map[ServiceName]Service{
 640                 ServiceNameRailsAPI:      svcs.RailsAPI,
 641                 ServiceNameController:    svcs.Controller,
 642                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 643                 ServiceNameDispatchLSF:   svcs.DispatchLSF,
 644                 ServiceNameHealth:        svcs.Health,
 645                 ServiceNameWorkbench1:    svcs.Workbench1,
 646                 ServiceNameWorkbench2:    svcs.Workbench2,
 647                 ServiceNameWebsocket:     svcs.Websocket,
 648                 ServiceNameKeepbalance:   svcs.Keepbalance,
 649                 ServiceNameKeepweb:       svcs.WebDAV,
 650                 ServiceNameKeepproxy:     svcs.Keepproxy,
 651                 ServiceNameKeepstore:     svcs.Keepstore,
 652         }
 653 }