sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13         "time"
  14
  15         "git.arvados.org/arvados.git/sdk/go/config"
  16 )
  17
  18 var DefaultConfigFile = func() string {
  19         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  20                 return path
  21         }
  22         return "/etc/arvados/config.yml"
  23 }()
  24
  25 type Config struct {
  26         Clusters         map[string]Cluster
  27         AutoReloadConfig bool
  28         SourceTimestamp  time.Time
  29         SourceSHA256     string
  30 }
  31
  32 // GetConfig returns the current system config, loading it from
  33 // configFile if needed.
  34 func GetConfig(configFile string) (*Config, error) {
  35         var cfg Config
  36         err := config.LoadFile(&cfg, configFile)
  37         return &cfg, err
  38 }
  39
  40 // GetCluster returns the cluster ID and config for the given
  41 // cluster, or the default/only configured cluster if clusterID is "".
  42 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  43         if clusterID == "" {
  44                 if len(sc.Clusters) == 0 {
  45                         return nil, fmt.Errorf("no clusters configured")
  46                 } else if len(sc.Clusters) > 1 {
  47                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  48                 } else {
  49                         for id, cc := range sc.Clusters {
  50                                 cc.ClusterID = id
  51                                 return &cc, nil
  52                         }
  53                 }
  54         }
  55         cc, ok := sc.Clusters[clusterID]
  56         if !ok {
  57                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  58         }
  59         cc.ClusterID = clusterID
  60         return &cc, nil
  61 }
  62
  63 type WebDAVCacheConfig struct {
  64         TTL                Duration
  65         MaxBlockEntries    int
  66         MaxCollectionBytes int64
  67         MaxSessions        int
  68 }
  69
  70 type UploadDownloadPermission struct {
  71         Upload   bool
  72         Download bool
  73 }
  74
  75 type UploadDownloadRolePermissions struct {
  76         User  UploadDownloadPermission
  77         Admin UploadDownloadPermission
  78 }
  79
  80 type ManagedProperties map[string]struct {
  81         Value     interface{}
  82         Function  string
  83         Protected bool
  84 }
  85
  86 type Cluster struct {
  87         ClusterID       string `json:"-"`
  88         ManagementToken string
  89         SystemRootToken string
  90         Services        Services
  91         InstanceTypes   InstanceTypeMap
  92         Containers      ContainersConfig
  93         RemoteClusters  map[string]RemoteCluster
  94         PostgreSQL      PostgreSQL
  95
  96         API struct {
  97                 AsyncPermissionsUpdateInterval   Duration
  98                 DisabledAPIs                     StringSet
  99                 MaxIndexDatabaseRead             int
 100                 MaxItemsPerResponse              int
 101                 MaxConcurrentRequests            int
 102                 MaxKeepBlobBuffers               int
 103                 MaxRequestAmplification          int
 104                 MaxRequestSize                   int
 105                 MaxTokenLifetime                 Duration
 106                 RequestTimeout                   Duration
 107                 SendTimeout                      Duration
 108                 WebsocketClientEventQueue        int
 109                 WebsocketServerEventQueue        int
 110                 KeepServiceRequestTimeout        Duration
 111                 VocabularyPath                   string
 112                 FreezeProjectRequiresDescription bool
 113                 FreezeProjectRequiresProperties  StringSet
 114                 UnfreezeProjectRequiresAdmin     bool
 115         }
 116         AuditLogs struct {
 117                 MaxAge             Duration
 118                 MaxDeleteBatch     int
 119                 UnloggedAttributes StringSet
 120         }
 121         Collections struct {
 122                 BlobSigning                  bool
 123                 BlobSigningKey               string
 124                 BlobSigningTTL               Duration
 125                 BlobTrash                    bool
 126                 BlobTrashLifetime            Duration
 127                 BlobTrashCheckInterval       Duration
 128                 BlobTrashConcurrency         int
 129                 BlobDeleteConcurrency        int
 130                 BlobReplicateConcurrency     int
 131                 CollectionVersioning         bool
 132                 DefaultTrashLifetime         Duration
 133                 DefaultReplication           int
 134                 ManagedProperties            ManagedProperties
 135                 PreserveVersionIfIdle        Duration
 136                 TrashSweepInterval           Duration
 137                 TrustAllContent              bool
 138                 ForwardSlashNameSubstitution string
 139                 S3FolderObjects              bool
 140
 141                 BlobMissingReport        string
 142                 BalancePeriod            Duration
 143                 BalanceCollectionBatch   int
 144                 BalanceCollectionBuffers int
 145                 BalanceTimeout           Duration
 146                 BalanceUpdateLimit       int
 147
 148                 WebDAVCache WebDAVCacheConfig
 149
 150                 KeepproxyPermission UploadDownloadRolePermissions
 151                 WebDAVPermission    UploadDownloadRolePermissions
 152                 WebDAVLogEvents     bool
 153         }
 154         Git struct {
 155                 GitCommand   string
 156                 GitoliteHome string
 157                 Repositories string
 158         }
 159         Login struct {
 160                 LDAP struct {
 161                         Enable             bool
 162                         URL                URL
 163                         StartTLS           bool
 164                         InsecureTLS        bool
 165                         StripDomain        string
 166                         AppendDomain       string
 167                         SearchAttribute    string
 168                         SearchBindUser     string
 169                         SearchBindPassword string
 170                         SearchBase         string
 171                         SearchFilters      string
 172                         EmailAttribute     string
 173                         UsernameAttribute  string
 174                 }
 175                 Google struct {
 176                         Enable                          bool
 177                         ClientID                        string
 178                         ClientSecret                    string
 179                         AlternateEmailAddresses         bool
 180                         AuthenticationRequestParameters map[string]string
 181                 }
 182                 OpenIDConnect struct {
 183                         Enable                          bool
 184                         Issuer                          string
 185                         ClientID                        string
 186                         ClientSecret                    string
 187                         EmailClaim                      string
 188                         EmailVerifiedClaim              string
 189                         UsernameClaim                   string
 190                         AcceptAccessToken               bool
 191                         AcceptAccessTokenScope          string
 192                         AuthenticationRequestParameters map[string]string
 193                 }
 194                 PAM struct {
 195                         Enable             bool
 196                         Service            string
 197                         DefaultEmailDomain string
 198                 }
 199                 Test struct {
 200                         Enable bool
 201                         Users  map[string]TestUser
 202                 }
 203                 LoginCluster         string
 204                 RemoteTokenRefresh   Duration
 205                 TokenLifetime        Duration
 206                 TrustedClients       map[URL]struct{}
 207                 TrustPrivateNetworks bool
 208                 IssueTrustedTokens   bool
 209         }
 210         Mail struct {
 211                 MailchimpAPIKey                string
 212                 MailchimpListID                string
 213                 SendUserSetupNotificationEmail bool
 214                 IssueReporterEmailFrom         string
 215                 IssueReporterEmailTo           string
 216                 SupportEmailAddress            string
 217                 EmailFrom                      string
 218         }
 219         SystemLogs struct {
 220                 LogLevel                string
 221                 Format                  string
 222                 MaxRequestLogParamsSize int
 223         }
 224         TLS struct {
 225                 Certificate string
 226                 Key         string
 227                 Insecure    bool
 228                 ACME        struct {
 229                         Server string
 230                 }
 231         }
 232         Users struct {
 233                 ActivatedUsersAreVisibleToOthers      bool
 234                 AnonymousUserToken                    string
 235                 AdminNotifierEmailFrom                string
 236                 AutoAdminFirstUser                    bool
 237                 AutoAdminUserWithEmail                string
 238                 AutoSetupNewUsers                     bool
 239                 AutoSetupNewUsersWithRepository       bool
 240                 AutoSetupNewUsersWithVmUUID           string
 241                 AutoSetupUsernameBlacklist            StringSet
 242                 EmailSubjectPrefix                    string
 243                 NewInactiveUserNotificationRecipients StringSet
 244                 NewUserNotificationRecipients         StringSet
 245                 NewUsersAreActive                     bool
 246                 UserNotifierEmailFrom                 string
 247                 UserNotifierEmailBcc                  StringSet
 248                 UserProfileNotificationAddress        string
 249                 PreferDomainForUsername               string
 250                 UserSetupMailText                     string
 251                 RoleGroupsVisibleToAll                bool
 252                 CanCreateRoleGroups                   bool
 253                 ActivityLoggingPeriod                 Duration
 254         }
 255         StorageClasses map[string]StorageClassConfig
 256         Volumes        map[string]Volume
 257         Workbench      struct {
 258                 ActivationContactLink            string
 259                 APIClientConnectTimeout          Duration
 260                 APIClientReceiveTimeout          Duration
 261                 APIResponseCompression           bool
 262                 ApplicationMimetypesWithViewIcon StringSet
 263                 ArvadosDocsite                   string
 264                 ArvadosPublicDataDocURL          string
 265                 DefaultOpenIdPrefix              string
 266                 DisableSharingURLsUI             bool
 267                 EnableGettingStartedPopup        bool
 268                 EnablePublicProjectsPage         bool
 269                 FileViewersConfigURL             string
 270                 LogViewerMaxBytes                ByteSize
 271                 MultiSiteSearch                  string
 272                 ProfilingEnabled                 bool
 273                 Repositories                     bool
 274                 RepositoryCache                  string
 275                 RunningJobLogRecordsToFetch      int
 276                 SecretKeyBase                    string
 277                 ShowRecentCollectionsOnDashboard bool
 278                 ShowUserAgreementInline          bool
 279                 ShowUserNotifications            bool
 280                 SiteName                         string
 281                 Theme                            string
 282                 UserProfileFormFields            map[string]struct {
 283                         Type                 string
 284                         FormFieldTitle       string
 285                         FormFieldDescription string
 286                         Required             bool
 287                         Position             int
 288                         Options              map[string]struct{}
 289                 }
 290                 UserProfileFormMessage string
 291                 WelcomePageHTML        string
 292                 InactivePageHTML       string
 293                 SSHHelpPageHTML        string
 294                 SSHHelpHostSuffix      string
 295                 IdleTimeout            Duration
 296                 BannerURL              string
 297         }
 298 }
 299
 300 type StorageClassConfig struct {
 301         Default  bool
 302         Priority int
 303 }
 304
 305 type Volume struct {
 306         AccessViaHosts   map[URL]VolumeAccess
 307         ReadOnly         bool
 308         Replication      int
 309         StorageClasses   map[string]bool
 310         Driver           string
 311         DriverParameters json.RawMessage
 312 }
 313
 314 type S3VolumeDriverParameters struct {
 315         IAMRole            string
 316         AccessKeyID        string
 317         SecretAccessKey    string
 318         Endpoint           string
 319         Region             string
 320         Bucket             string
 321         LocationConstraint bool
 322         V2Signature        bool
 323         UseAWSS3v2Driver   bool
 324         IndexPageSize      int
 325         ConnectTimeout     Duration
 326         ReadTimeout        Duration
 327         RaceWindow         Duration
 328         UnsafeDelete       bool
 329         PrefixLength       int
 330 }
 331
 332 type AzureVolumeDriverParameters struct {
 333         StorageAccountName   string
 334         StorageAccountKey    string
 335         StorageBaseURL       string
 336         ContainerName        string
 337         RequestTimeout       Duration
 338         ListBlobsRetryDelay  Duration
 339         ListBlobsMaxAttempts int
 340 }
 341
 342 type DirectoryVolumeDriverParameters struct {
 343         Root      string
 344         Serialize bool
 345 }
 346
 347 type VolumeAccess struct {
 348         ReadOnly bool
 349 }
 350
 351 type Services struct {
 352         Composer       Service
 353         Controller     Service
 354         DispatchCloud  Service
 355         DispatchLSF    Service
 356         DispatchSLURM  Service
 357         GitHTTP        Service
 358         GitSSH         Service
 359         Health         Service
 360         Keepbalance    Service
 361         Keepproxy      Service
 362         Keepstore      Service
 363         RailsAPI       Service
 364         WebDAVDownload Service
 365         WebDAV         Service
 366         WebShell       Service
 367         Websocket      Service
 368         Workbench1     Service
 369         Workbench2     Service
 370 }
 371
 372 type Service struct {
 373         InternalURLs map[URL]ServiceInstance
 374         ExternalURL  URL
 375 }
 376
 377 type TestUser struct {
 378         Email    string
 379         Password string
 380 }
 381
 382 // URL is a url.URL that is also usable as a JSON key/value.
 383 type URL url.URL
 384
 385 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 386 // used as a JSON key/value.
 387 func (su *URL) UnmarshalText(text []byte) error {
 388         u, err := url.Parse(string(text))
 389         if err == nil {
 390                 *su = URL(*u)
 391                 if su.Path == "" && su.Host != "" {
 392                         // http://example really means http://example/
 393                         su.Path = "/"
 394                 }
 395         }
 396         return err
 397 }
 398
 399 func (su URL) MarshalText() ([]byte, error) {
 400         return []byte(su.String()), nil
 401 }
 402
 403 func (su URL) String() string {
 404         return (*url.URL)(&su).String()
 405 }
 406
 407 type ServiceInstance struct {
 408         ListenURL  URL
 409         Rendezvous string `json:",omitempty"`
 410 }
 411
 412 type PostgreSQL struct {
 413         Connection     PostgreSQLConnection
 414         ConnectionPool int
 415 }
 416
 417 type PostgreSQLConnection map[string]string
 418
 419 type RemoteCluster struct {
 420         Host          string
 421         Proxy         bool
 422         Scheme        string
 423         Insecure      bool
 424         ActivateUsers bool
 425 }
 426
 427 type CUDAFeatures struct {
 428         DriverVersion      string
 429         HardwareCapability string
 430         DeviceCount        int
 431 }
 432
 433 type InstanceType struct {
 434         Name            string `json:"-"`
 435         ProviderType    string
 436         VCPUs           int
 437         RAM             ByteSize
 438         Scratch         ByteSize `json:"-"`
 439         IncludedScratch ByteSize
 440         AddedScratch    ByteSize
 441         Price           float64
 442         Preemptible     bool
 443         CUDA            CUDAFeatures
 444 }
 445
 446 type ContainersConfig struct {
 447         CloudVMs                      CloudVMsConfig
 448         CrunchRunCommand              string
 449         CrunchRunArgumentsList        []string
 450         DefaultKeepCacheRAM           ByteSize
 451         DefaultKeepCacheDisk          ByteSize
 452         DispatchPrivateKey            string
 453         LogReuseDecisions             bool
 454         MaxComputeVMs                 int
 455         MaxDispatchAttempts           int
 456         MaxRetryAttempts              int
 457         MinRetryPeriod                Duration
 458         ReserveExtraRAM               ByteSize
 459         StaleLockTimeout              Duration
 460         SupportedDockerImageFormats   StringSet
 461         AlwaysUsePreemptibleInstances bool
 462         PreemptiblePriceFactor        float64
 463         RuntimeEngine                 string
 464         LocalKeepBlobBuffersPerVCPU   int
 465         LocalKeepLogsToContainerLog   string
 466
 467         JobsAPI struct {
 468                 Enable         string
 469                 GitInternalDir string
 470         }
 471         Logging struct {
 472                 MaxAge                       Duration
 473                 SweepInterval                Duration
 474                 LogBytesPerEvent             int
 475                 LogSecondsBetweenEvents      Duration
 476                 LogThrottlePeriod            Duration
 477                 LogThrottleBytes             int
 478                 LogThrottleLines             int
 479                 LimitLogBytesPerJob          int
 480                 LogPartialLineThrottlePeriod Duration
 481                 LogUpdatePeriod              Duration
 482                 LogUpdateSize                ByteSize
 483         }
 484         ShellAccess struct {
 485                 Admin bool
 486                 User  bool
 487         }
 488         SLURM struct {
 489                 PrioritySpread             int64
 490                 SbatchArgumentsList        []string
 491                 SbatchEnvironmentVariables map[string]string
 492                 Managed                    struct {
 493                         DNSServerConfDir       string
 494                         DNSServerConfTemplate  string
 495                         DNSServerReloadCommand string
 496                         DNSServerUpdateCommand string
 497                         ComputeNodeDomain      string
 498                         ComputeNodeNameservers StringSet
 499                         AssignNodeHostname     string
 500                 }
 501         }
 502         LSF struct {
 503                 BsubSudoUser      string
 504                 BsubArgumentsList []string
 505                 BsubCUDAArguments []string
 506         }
 507 }
 508
 509 type CloudVMsConfig struct {
 510         Enable bool
 511
 512         BootProbeCommand               string
 513         DeployRunnerBinary             string
 514         ImageID                        string
 515         MaxCloudOpsPerSecond           int
 516         MaxProbesPerSecond             int
 517         MaxConcurrentInstanceCreateOps int
 518         PollInterval                   Duration
 519         ProbeInterval                  Duration
 520         SSHPort                        string
 521         SyncInterval                   Duration
 522         TimeoutBooting                 Duration
 523         TimeoutIdle                    Duration
 524         TimeoutProbe                   Duration
 525         TimeoutShutdown                Duration
 526         TimeoutSignal                  Duration
 527         TimeoutStaleRunLock            Duration
 528         TimeoutTERM                    Duration
 529         ResourceTags                   map[string]string
 530         TagKeyPrefix                   string
 531
 532         Driver           string
 533         DriverParameters json.RawMessage
 534 }
 535
 536 type InstanceTypeMap map[string]InstanceType
 537
 538 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 539
 540 // UnmarshalJSON does special handling of InstanceTypes:
 541 //
 542 // - populate computed fields (Name and Scratch)
 543 //
 544 // - error out if InstancesTypes are populated as an array, which was
 545 // deprecated in Arvados 1.2.0
 546 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 547         fixup := func(t InstanceType) (InstanceType, error) {
 548                 if t.ProviderType == "" {
 549                         t.ProviderType = t.Name
 550                 }
 551                 // If t.Scratch is set in the configuration file, it will be ignored and overwritten.
 552                 // It will also generate a "deprecated or unknown config entry" warning.
 553                 t.Scratch = t.IncludedScratch + t.AddedScratch
 554                 return t, nil
 555         }
 556
 557         if len(data) > 0 && data[0] == '[' {
 558                 return fmt.Errorf("InstanceTypes must be specified as a map, not an array, see https://doc.arvados.org/admin/config.html")
 559         }
 560         var hash map[string]InstanceType
 561         err := json.Unmarshal(data, &hash)
 562         if err != nil {
 563                 return err
 564         }
 565         // Fill in Name field (and ProviderType field, if not
 566         // specified) using hash key.
 567         *it = InstanceTypeMap(hash)
 568         for name, t := range *it {
 569                 t.Name = name
 570                 t, err := fixup(t)
 571                 if err != nil {
 572                         return err
 573                 }
 574                 (*it)[name] = t
 575         }
 576         return nil
 577 }
 578
 579 type StringSet map[string]struct{}
 580
 581 // UnmarshalJSON handles old config files that provide an array of
 582 // instance types instead of a hash.
 583 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 584         if len(data) > 0 && data[0] == '[' {
 585                 var arr []string
 586                 err := json.Unmarshal(data, &arr)
 587                 if err != nil {
 588                         return err
 589                 }
 590                 if len(arr) == 0 {
 591                         *ss = nil
 592                         return nil
 593                 }
 594                 *ss = make(map[string]struct{}, len(arr))
 595                 for _, t := range arr {
 596                         (*ss)[t] = struct{}{}
 597                 }
 598                 return nil
 599         }
 600         var hash map[string]struct{}
 601         err := json.Unmarshal(data, &hash)
 602         if err != nil {
 603                 return err
 604         }
 605         *ss = make(map[string]struct{}, len(hash))
 606         for t := range hash {
 607                 (*ss)[t] = struct{}{}
 608         }
 609
 610         return nil
 611 }
 612
 613 type ServiceName string
 614
 615 const (
 616         ServiceNameController    ServiceName = "arvados-controller"
 617         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 618         ServiceNameDispatchLSF   ServiceName = "arvados-dispatch-lsf"
 619         ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm"
 620         ServiceNameGitHTTP       ServiceName = "arvados-git-httpd"
 621         ServiceNameHealth        ServiceName = "arvados-health"
 622         ServiceNameKeepbalance   ServiceName = "keep-balance"
 623         ServiceNameKeepproxy     ServiceName = "keepproxy"
 624         ServiceNameKeepstore     ServiceName = "keepstore"
 625         ServiceNameKeepweb       ServiceName = "keep-web"
 626         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 627         ServiceNameWebsocket     ServiceName = "arvados-ws"
 628         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 629         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 630 )
 631
 632 // Map returns all services as a map, suitable for iterating over all
 633 // services or looking up a service by name.
 634 func (svcs Services) Map() map[ServiceName]Service {
 635         return map[ServiceName]Service{
 636                 ServiceNameController:    svcs.Controller,
 637                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 638                 ServiceNameDispatchLSF:   svcs.DispatchLSF,
 639                 ServiceNameDispatchSLURM: svcs.DispatchSLURM,
 640                 ServiceNameGitHTTP:       svcs.GitHTTP,
 641                 ServiceNameHealth:        svcs.Health,
 642                 ServiceNameKeepbalance:   svcs.Keepbalance,
 643                 ServiceNameKeepproxy:     svcs.Keepproxy,
 644                 ServiceNameKeepstore:     svcs.Keepstore,
 645                 ServiceNameKeepweb:       svcs.WebDAV,
 646                 ServiceNameRailsAPI:      svcs.RailsAPI,
 647                 ServiceNameWebsocket:     svcs.Websocket,
 648                 ServiceNameWorkbench1:    svcs.Workbench1,
 649                 ServiceNameWorkbench2:    svcs.Workbench2,
 650         }
 651 }