sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13         "time"
  14
  15         "git.arvados.org/arvados.git/sdk/go/config"
  16 )
  17
  18 var DefaultConfigFile = func() string {
  19         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  20                 return path
  21         }
  22         return "/etc/arvados/config.yml"
  23 }()
  24
  25 type Config struct {
  26         Clusters         map[string]Cluster
  27         AutoReloadConfig bool
  28         SourceTimestamp  time.Time
  29         SourceSHA256     string
  30 }
  31
  32 // GetConfig returns the current system config, loading it from
  33 // configFile if needed.
  34 func GetConfig(configFile string) (*Config, error) {
  35         var cfg Config
  36         err := config.LoadFile(&cfg, configFile)
  37         return &cfg, err
  38 }
  39
  40 // GetCluster returns the cluster ID and config for the given
  41 // cluster, or the default/only configured cluster if clusterID is "".
  42 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  43         if clusterID == "" {
  44                 if len(sc.Clusters) == 0 {
  45                         return nil, fmt.Errorf("no clusters configured")
  46                 } else if len(sc.Clusters) > 1 {
  47                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  48                 } else {
  49                         for id, cc := range sc.Clusters {
  50                                 cc.ClusterID = id
  51                                 return &cc, nil
  52                         }
  53                 }
  54         }
  55         cc, ok := sc.Clusters[clusterID]
  56         if !ok {
  57                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  58         }
  59         cc.ClusterID = clusterID
  60         return &cc, nil
  61 }
  62
  63 type WebDAVCacheConfig struct {
  64         TTL                Duration
  65         MaxBlockEntries    int
  66         MaxCollectionBytes int64
  67         MaxSessions        int
  68 }
  69
  70 type UploadDownloadPermission struct {
  71         Upload   bool
  72         Download bool
  73 }
  74
  75 type UploadDownloadRolePermissions struct {
  76         User  UploadDownloadPermission
  77         Admin UploadDownloadPermission
  78 }
  79
  80 type ManagedProperties map[string]struct {
  81         Value     interface{}
  82         Function  string
  83         Protected bool
  84 }
  85
  86 type Cluster struct {
  87         ClusterID       string `json:"-"`
  88         ManagementToken string
  89         SystemRootToken string
  90         Services        Services
  91         InstanceTypes   InstanceTypeMap
  92         Containers      ContainersConfig
  93         RemoteClusters  map[string]RemoteCluster
  94         PostgreSQL      PostgreSQL
  95
  96         API struct {
  97                 AsyncPermissionsUpdateInterval   Duration
  98                 DisabledAPIs                     StringSet
  99                 MaxIndexDatabaseRead             int
 100                 MaxItemsPerResponse              int
 101                 MaxConcurrentRequests            int
 102                 MaxKeepBlobBuffers               int
 103                 MaxRequestAmplification          int
 104                 MaxRequestSize                   int
 105                 MaxTokenLifetime                 Duration
 106                 RequestTimeout                   Duration
 107                 SendTimeout                      Duration
 108                 WebsocketClientEventQueue        int
 109                 WebsocketServerEventQueue        int
 110                 KeepServiceRequestTimeout        Duration
 111                 VocabularyPath                   string
 112                 FreezeProjectRequiresDescription bool
 113                 FreezeProjectRequiresProperties  StringSet
 114                 UnfreezeProjectRequiresAdmin     bool
 115         }
 116         AuditLogs struct {
 117                 MaxAge             Duration
 118                 MaxDeleteBatch     int
 119                 UnloggedAttributes StringSet
 120         }
 121         Collections struct {
 122                 BlobSigning                  bool
 123                 BlobSigningKey               string
 124                 BlobSigningTTL               Duration
 125                 BlobTrash                    bool
 126                 BlobTrashLifetime            Duration
 127                 BlobTrashCheckInterval       Duration
 128                 BlobTrashConcurrency         int
 129                 BlobDeleteConcurrency        int
 130                 BlobReplicateConcurrency     int
 131                 CollectionVersioning         bool
 132                 DefaultTrashLifetime         Duration
 133                 DefaultReplication           int
 134                 ManagedProperties            ManagedProperties
 135                 PreserveVersionIfIdle        Duration
 136                 TrashSweepInterval           Duration
 137                 TrustAllContent              bool
 138                 ForwardSlashNameSubstitution string
 139                 S3FolderObjects              bool
 140
 141                 BlobMissingReport        string
 142                 BalancePeriod            Duration
 143                 BalanceCollectionBatch   int
 144                 BalanceCollectionBuffers int
 145                 BalanceTimeout           Duration
 146                 BalanceUpdateLimit       int
 147
 148                 WebDAVCache WebDAVCacheConfig
 149
 150                 KeepproxyPermission UploadDownloadRolePermissions
 151                 WebDAVPermission    UploadDownloadRolePermissions
 152                 WebDAVLogEvents     bool
 153         }
 154         Git struct {
 155                 GitCommand   string
 156                 GitoliteHome string
 157                 Repositories string
 158         }
 159         Login struct {
 160                 LDAP struct {
 161                         Enable             bool
 162                         URL                URL
 163                         StartTLS           bool
 164                         InsecureTLS        bool
 165                         StripDomain        string
 166                         AppendDomain       string
 167                         SearchAttribute    string
 168                         SearchBindUser     string
 169                         SearchBindPassword string
 170                         SearchBase         string
 171                         SearchFilters      string
 172                         EmailAttribute     string
 173                         UsernameAttribute  string
 174                 }
 175                 Google struct {
 176                         Enable                          bool
 177                         ClientID                        string
 178                         ClientSecret                    string
 179                         AlternateEmailAddresses         bool
 180                         AuthenticationRequestParameters map[string]string
 181                 }
 182                 OpenIDConnect struct {
 183                         Enable                          bool
 184                         Issuer                          string
 185                         ClientID                        string
 186                         ClientSecret                    string
 187                         EmailClaim                      string
 188                         EmailVerifiedClaim              string
 189                         UsernameClaim                   string
 190                         AcceptAccessToken               bool
 191                         AcceptAccessTokenScope          string
 192                         AuthenticationRequestParameters map[string]string
 193                 }
 194                 PAM struct {
 195                         Enable             bool
 196                         Service            string
 197                         DefaultEmailDomain string
 198                 }
 199                 Test struct {
 200                         Enable bool
 201                         Users  map[string]TestUser
 202                 }
 203                 LoginCluster       string
 204                 RemoteTokenRefresh Duration
 205                 TokenLifetime      Duration
 206                 TrustedClients     map[string]struct{}
 207                 IssueTrustedTokens bool
 208         }
 209         Mail struct {
 210                 MailchimpAPIKey                string
 211                 MailchimpListID                string
 212                 SendUserSetupNotificationEmail bool
 213                 IssueReporterEmailFrom         string
 214                 IssueReporterEmailTo           string
 215                 SupportEmailAddress            string
 216                 EmailFrom                      string
 217         }
 218         SystemLogs struct {
 219                 LogLevel                string
 220                 Format                  string
 221                 MaxRequestLogParamsSize int
 222         }
 223         TLS struct {
 224                 Certificate string
 225                 Key         string
 226                 Insecure    bool
 227                 ACME        struct {
 228                         Server string
 229                 }
 230         }
 231         Users struct {
 232                 ActivatedUsersAreVisibleToOthers      bool
 233                 AnonymousUserToken                    string
 234                 AdminNotifierEmailFrom                string
 235                 AutoAdminFirstUser                    bool
 236                 AutoAdminUserWithEmail                string
 237                 AutoSetupNewUsers                     bool
 238                 AutoSetupNewUsersWithRepository       bool
 239                 AutoSetupNewUsersWithVmUUID           string
 240                 AutoSetupUsernameBlacklist            StringSet
 241                 EmailSubjectPrefix                    string
 242                 NewInactiveUserNotificationRecipients StringSet
 243                 NewUserNotificationRecipients         StringSet
 244                 NewUsersAreActive                     bool
 245                 UserNotifierEmailFrom                 string
 246                 UserNotifierEmailBcc                  StringSet
 247                 UserProfileNotificationAddress        string
 248                 PreferDomainForUsername               string
 249                 UserSetupMailText                     string
 250                 RoleGroupsVisibleToAll                bool
 251                 ActivityLoggingPeriod                 Duration
 252         }
 253         StorageClasses map[string]StorageClassConfig
 254         Volumes        map[string]Volume
 255         Workbench      struct {
 256                 ActivationContactLink            string
 257                 APIClientConnectTimeout          Duration
 258                 APIClientReceiveTimeout          Duration
 259                 APIResponseCompression           bool
 260                 ApplicationMimetypesWithViewIcon StringSet
 261                 ArvadosDocsite                   string
 262                 ArvadosPublicDataDocURL          string
 263                 DefaultOpenIdPrefix              string
 264                 DisableSharingURLsUI             bool
 265                 EnableGettingStartedPopup        bool
 266                 EnablePublicProjectsPage         bool
 267                 FileViewersConfigURL             string
 268                 LogViewerMaxBytes                ByteSize
 269                 MultiSiteSearch                  string
 270                 ProfilingEnabled                 bool
 271                 Repositories                     bool
 272                 RepositoryCache                  string
 273                 RunningJobLogRecordsToFetch      int
 274                 SecretKeyBase                    string
 275                 ShowRecentCollectionsOnDashboard bool
 276                 ShowUserAgreementInline          bool
 277                 ShowUserNotifications            bool
 278                 SiteName                         string
 279                 Theme                            string
 280                 UserProfileFormFields            map[string]struct {
 281                         Type                 string
 282                         FormFieldTitle       string
 283                         FormFieldDescription string
 284                         Required             bool
 285                         Position             int
 286                         Options              map[string]struct{}
 287                 }
 288                 UserProfileFormMessage string
 289                 WelcomePageHTML        string
 290                 InactivePageHTML       string
 291                 SSHHelpPageHTML        string
 292                 SSHHelpHostSuffix      string
 293                 IdleTimeout            Duration
 294         }
 295 }
 296
 297 type StorageClassConfig struct {
 298         Default  bool
 299         Priority int
 300 }
 301
 302 type Volume struct {
 303         AccessViaHosts   map[URL]VolumeAccess
 304         ReadOnly         bool
 305         Replication      int
 306         StorageClasses   map[string]bool
 307         Driver           string
 308         DriverParameters json.RawMessage
 309 }
 310
 311 type S3VolumeDriverParameters struct {
 312         IAMRole            string
 313         AccessKeyID        string
 314         SecretAccessKey    string
 315         Endpoint           string
 316         Region             string
 317         Bucket             string
 318         LocationConstraint bool
 319         V2Signature        bool
 320         UseAWSS3v2Driver   bool
 321         IndexPageSize      int
 322         ConnectTimeout     Duration
 323         ReadTimeout        Duration
 324         RaceWindow         Duration
 325         UnsafeDelete       bool
 326         PrefixLength       int
 327 }
 328
 329 type AzureVolumeDriverParameters struct {
 330         StorageAccountName   string
 331         StorageAccountKey    string
 332         StorageBaseURL       string
 333         ContainerName        string
 334         RequestTimeout       Duration
 335         ListBlobsRetryDelay  Duration
 336         ListBlobsMaxAttempts int
 337 }
 338
 339 type DirectoryVolumeDriverParameters struct {
 340         Root      string
 341         Serialize bool
 342 }
 343
 344 type VolumeAccess struct {
 345         ReadOnly bool
 346 }
 347
 348 type Services struct {
 349         Composer       Service
 350         Controller     Service
 351         DispatchCloud  Service
 352         DispatchLSF    Service
 353         DispatchSLURM  Service
 354         GitHTTP        Service
 355         GitSSH         Service
 356         Health         Service
 357         Keepbalance    Service
 358         Keepproxy      Service
 359         Keepstore      Service
 360         RailsAPI       Service
 361         WebDAVDownload Service
 362         WebDAV         Service
 363         WebShell       Service
 364         Websocket      Service
 365         Workbench1     Service
 366         Workbench2     Service
 367 }
 368
 369 type Service struct {
 370         InternalURLs map[URL]ServiceInstance
 371         ExternalURL  URL
 372 }
 373
 374 type TestUser struct {
 375         Email    string
 376         Password string
 377 }
 378
 379 // URL is a url.URL that is also usable as a JSON key/value.
 380 type URL url.URL
 381
 382 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 383 // used as a JSON key/value.
 384 func (su *URL) UnmarshalText(text []byte) error {
 385         u, err := url.Parse(string(text))
 386         if err == nil {
 387                 *su = URL(*u)
 388                 if su.Path == "" && su.Host != "" {
 389                         // http://example really means http://example/
 390                         su.Path = "/"
 391                 }
 392         }
 393         return err
 394 }
 395
 396 func (su URL) MarshalText() ([]byte, error) {
 397         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 398 }
 399
 400 func (su URL) String() string {
 401         return (*url.URL)(&su).String()
 402 }
 403
 404 type ServiceInstance struct {
 405         ListenURL  URL
 406         Rendezvous string `json:",omitempty"`
 407 }
 408
 409 type PostgreSQL struct {
 410         Connection     PostgreSQLConnection
 411         ConnectionPool int
 412 }
 413
 414 type PostgreSQLConnection map[string]string
 415
 416 type RemoteCluster struct {
 417         Host          string
 418         Proxy         bool
 419         Scheme        string
 420         Insecure      bool
 421         ActivateUsers bool
 422 }
 423
 424 type CUDAFeatures struct {
 425         DriverVersion      string
 426         HardwareCapability string
 427         DeviceCount        int
 428 }
 429
 430 type InstanceType struct {
 431         Name            string `json:"-"`
 432         ProviderType    string
 433         VCPUs           int
 434         RAM             ByteSize
 435         Scratch         ByteSize `json:"-"`
 436         IncludedScratch ByteSize
 437         AddedScratch    ByteSize
 438         Price           float64
 439         Preemptible     bool
 440         CUDA            CUDAFeatures
 441 }
 442
 443 type ContainersConfig struct {
 444         CloudVMs                      CloudVMsConfig
 445         CrunchRunCommand              string
 446         CrunchRunArgumentsList        []string
 447         DefaultKeepCacheRAM           ByteSize
 448         DispatchPrivateKey            string
 449         LogReuseDecisions             bool
 450         MaxComputeVMs                 int
 451         MaxDispatchAttempts           int
 452         MaxRetryAttempts              int
 453         MinRetryPeriod                Duration
 454         ReserveExtraRAM               ByteSize
 455         StaleLockTimeout              Duration
 456         SupportedDockerImageFormats   StringSet
 457         AlwaysUsePreemptibleInstances bool
 458         PreemptiblePriceFactor        float64
 459         RuntimeEngine                 string
 460         LocalKeepBlobBuffersPerVCPU   int
 461         LocalKeepLogsToContainerLog   string
 462
 463         JobsAPI struct {
 464                 Enable         string
 465                 GitInternalDir string
 466         }
 467         Logging struct {
 468                 MaxAge                       Duration
 469                 LogBytesPerEvent             int
 470                 LogSecondsBetweenEvents      Duration
 471                 LogThrottlePeriod            Duration
 472                 LogThrottleBytes             int
 473                 LogThrottleLines             int
 474                 LimitLogBytesPerJob          int
 475                 LogPartialLineThrottlePeriod Duration
 476                 LogUpdatePeriod              Duration
 477                 LogUpdateSize                ByteSize
 478         }
 479         ShellAccess struct {
 480                 Admin bool
 481                 User  bool
 482         }
 483         SLURM struct {
 484                 PrioritySpread             int64
 485                 SbatchArgumentsList        []string
 486                 SbatchEnvironmentVariables map[string]string
 487                 Managed                    struct {
 488                         DNSServerConfDir       string
 489                         DNSServerConfTemplate  string
 490                         DNSServerReloadCommand string
 491                         DNSServerUpdateCommand string
 492                         ComputeNodeDomain      string
 493                         ComputeNodeNameservers StringSet
 494                         AssignNodeHostname     string
 495                 }
 496         }
 497         LSF struct {
 498                 BsubSudoUser      string
 499                 BsubArgumentsList []string
 500                 BsubCUDAArguments []string
 501         }
 502 }
 503
 504 type CloudVMsConfig struct {
 505         Enable bool
 506
 507         BootProbeCommand               string
 508         DeployRunnerBinary             string
 509         ImageID                        string
 510         MaxCloudOpsPerSecond           int
 511         MaxProbesPerSecond             int
 512         MaxConcurrentInstanceCreateOps int
 513         PollInterval                   Duration
 514         ProbeInterval                  Duration
 515         SSHPort                        string
 516         SyncInterval                   Duration
 517         TimeoutBooting                 Duration
 518         TimeoutIdle                    Duration
 519         TimeoutProbe                   Duration
 520         TimeoutShutdown                Duration
 521         TimeoutSignal                  Duration
 522         TimeoutStaleRunLock            Duration
 523         TimeoutTERM                    Duration
 524         ResourceTags                   map[string]string
 525         TagKeyPrefix                   string
 526
 527         Driver           string
 528         DriverParameters json.RawMessage
 529 }
 530
 531 type InstanceTypeMap map[string]InstanceType
 532
 533 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 534
 535 // UnmarshalJSON does special handling of InstanceTypes:
 536 // * populate computed fields (Name and Scratch)
 537 // * error out if InstancesTypes are populated as an array, which was
 538 //   deprecated in Arvados 1.2.0
 539 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 540         fixup := func(t InstanceType) (InstanceType, error) {
 541                 if t.ProviderType == "" {
 542                         t.ProviderType = t.Name
 543                 }
 544                 // If t.Scratch is set in the configuration file, it will be ignored and overwritten.
 545                 // It will also generate a "deprecated or unknown config entry" warning.
 546                 t.Scratch = t.IncludedScratch + t.AddedScratch
 547                 return t, nil
 548         }
 549
 550         if len(data) > 0 && data[0] == '[' {
 551                 return fmt.Errorf("InstanceTypes must be specified as a map, not an array, see https://doc.arvados.org/admin/config.html")
 552         }
 553         var hash map[string]InstanceType
 554         err := json.Unmarshal(data, &hash)
 555         if err != nil {
 556                 return err
 557         }
 558         // Fill in Name field (and ProviderType field, if not
 559         // specified) using hash key.
 560         *it = InstanceTypeMap(hash)
 561         for name, t := range *it {
 562                 t.Name = name
 563                 t, err := fixup(t)
 564                 if err != nil {
 565                         return err
 566                 }
 567                 (*it)[name] = t
 568         }
 569         return nil
 570 }
 571
 572 type StringSet map[string]struct{}
 573
 574 // UnmarshalJSON handles old config files that provide an array of
 575 // instance types instead of a hash.
 576 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 577         if len(data) > 0 && data[0] == '[' {
 578                 var arr []string
 579                 err := json.Unmarshal(data, &arr)
 580                 if err != nil {
 581                         return err
 582                 }
 583                 if len(arr) == 0 {
 584                         *ss = nil
 585                         return nil
 586                 }
 587                 *ss = make(map[string]struct{}, len(arr))
 588                 for _, t := range arr {
 589                         (*ss)[t] = struct{}{}
 590                 }
 591                 return nil
 592         }
 593         var hash map[string]struct{}
 594         err := json.Unmarshal(data, &hash)
 595         if err != nil {
 596                 return err
 597         }
 598         *ss = make(map[string]struct{}, len(hash))
 599         for t := range hash {
 600                 (*ss)[t] = struct{}{}
 601         }
 602
 603         return nil
 604 }
 605
 606 type ServiceName string
 607
 608 const (
 609         ServiceNameController    ServiceName = "arvados-controller"
 610         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 611         ServiceNameDispatchLSF   ServiceName = "arvados-dispatch-lsf"
 612         ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm"
 613         ServiceNameGitHTTP       ServiceName = "arvados-git-httpd"
 614         ServiceNameHealth        ServiceName = "arvados-health"
 615         ServiceNameKeepbalance   ServiceName = "keep-balance"
 616         ServiceNameKeepproxy     ServiceName = "keepproxy"
 617         ServiceNameKeepstore     ServiceName = "keepstore"
 618         ServiceNameKeepweb       ServiceName = "keep-web"
 619         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 620         ServiceNameWebsocket     ServiceName = "arvados-ws"
 621         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 622         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 623 )
 624
 625 // Map returns all services as a map, suitable for iterating over all
 626 // services or looking up a service by name.
 627 func (svcs Services) Map() map[ServiceName]Service {
 628         return map[ServiceName]Service{
 629                 ServiceNameController:    svcs.Controller,
 630                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 631                 ServiceNameDispatchLSF:   svcs.DispatchLSF,
 632                 ServiceNameDispatchSLURM: svcs.DispatchSLURM,
 633                 ServiceNameGitHTTP:       svcs.GitHTTP,
 634                 ServiceNameHealth:        svcs.Health,
 635                 ServiceNameKeepbalance:   svcs.Keepbalance,
 636                 ServiceNameKeepproxy:     svcs.Keepproxy,
 637                 ServiceNameKeepstore:     svcs.Keepstore,
 638                 ServiceNameKeepweb:       svcs.WebDAV,
 639                 ServiceNameRailsAPI:      svcs.RailsAPI,
 640                 ServiceNameWebsocket:     svcs.Websocket,
 641                 ServiceNameWorkbench1:    svcs.Workbench1,
 642                 ServiceNameWorkbench2:    svcs.Workbench2,
 643         }
 644 }