sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13         "time"
  14
  15         "git.arvados.org/arvados.git/sdk/go/config"
  16 )
  17
  18 var DefaultConfigFile = func() string {
  19         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  20                 return path
  21         }
  22         return "/etc/arvados/config.yml"
  23 }()
  24
  25 type Config struct {
  26         Clusters         map[string]Cluster
  27         AutoReloadConfig bool
  28         SourceTimestamp  time.Time
  29         SourceSHA256     string
  30 }
  31
  32 // GetConfig returns the current system config, loading it from
  33 // configFile if needed.
  34 func GetConfig(configFile string) (*Config, error) {
  35         var cfg Config
  36         err := config.LoadFile(&cfg, configFile)
  37         return &cfg, err
  38 }
  39
  40 // GetCluster returns the cluster ID and config for the given
  41 // cluster, or the default/only configured cluster if clusterID is "".
  42 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  43         if clusterID == "" {
  44                 if len(sc.Clusters) == 0 {
  45                         return nil, fmt.Errorf("no clusters configured")
  46                 } else if len(sc.Clusters) > 1 {
  47                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  48                 } else {
  49                         for id, cc := range sc.Clusters {
  50                                 cc.ClusterID = id
  51                                 return &cc, nil
  52                         }
  53                 }
  54         }
  55         cc, ok := sc.Clusters[clusterID]
  56         if !ok {
  57                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  58         }
  59         cc.ClusterID = clusterID
  60         return &cc, nil
  61 }
  62
  63 type WebDAVCacheConfig struct {
  64         TTL                  Duration
  65         UUIDTTL              Duration
  66         MaxBlockEntries      int
  67         MaxCollectionEntries int
  68         MaxCollectionBytes   int64
  69         MaxUUIDEntries       int
  70         MaxSessions          int
  71 }
  72
  73 type UploadDownloadPermission struct {
  74         Upload   bool
  75         Download bool
  76 }
  77
  78 type UploadDownloadRolePermissions struct {
  79         User  UploadDownloadPermission
  80         Admin UploadDownloadPermission
  81 }
  82
  83 type ManagedProperties map[string]struct {
  84         Value     interface{}
  85         Function  string
  86         Protected bool
  87 }
  88
  89 type Cluster struct {
  90         ClusterID       string `json:"-"`
  91         ManagementToken string
  92         SystemRootToken string
  93         Services        Services
  94         InstanceTypes   InstanceTypeMap
  95         Containers      ContainersConfig
  96         RemoteClusters  map[string]RemoteCluster
  97         PostgreSQL      PostgreSQL
  98
  99         API struct {
 100                 AsyncPermissionsUpdateInterval   Duration
 101                 DisabledAPIs                     StringSet
 102                 MaxIndexDatabaseRead             int
 103                 MaxItemsPerResponse              int
 104                 MaxConcurrentRequests            int
 105                 MaxKeepBlobBuffers               int
 106                 MaxRequestAmplification          int
 107                 MaxRequestSize                   int
 108                 MaxTokenLifetime                 Duration
 109                 RequestTimeout                   Duration
 110                 SendTimeout                      Duration
 111                 WebsocketClientEventQueue        int
 112                 WebsocketServerEventQueue        int
 113                 KeepServiceRequestTimeout        Duration
 114                 VocabularyPath                   string
 115                 FreezeProjectRequiresDescription bool
 116                 FreezeProjectRequiresProperties  StringSet
 117                 UnfreezeProjectRequiresAdmin     bool
 118         }
 119         AuditLogs struct {
 120                 MaxAge             Duration
 121                 MaxDeleteBatch     int
 122                 UnloggedAttributes StringSet
 123         }
 124         Collections struct {
 125                 BlobSigning                  bool
 126                 BlobSigningKey               string
 127                 BlobSigningTTL               Duration
 128                 BlobTrash                    bool
 129                 BlobTrashLifetime            Duration
 130                 BlobTrashCheckInterval       Duration
 131                 BlobTrashConcurrency         int
 132                 BlobDeleteConcurrency        int
 133                 BlobReplicateConcurrency     int
 134                 CollectionVersioning         bool
 135                 DefaultTrashLifetime         Duration
 136                 DefaultReplication           int
 137                 ManagedProperties            ManagedProperties
 138                 PreserveVersionIfIdle        Duration
 139                 TrashSweepInterval           Duration
 140                 TrustAllContent              bool
 141                 ForwardSlashNameSubstitution string
 142                 S3FolderObjects              bool
 143
 144                 BlobMissingReport        string
 145                 BalancePeriod            Duration
 146                 BalanceCollectionBatch   int
 147                 BalanceCollectionBuffers int
 148                 BalanceTimeout           Duration
 149                 BalanceUpdateLimit       int
 150
 151                 WebDAVCache WebDAVCacheConfig
 152
 153                 KeepproxyPermission UploadDownloadRolePermissions
 154                 WebDAVPermission    UploadDownloadRolePermissions
 155                 WebDAVLogEvents     bool
 156         }
 157         Git struct {
 158                 GitCommand   string
 159                 GitoliteHome string
 160                 Repositories string
 161         }
 162         Login struct {
 163                 LDAP struct {
 164                         Enable             bool
 165                         URL                URL
 166                         StartTLS           bool
 167                         InsecureTLS        bool
 168                         StripDomain        string
 169                         AppendDomain       string
 170                         SearchAttribute    string
 171                         SearchBindUser     string
 172                         SearchBindPassword string
 173                         SearchBase         string
 174                         SearchFilters      string
 175                         EmailAttribute     string
 176                         UsernameAttribute  string
 177                 }
 178                 Google struct {
 179                         Enable                          bool
 180                         ClientID                        string
 181                         ClientSecret                    string
 182                         AlternateEmailAddresses         bool
 183                         AuthenticationRequestParameters map[string]string
 184                 }
 185                 OpenIDConnect struct {
 186                         Enable                          bool
 187                         Issuer                          string
 188                         ClientID                        string
 189                         ClientSecret                    string
 190                         EmailClaim                      string
 191                         EmailVerifiedClaim              string
 192                         UsernameClaim                   string
 193                         AcceptAccessToken               bool
 194                         AcceptAccessTokenScope          string
 195                         AuthenticationRequestParameters map[string]string
 196                 }
 197                 PAM struct {
 198                         Enable             bool
 199                         Service            string
 200                         DefaultEmailDomain string
 201                 }
 202                 Test struct {
 203                         Enable bool
 204                         Users  map[string]TestUser
 205                 }
 206                 LoginCluster       string
 207                 RemoteTokenRefresh Duration
 208                 TokenLifetime      Duration
 209                 TrustedClients     map[string]struct{}
 210                 IssueTrustedTokens bool
 211         }
 212         Mail struct {
 213                 MailchimpAPIKey                string
 214                 MailchimpListID                string
 215                 SendUserSetupNotificationEmail bool
 216                 IssueReporterEmailFrom         string
 217                 IssueReporterEmailTo           string
 218                 SupportEmailAddress            string
 219                 EmailFrom                      string
 220         }
 221         SystemLogs struct {
 222                 LogLevel                string
 223                 Format                  string
 224                 MaxRequestLogParamsSize int
 225         }
 226         TLS struct {
 227                 Certificate string
 228                 Key         string
 229                 Insecure    bool
 230                 Automatic   bool
 231                 Staging     bool
 232         }
 233         Users struct {
 234                 ActivatedUsersAreVisibleToOthers      bool
 235                 AnonymousUserToken                    string
 236                 AdminNotifierEmailFrom                string
 237                 AutoAdminFirstUser                    bool
 238                 AutoAdminUserWithEmail                string
 239                 AutoSetupNewUsers                     bool
 240                 AutoSetupNewUsersWithRepository       bool
 241                 AutoSetupNewUsersWithVmUUID           string
 242                 AutoSetupUsernameBlacklist            StringSet
 243                 EmailSubjectPrefix                    string
 244                 NewInactiveUserNotificationRecipients StringSet
 245                 NewUserNotificationRecipients         StringSet
 246                 NewUsersAreActive                     bool
 247                 UserNotifierEmailFrom                 string
 248                 UserNotifierEmailBcc                  StringSet
 249                 UserProfileNotificationAddress        string
 250                 PreferDomainForUsername               string
 251                 UserSetupMailText                     string
 252                 RoleGroupsVisibleToAll                bool
 253         }
 254         StorageClasses map[string]StorageClassConfig
 255         Volumes        map[string]Volume
 256         Workbench      struct {
 257                 ActivationContactLink            string
 258                 APIClientConnectTimeout          Duration
 259                 APIClientReceiveTimeout          Duration
 260                 APIResponseCompression           bool
 261                 ApplicationMimetypesWithViewIcon StringSet
 262                 ArvadosDocsite                   string
 263                 ArvadosPublicDataDocURL          string
 264                 DefaultOpenIdPrefix              string
 265                 DisableSharingURLsUI             bool
 266                 EnableGettingStartedPopup        bool
 267                 EnablePublicProjectsPage         bool
 268                 FileViewersConfigURL             string
 269                 LogViewerMaxBytes                ByteSize
 270                 MultiSiteSearch                  string
 271                 ProfilingEnabled                 bool
 272                 Repositories                     bool
 273                 RepositoryCache                  string
 274                 RunningJobLogRecordsToFetch      int
 275                 SecretKeyBase                    string
 276                 ShowRecentCollectionsOnDashboard bool
 277                 ShowUserAgreementInline          bool
 278                 ShowUserNotifications            bool
 279                 SiteName                         string
 280                 Theme                            string
 281                 UserProfileFormFields            map[string]struct {
 282                         Type                 string
 283                         FormFieldTitle       string
 284                         FormFieldDescription string
 285                         Required             bool
 286                         Position             int
 287                         Options              map[string]struct{}
 288                 }
 289                 UserProfileFormMessage string
 290                 WelcomePageHTML        string
 291                 InactivePageHTML       string
 292                 SSHHelpPageHTML        string
 293                 SSHHelpHostSuffix      string
 294                 IdleTimeout            Duration
 295         }
 296 }
 297
 298 type StorageClassConfig struct {
 299         Default  bool
 300         Priority int
 301 }
 302
 303 type Volume struct {
 304         AccessViaHosts   map[URL]VolumeAccess
 305         ReadOnly         bool
 306         Replication      int
 307         StorageClasses   map[string]bool
 308         Driver           string
 309         DriverParameters json.RawMessage
 310 }
 311
 312 type S3VolumeDriverParameters struct {
 313         IAMRole            string
 314         AccessKeyID        string
 315         SecretAccessKey    string
 316         Endpoint           string
 317         Region             string
 318         Bucket             string
 319         LocationConstraint bool
 320         V2Signature        bool
 321         UseAWSS3v2Driver   bool
 322         IndexPageSize      int
 323         ConnectTimeout     Duration
 324         ReadTimeout        Duration
 325         RaceWindow         Duration
 326         UnsafeDelete       bool
 327         PrefixLength       int
 328 }
 329
 330 type AzureVolumeDriverParameters struct {
 331         StorageAccountName   string
 332         StorageAccountKey    string
 333         StorageBaseURL       string
 334         ContainerName        string
 335         RequestTimeout       Duration
 336         ListBlobsRetryDelay  Duration
 337         ListBlobsMaxAttempts int
 338 }
 339
 340 type DirectoryVolumeDriverParameters struct {
 341         Root      string
 342         Serialize bool
 343 }
 344
 345 type VolumeAccess struct {
 346         ReadOnly bool
 347 }
 348
 349 type Services struct {
 350         Composer       Service
 351         Controller     Service
 352         DispatchCloud  Service
 353         DispatchLSF    Service
 354         DispatchSLURM  Service
 355         GitHTTP        Service
 356         GitSSH         Service
 357         Health         Service
 358         Keepbalance    Service
 359         Keepproxy      Service
 360         Keepstore      Service
 361         RailsAPI       Service
 362         WebDAVDownload Service
 363         WebDAV         Service
 364         WebShell       Service
 365         Websocket      Service
 366         Workbench1     Service
 367         Workbench2     Service
 368 }
 369
 370 type Service struct {
 371         InternalURLs map[URL]ServiceInstance
 372         ExternalURL  URL
 373 }
 374
 375 type TestUser struct {
 376         Email    string
 377         Password string
 378 }
 379
 380 // URL is a url.URL that is also usable as a JSON key/value.
 381 type URL url.URL
 382
 383 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 384 // used as a JSON key/value.
 385 func (su *URL) UnmarshalText(text []byte) error {
 386         u, err := url.Parse(string(text))
 387         if err == nil {
 388                 *su = URL(*u)
 389                 if su.Path == "" && su.Host != "" {
 390                         // http://example really means http://example/
 391                         su.Path = "/"
 392                 }
 393         }
 394         return err
 395 }
 396
 397 func (su URL) MarshalText() ([]byte, error) {
 398         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 399 }
 400
 401 func (su URL) String() string {
 402         return (*url.URL)(&su).String()
 403 }
 404
 405 type ServiceInstance struct {
 406         ListenURL  URL
 407         Rendezvous string `json:",omitempty"`
 408 }
 409
 410 type PostgreSQL struct {
 411         Connection     PostgreSQLConnection
 412         ConnectionPool int
 413 }
 414
 415 type PostgreSQLConnection map[string]string
 416
 417 type RemoteCluster struct {
 418         Host          string
 419         Proxy         bool
 420         Scheme        string
 421         Insecure      bool
 422         ActivateUsers bool
 423 }
 424
 425 type CUDAFeatures struct {
 426         DriverVersion      string
 427         HardwareCapability string
 428         DeviceCount        int
 429 }
 430
 431 type InstanceType struct {
 432         Name            string `json:"-"`
 433         ProviderType    string
 434         VCPUs           int
 435         RAM             ByteSize
 436         Scratch         ByteSize `json:"-"`
 437         IncludedScratch ByteSize
 438         AddedScratch    ByteSize
 439         Price           float64
 440         Preemptible     bool
 441         CUDA            CUDAFeatures
 442 }
 443
 444 type ContainersConfig struct {
 445         CloudVMs                      CloudVMsConfig
 446         CrunchRunCommand              string
 447         CrunchRunArgumentsList        []string
 448         DefaultKeepCacheRAM           ByteSize
 449         DispatchPrivateKey            string
 450         LogReuseDecisions             bool
 451         MaxComputeVMs                 int
 452         MaxDispatchAttempts           int
 453         MaxRetryAttempts              int
 454         MinRetryPeriod                Duration
 455         ReserveExtraRAM               ByteSize
 456         StaleLockTimeout              Duration
 457         SupportedDockerImageFormats   StringSet
 458         AlwaysUsePreemptibleInstances bool
 459         PreemptiblePriceFactor        float64
 460         RuntimeEngine                 string
 461         LocalKeepBlobBuffersPerVCPU   int
 462         LocalKeepLogsToContainerLog   string
 463
 464         JobsAPI struct {
 465                 Enable         string
 466                 GitInternalDir string
 467         }
 468         Logging struct {
 469                 MaxAge                       Duration
 470                 LogBytesPerEvent             int
 471                 LogSecondsBetweenEvents      Duration
 472                 LogThrottlePeriod            Duration
 473                 LogThrottleBytes             int
 474                 LogThrottleLines             int
 475                 LimitLogBytesPerJob          int
 476                 LogPartialLineThrottlePeriod Duration
 477                 LogUpdatePeriod              Duration
 478                 LogUpdateSize                ByteSize
 479         }
 480         ShellAccess struct {
 481                 Admin bool
 482                 User  bool
 483         }
 484         SLURM struct {
 485                 PrioritySpread             int64
 486                 SbatchArgumentsList        []string
 487                 SbatchEnvironmentVariables map[string]string
 488                 Managed                    struct {
 489                         DNSServerConfDir       string
 490                         DNSServerConfTemplate  string
 491                         DNSServerReloadCommand string
 492                         DNSServerUpdateCommand string
 493                         ComputeNodeDomain      string
 494                         ComputeNodeNameservers StringSet
 495                         AssignNodeHostname     string
 496                 }
 497         }
 498         LSF struct {
 499                 BsubSudoUser      string
 500                 BsubArgumentsList []string
 501                 BsubCUDAArguments []string
 502         }
 503 }
 504
 505 type CloudVMsConfig struct {
 506         Enable bool
 507
 508         BootProbeCommand               string
 509         DeployRunnerBinary             string
 510         ImageID                        string
 511         MaxCloudOpsPerSecond           int
 512         MaxProbesPerSecond             int
 513         MaxConcurrentInstanceCreateOps int
 514         PollInterval                   Duration
 515         ProbeInterval                  Duration
 516         SSHPort                        string
 517         SyncInterval                   Duration
 518         TimeoutBooting                 Duration
 519         TimeoutIdle                    Duration
 520         TimeoutProbe                   Duration
 521         TimeoutShutdown                Duration
 522         TimeoutSignal                  Duration
 523         TimeoutStaleRunLock            Duration
 524         TimeoutTERM                    Duration
 525         ResourceTags                   map[string]string
 526         TagKeyPrefix                   string
 527
 528         Driver           string
 529         DriverParameters json.RawMessage
 530 }
 531
 532 type InstanceTypeMap map[string]InstanceType
 533
 534 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 535
 536 // UnmarshalJSON does special handling of InstanceTypes:
 537 // * populate computed fields (Name and Scratch)
 538 // * error out if InstancesTypes are populated as an array, which was
 539 //   deprecated in Arvados 1.2.0
 540 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 541         fixup := func(t InstanceType) (InstanceType, error) {
 542                 if t.ProviderType == "" {
 543                         t.ProviderType = t.Name
 544                 }
 545                 // If t.Scratch is set in the configuration file, it will be ignored and overwritten.
 546                 // It will also generate a "deprecated or unknown config entry" warning.
 547                 t.Scratch = t.IncludedScratch + t.AddedScratch
 548                 return t, nil
 549         }
 550
 551         if len(data) > 0 && data[0] == '[' {
 552                 return fmt.Errorf("InstanceTypes must be specified as a map, not an array, see https://doc.arvados.org/admin/config.html")
 553         }
 554         var hash map[string]InstanceType
 555         err := json.Unmarshal(data, &hash)
 556         if err != nil {
 557                 return err
 558         }
 559         // Fill in Name field (and ProviderType field, if not
 560         // specified) using hash key.
 561         *it = InstanceTypeMap(hash)
 562         for name, t := range *it {
 563                 t.Name = name
 564                 t, err := fixup(t)
 565                 if err != nil {
 566                         return err
 567                 }
 568                 (*it)[name] = t
 569         }
 570         return nil
 571 }
 572
 573 type StringSet map[string]struct{}
 574
 575 // UnmarshalJSON handles old config files that provide an array of
 576 // instance types instead of a hash.
 577 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 578         if len(data) > 0 && data[0] == '[' {
 579                 var arr []string
 580                 err := json.Unmarshal(data, &arr)
 581                 if err != nil {
 582                         return err
 583                 }
 584                 if len(arr) == 0 {
 585                         *ss = nil
 586                         return nil
 587                 }
 588                 *ss = make(map[string]struct{}, len(arr))
 589                 for _, t := range arr {
 590                         (*ss)[t] = struct{}{}
 591                 }
 592                 return nil
 593         }
 594         var hash map[string]struct{}
 595         err := json.Unmarshal(data, &hash)
 596         if err != nil {
 597                 return err
 598         }
 599         *ss = make(map[string]struct{}, len(hash))
 600         for t := range hash {
 601                 (*ss)[t] = struct{}{}
 602         }
 603
 604         return nil
 605 }
 606
 607 type ServiceName string
 608
 609 const (
 610         ServiceNameController    ServiceName = "arvados-controller"
 611         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 612         ServiceNameDispatchLSF   ServiceName = "arvados-dispatch-lsf"
 613         ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm"
 614         ServiceNameGitHTTP       ServiceName = "arvados-git-httpd"
 615         ServiceNameHealth        ServiceName = "arvados-health"
 616         ServiceNameKeepbalance   ServiceName = "keep-balance"
 617         ServiceNameKeepproxy     ServiceName = "keepproxy"
 618         ServiceNameKeepstore     ServiceName = "keepstore"
 619         ServiceNameKeepweb       ServiceName = "keep-web"
 620         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 621         ServiceNameWebsocket     ServiceName = "arvados-ws"
 622         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 623         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 624 )
 625
 626 // Map returns all services as a map, suitable for iterating over all
 627 // services or looking up a service by name.
 628 func (svcs Services) Map() map[ServiceName]Service {
 629         return map[ServiceName]Service{
 630                 ServiceNameController:    svcs.Controller,
 631                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 632                 ServiceNameDispatchLSF:   svcs.DispatchLSF,
 633                 ServiceNameDispatchSLURM: svcs.DispatchSLURM,
 634                 ServiceNameGitHTTP:       svcs.GitHTTP,
 635                 ServiceNameHealth:        svcs.Health,
 636                 ServiceNameKeepbalance:   svcs.Keepbalance,
 637                 ServiceNameKeepproxy:     svcs.Keepproxy,
 638                 ServiceNameKeepstore:     svcs.Keepstore,
 639                 ServiceNameKeepweb:       svcs.WebDAV,
 640                 ServiceNameRailsAPI:      svcs.RailsAPI,
 641                 ServiceNameWebsocket:     svcs.Websocket,
 642                 ServiceNameWorkbench1:    svcs.Workbench1,
 643                 ServiceNameWorkbench2:    svcs.Workbench2,
 644         }
 645 }