sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13         "time"
  14
  15         "git.arvados.org/arvados.git/sdk/go/config"
  16 )
  17
  18 var DefaultConfigFile = func() string {
  19         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  20                 return path
  21         }
  22         return "/etc/arvados/config.yml"
  23 }()
  24
  25 type Config struct {
  26         Clusters         map[string]Cluster
  27         AutoReloadConfig bool
  28         SourceTimestamp  time.Time
  29         SourceSHA256     string
  30 }
  31
  32 // GetConfig returns the current system config, loading it from
  33 // configFile if needed.
  34 func GetConfig(configFile string) (*Config, error) {
  35         var cfg Config
  36         err := config.LoadFile(&cfg, configFile)
  37         return &cfg, err
  38 }
  39
  40 // GetCluster returns the cluster ID and config for the given
  41 // cluster, or the default/only configured cluster if clusterID is "".
  42 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  43         if clusterID == "" {
  44                 if len(sc.Clusters) == 0 {
  45                         return nil, fmt.Errorf("no clusters configured")
  46                 } else if len(sc.Clusters) > 1 {
  47                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  48                 } else {
  49                         for id, cc := range sc.Clusters {
  50                                 cc.ClusterID = id
  51                                 return &cc, nil
  52                         }
  53                 }
  54         }
  55         cc, ok := sc.Clusters[clusterID]
  56         if !ok {
  57                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  58         }
  59         cc.ClusterID = clusterID
  60         return &cc, nil
  61 }
  62
  63 type WebDAVCacheConfig struct {
  64         TTL                  Duration
  65         UUIDTTL              Duration
  66         MaxBlockEntries      int
  67         MaxCollectionEntries int
  68         MaxCollectionBytes   int64
  69         MaxUUIDEntries       int
  70         MaxSessions          int
  71 }
  72
  73 type UploadDownloadPermission struct {
  74         Upload   bool
  75         Download bool
  76 }
  77
  78 type UploadDownloadRolePermissions struct {
  79         User  UploadDownloadPermission
  80         Admin UploadDownloadPermission
  81 }
  82
  83 type ManagedProperties map[string]struct {
  84         Value     interface{}
  85         Function  string
  86         Protected bool
  87 }
  88
  89 type Cluster struct {
  90         ClusterID       string `json:"-"`
  91         ManagementToken string
  92         SystemRootToken string
  93         Services        Services
  94         InstanceTypes   InstanceTypeMap
  95         Containers      ContainersConfig
  96         RemoteClusters  map[string]RemoteCluster
  97         PostgreSQL      PostgreSQL
  98
  99         API struct {
 100                 AsyncPermissionsUpdateInterval   Duration
 101                 DisabledAPIs                     StringSet
 102                 MaxIndexDatabaseRead             int
 103                 MaxItemsPerResponse              int
 104                 MaxConcurrentRequests            int
 105                 MaxKeepBlobBuffers               int
 106                 MaxRequestAmplification          int
 107                 MaxRequestSize                   int
 108                 MaxTokenLifetime                 Duration
 109                 RequestTimeout                   Duration
 110                 SendTimeout                      Duration
 111                 WebsocketClientEventQueue        int
 112                 WebsocketServerEventQueue        int
 113                 KeepServiceRequestTimeout        Duration
 114                 VocabularyPath                   string
 115                 FreezeProjectRequiresDescription bool
 116                 FreezeProjectRequiresProperties  StringSet
 117                 UnfreezeProjectRequiresAdmin     bool
 118         }
 119         AuditLogs struct {
 120                 MaxAge             Duration
 121                 MaxDeleteBatch     int
 122                 UnloggedAttributes StringSet
 123         }
 124         Collections struct {
 125                 BlobSigning                  bool
 126                 BlobSigningKey               string
 127                 BlobSigningTTL               Duration
 128                 BlobTrash                    bool
 129                 BlobTrashLifetime            Duration
 130                 BlobTrashCheckInterval       Duration
 131                 BlobTrashConcurrency         int
 132                 BlobDeleteConcurrency        int
 133                 BlobReplicateConcurrency     int
 134                 CollectionVersioning         bool
 135                 DefaultTrashLifetime         Duration
 136                 DefaultReplication           int
 137                 ManagedProperties            ManagedProperties
 138                 PreserveVersionIfIdle        Duration
 139                 TrashSweepInterval           Duration
 140                 TrustAllContent              bool
 141                 ForwardSlashNameSubstitution string
 142                 S3FolderObjects              bool
 143
 144                 BlobMissingReport        string
 145                 BalancePeriod            Duration
 146                 BalanceCollectionBatch   int
 147                 BalanceCollectionBuffers int
 148                 BalanceTimeout           Duration
 149                 BalanceUpdateLimit       int
 150
 151                 WebDAVCache WebDAVCacheConfig
 152
 153                 KeepproxyPermission UploadDownloadRolePermissions
 154                 WebDAVPermission    UploadDownloadRolePermissions
 155                 WebDAVLogEvents     bool
 156         }
 157         Git struct {
 158                 GitCommand   string
 159                 GitoliteHome string
 160                 Repositories string
 161         }
 162         Login struct {
 163                 LDAP struct {
 164                         Enable             bool
 165                         URL                URL
 166                         StartTLS           bool
 167                         InsecureTLS        bool
 168                         StripDomain        string
 169                         AppendDomain       string
 170                         SearchAttribute    string
 171                         SearchBindUser     string
 172                         SearchBindPassword string
 173                         SearchBase         string
 174                         SearchFilters      string
 175                         EmailAttribute     string
 176                         UsernameAttribute  string
 177                 }
 178                 Google struct {
 179                         Enable                          bool
 180                         ClientID                        string
 181                         ClientSecret                    string
 182                         AlternateEmailAddresses         bool
 183                         AuthenticationRequestParameters map[string]string
 184                 }
 185                 OpenIDConnect struct {
 186                         Enable                          bool
 187                         Issuer                          string
 188                         ClientID                        string
 189                         ClientSecret                    string
 190                         EmailClaim                      string
 191                         EmailVerifiedClaim              string
 192                         UsernameClaim                   string
 193                         AcceptAccessToken               bool
 194                         AcceptAccessTokenScope          string
 195                         AuthenticationRequestParameters map[string]string
 196                 }
 197                 PAM struct {
 198                         Enable             bool
 199                         Service            string
 200                         DefaultEmailDomain string
 201                 }
 202                 Test struct {
 203                         Enable bool
 204                         Users  map[string]TestUser
 205                 }
 206                 LoginCluster       string
 207                 RemoteTokenRefresh Duration
 208                 TokenLifetime      Duration
 209                 TrustedClients     map[string]struct{}
 210                 IssueTrustedTokens bool
 211         }
 212         Mail struct {
 213                 MailchimpAPIKey                string
 214                 MailchimpListID                string
 215                 SendUserSetupNotificationEmail bool
 216                 IssueReporterEmailFrom         string
 217                 IssueReporterEmailTo           string
 218                 SupportEmailAddress            string
 219                 EmailFrom                      string
 220         }
 221         SystemLogs struct {
 222                 LogLevel                string
 223                 Format                  string
 224                 MaxRequestLogParamsSize int
 225         }
 226         TLS struct {
 227                 Certificate string
 228                 Key         string
 229                 Insecure    bool
 230                 ACME        struct {
 231                         Server string
 232                 }
 233         }
 234         Users struct {
 235                 ActivatedUsersAreVisibleToOthers      bool
 236                 AnonymousUserToken                    string
 237                 AdminNotifierEmailFrom                string
 238                 AutoAdminFirstUser                    bool
 239                 AutoAdminUserWithEmail                string
 240                 AutoSetupNewUsers                     bool
 241                 AutoSetupNewUsersWithRepository       bool
 242                 AutoSetupNewUsersWithVmUUID           string
 243                 AutoSetupUsernameBlacklist            StringSet
 244                 EmailSubjectPrefix                    string
 245                 NewInactiveUserNotificationRecipients StringSet
 246                 NewUserNotificationRecipients         StringSet
 247                 NewUsersAreActive                     bool
 248                 UserNotifierEmailFrom                 string
 249                 UserNotifierEmailBcc                  StringSet
 250                 UserProfileNotificationAddress        string
 251                 PreferDomainForUsername               string
 252                 UserSetupMailText                     string
 253                 RoleGroupsVisibleToAll                bool
 254         }
 255         StorageClasses map[string]StorageClassConfig
 256         Volumes        map[string]Volume
 257         Workbench      struct {
 258                 ActivationContactLink            string
 259                 APIClientConnectTimeout          Duration
 260                 APIClientReceiveTimeout          Duration
 261                 APIResponseCompression           bool
 262                 ApplicationMimetypesWithViewIcon StringSet
 263                 ArvadosDocsite                   string
 264                 ArvadosPublicDataDocURL          string
 265                 DefaultOpenIdPrefix              string
 266                 DisableSharingURLsUI             bool
 267                 EnableGettingStartedPopup        bool
 268                 EnablePublicProjectsPage         bool
 269                 FileViewersConfigURL             string
 270                 LogViewerMaxBytes                ByteSize
 271                 MultiSiteSearch                  string
 272                 ProfilingEnabled                 bool
 273                 Repositories                     bool
 274                 RepositoryCache                  string
 275                 RunningJobLogRecordsToFetch      int
 276                 SecretKeyBase                    string
 277                 ShowRecentCollectionsOnDashboard bool
 278                 ShowUserAgreementInline          bool
 279                 ShowUserNotifications            bool
 280                 SiteName                         string
 281                 Theme                            string
 282                 UserProfileFormFields            map[string]struct {
 283                         Type                 string
 284                         FormFieldTitle       string
 285                         FormFieldDescription string
 286                         Required             bool
 287                         Position             int
 288                         Options              map[string]struct{}
 289                 }
 290                 UserProfileFormMessage string
 291                 WelcomePageHTML        string
 292                 InactivePageHTML       string
 293                 SSHHelpPageHTML        string
 294                 SSHHelpHostSuffix      string
 295                 IdleTimeout            Duration
 296         }
 297 }
 298
 299 type StorageClassConfig struct {
 300         Default  bool
 301         Priority int
 302 }
 303
 304 type Volume struct {
 305         AccessViaHosts   map[URL]VolumeAccess
 306         ReadOnly         bool
 307         Replication      int
 308         StorageClasses   map[string]bool
 309         Driver           string
 310         DriverParameters json.RawMessage
 311 }
 312
 313 type S3VolumeDriverParameters struct {
 314         IAMRole            string
 315         AccessKeyID        string
 316         SecretAccessKey    string
 317         Endpoint           string
 318         Region             string
 319         Bucket             string
 320         LocationConstraint bool
 321         V2Signature        bool
 322         UseAWSS3v2Driver   bool
 323         IndexPageSize      int
 324         ConnectTimeout     Duration
 325         ReadTimeout        Duration
 326         RaceWindow         Duration
 327         UnsafeDelete       bool
 328         PrefixLength       int
 329 }
 330
 331 type AzureVolumeDriverParameters struct {
 332         StorageAccountName   string
 333         StorageAccountKey    string
 334         StorageBaseURL       string
 335         ContainerName        string
 336         RequestTimeout       Duration
 337         ListBlobsRetryDelay  Duration
 338         ListBlobsMaxAttempts int
 339 }
 340
 341 type DirectoryVolumeDriverParameters struct {
 342         Root      string
 343         Serialize bool
 344 }
 345
 346 type VolumeAccess struct {
 347         ReadOnly bool
 348 }
 349
 350 type Services struct {
 351         Composer       Service
 352         Controller     Service
 353         DispatchCloud  Service
 354         DispatchLSF    Service
 355         DispatchSLURM  Service
 356         GitHTTP        Service
 357         GitSSH         Service
 358         Health         Service
 359         Keepbalance    Service
 360         Keepproxy      Service
 361         Keepstore      Service
 362         RailsAPI       Service
 363         WebDAVDownload Service
 364         WebDAV         Service
 365         WebShell       Service
 366         Websocket      Service
 367         Workbench1     Service
 368         Workbench2     Service
 369 }
 370
 371 type Service struct {
 372         InternalURLs map[URL]ServiceInstance
 373         ExternalURL  URL
 374 }
 375
 376 type TestUser struct {
 377         Email    string
 378         Password string
 379 }
 380
 381 // URL is a url.URL that is also usable as a JSON key/value.
 382 type URL url.URL
 383
 384 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 385 // used as a JSON key/value.
 386 func (su *URL) UnmarshalText(text []byte) error {
 387         u, err := url.Parse(string(text))
 388         if err == nil {
 389                 *su = URL(*u)
 390                 if su.Path == "" && su.Host != "" {
 391                         // http://example really means http://example/
 392                         su.Path = "/"
 393                 }
 394         }
 395         return err
 396 }
 397
 398 func (su URL) MarshalText() ([]byte, error) {
 399         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 400 }
 401
 402 func (su URL) String() string {
 403         return (*url.URL)(&su).String()
 404 }
 405
 406 type ServiceInstance struct {
 407         ListenURL  URL
 408         Rendezvous string `json:",omitempty"`
 409 }
 410
 411 type PostgreSQL struct {
 412         Connection     PostgreSQLConnection
 413         ConnectionPool int
 414 }
 415
 416 type PostgreSQLConnection map[string]string
 417
 418 type RemoteCluster struct {
 419         Host          string
 420         Proxy         bool
 421         Scheme        string
 422         Insecure      bool
 423         ActivateUsers bool
 424 }
 425
 426 type CUDAFeatures struct {
 427         DriverVersion      string
 428         HardwareCapability string
 429         DeviceCount        int
 430 }
 431
 432 type InstanceType struct {
 433         Name            string `json:"-"`
 434         ProviderType    string
 435         VCPUs           int
 436         RAM             ByteSize
 437         Scratch         ByteSize `json:"-"`
 438         IncludedScratch ByteSize
 439         AddedScratch    ByteSize
 440         Price           float64
 441         Preemptible     bool
 442         CUDA            CUDAFeatures
 443 }
 444
 445 type ContainersConfig struct {
 446         CloudVMs                      CloudVMsConfig
 447         CrunchRunCommand              string
 448         CrunchRunArgumentsList        []string
 449         DefaultKeepCacheRAM           ByteSize
 450         DispatchPrivateKey            string
 451         LogReuseDecisions             bool
 452         MaxComputeVMs                 int
 453         MaxDispatchAttempts           int
 454         MaxRetryAttempts              int
 455         MinRetryPeriod                Duration
 456         ReserveExtraRAM               ByteSize
 457         StaleLockTimeout              Duration
 458         SupportedDockerImageFormats   StringSet
 459         AlwaysUsePreemptibleInstances bool
 460         PreemptiblePriceFactor        float64
 461         RuntimeEngine                 string
 462         LocalKeepBlobBuffersPerVCPU   int
 463         LocalKeepLogsToContainerLog   string
 464
 465         JobsAPI struct {
 466                 Enable         string
 467                 GitInternalDir string
 468         }
 469         Logging struct {
 470                 MaxAge                       Duration
 471                 LogBytesPerEvent             int
 472                 LogSecondsBetweenEvents      Duration
 473                 LogThrottlePeriod            Duration
 474                 LogThrottleBytes             int
 475                 LogThrottleLines             int
 476                 LimitLogBytesPerJob          int
 477                 LogPartialLineThrottlePeriod Duration
 478                 LogUpdatePeriod              Duration
 479                 LogUpdateSize                ByteSize
 480         }
 481         ShellAccess struct {
 482                 Admin bool
 483                 User  bool
 484         }
 485         SLURM struct {
 486                 PrioritySpread             int64
 487                 SbatchArgumentsList        []string
 488                 SbatchEnvironmentVariables map[string]string
 489                 Managed                    struct {
 490                         DNSServerConfDir       string
 491                         DNSServerConfTemplate  string
 492                         DNSServerReloadCommand string
 493                         DNSServerUpdateCommand string
 494                         ComputeNodeDomain      string
 495                         ComputeNodeNameservers StringSet
 496                         AssignNodeHostname     string
 497                 }
 498         }
 499         LSF struct {
 500                 BsubSudoUser      string
 501                 BsubArgumentsList []string
 502                 BsubCUDAArguments []string
 503         }
 504 }
 505
 506 type CloudVMsConfig struct {
 507         Enable bool
 508
 509         BootProbeCommand               string
 510         DeployRunnerBinary             string
 511         ImageID                        string
 512         MaxCloudOpsPerSecond           int
 513         MaxProbesPerSecond             int
 514         MaxConcurrentInstanceCreateOps int
 515         PollInterval                   Duration
 516         ProbeInterval                  Duration
 517         SSHPort                        string
 518         SyncInterval                   Duration
 519         TimeoutBooting                 Duration
 520         TimeoutIdle                    Duration
 521         TimeoutProbe                   Duration
 522         TimeoutShutdown                Duration
 523         TimeoutSignal                  Duration
 524         TimeoutStaleRunLock            Duration
 525         TimeoutTERM                    Duration
 526         ResourceTags                   map[string]string
 527         TagKeyPrefix                   string
 528
 529         Driver           string
 530         DriverParameters json.RawMessage
 531 }
 532
 533 type InstanceTypeMap map[string]InstanceType
 534
 535 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 536
 537 // UnmarshalJSON does special handling of InstanceTypes:
 538 // * populate computed fields (Name and Scratch)
 539 // * error out if InstancesTypes are populated as an array, which was
 540 //   deprecated in Arvados 1.2.0
 541 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 542         fixup := func(t InstanceType) (InstanceType, error) {
 543                 if t.ProviderType == "" {
 544                         t.ProviderType = t.Name
 545                 }
 546                 // If t.Scratch is set in the configuration file, it will be ignored and overwritten.
 547                 // It will also generate a "deprecated or unknown config entry" warning.
 548                 t.Scratch = t.IncludedScratch + t.AddedScratch
 549                 return t, nil
 550         }
 551
 552         if len(data) > 0 && data[0] == '[' {
 553                 return fmt.Errorf("InstanceTypes must be specified as a map, not an array, see https://doc.arvados.org/admin/config.html")
 554         }
 555         var hash map[string]InstanceType
 556         err := json.Unmarshal(data, &hash)
 557         if err != nil {
 558                 return err
 559         }
 560         // Fill in Name field (and ProviderType field, if not
 561         // specified) using hash key.
 562         *it = InstanceTypeMap(hash)
 563         for name, t := range *it {
 564                 t.Name = name
 565                 t, err := fixup(t)
 566                 if err != nil {
 567                         return err
 568                 }
 569                 (*it)[name] = t
 570         }
 571         return nil
 572 }
 573
 574 type StringSet map[string]struct{}
 575
 576 // UnmarshalJSON handles old config files that provide an array of
 577 // instance types instead of a hash.
 578 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 579         if len(data) > 0 && data[0] == '[' {
 580                 var arr []string
 581                 err := json.Unmarshal(data, &arr)
 582                 if err != nil {
 583                         return err
 584                 }
 585                 if len(arr) == 0 {
 586                         *ss = nil
 587                         return nil
 588                 }
 589                 *ss = make(map[string]struct{}, len(arr))
 590                 for _, t := range arr {
 591                         (*ss)[t] = struct{}{}
 592                 }
 593                 return nil
 594         }
 595         var hash map[string]struct{}
 596         err := json.Unmarshal(data, &hash)
 597         if err != nil {
 598                 return err
 599         }
 600         *ss = make(map[string]struct{}, len(hash))
 601         for t := range hash {
 602                 (*ss)[t] = struct{}{}
 603         }
 604
 605         return nil
 606 }
 607
 608 type ServiceName string
 609
 610 const (
 611         ServiceNameController    ServiceName = "arvados-controller"
 612         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 613         ServiceNameDispatchLSF   ServiceName = "arvados-dispatch-lsf"
 614         ServiceNameDispatchSLURM ServiceName = "crunch-dispatch-slurm"
 615         ServiceNameGitHTTP       ServiceName = "arvados-git-httpd"
 616         ServiceNameHealth        ServiceName = "arvados-health"
 617         ServiceNameKeepbalance   ServiceName = "keep-balance"
 618         ServiceNameKeepproxy     ServiceName = "keepproxy"
 619         ServiceNameKeepstore     ServiceName = "keepstore"
 620         ServiceNameKeepweb       ServiceName = "keep-web"
 621         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 622         ServiceNameWebsocket     ServiceName = "arvados-ws"
 623         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 624         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 625 )
 626
 627 // Map returns all services as a map, suitable for iterating over all
 628 // services or looking up a service by name.
 629 func (svcs Services) Map() map[ServiceName]Service {
 630         return map[ServiceName]Service{
 631                 ServiceNameController:    svcs.Controller,
 632                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 633                 ServiceNameDispatchLSF:   svcs.DispatchLSF,
 634                 ServiceNameDispatchSLURM: svcs.DispatchSLURM,
 635                 ServiceNameGitHTTP:       svcs.GitHTTP,
 636                 ServiceNameHealth:        svcs.Health,
 637                 ServiceNameKeepbalance:   svcs.Keepbalance,
 638                 ServiceNameKeepproxy:     svcs.Keepproxy,
 639                 ServiceNameKeepstore:     svcs.Keepstore,
 640                 ServiceNameKeepweb:       svcs.WebDAV,
 641                 ServiceNameRailsAPI:      svcs.RailsAPI,
 642                 ServiceNameWebsocket:     svcs.Websocket,
 643                 ServiceNameWorkbench1:    svcs.Workbench1,
 644                 ServiceNameWorkbench2:    svcs.Workbench2,
 645         }
 646 }