sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13         "time"
  14
  15         "git.arvados.org/arvados.git/sdk/go/config"
  16 )
  17
  18 var DefaultConfigFile = func() string {
  19         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  20                 return path
  21         }
  22         return "/etc/arvados/config.yml"
  23 }()
  24
  25 type Config struct {
  26         Clusters         map[string]Cluster
  27         AutoReloadConfig bool
  28         SourceTimestamp  time.Time
  29         SourceSHA256     string
  30 }
  31
  32 // GetConfig returns the current system config, loading it from
  33 // configFile if needed.
  34 func GetConfig(configFile string) (*Config, error) {
  35         var cfg Config
  36         err := config.LoadFile(&cfg, configFile)
  37         return &cfg, err
  38 }
  39
  40 // GetCluster returns the cluster ID and config for the given
  41 // cluster, or the default/only configured cluster if clusterID is "".
  42 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  43         if clusterID == "" {
  44                 if len(sc.Clusters) == 0 {
  45                         return nil, fmt.Errorf("no clusters configured")
  46                 } else if len(sc.Clusters) > 1 {
  47                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  48                 } else {
  49                         for id, cc := range sc.Clusters {
  50                                 cc.ClusterID = id
  51                                 return &cc, nil
  52                         }
  53                 }
  54         }
  55         cc, ok := sc.Clusters[clusterID]
  56         if !ok {
  57                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  58         }
  59         cc.ClusterID = clusterID
  60         return &cc, nil
  61 }
  62
  63 type WebDAVCacheConfig struct {
  64         TTL                  Duration
  65         UUIDTTL              Duration
  66         MaxBlockEntries      int
  67         MaxCollectionEntries int
  68         MaxCollectionBytes   int64
  69         MaxUUIDEntries       int
  70         MaxSessions          int
  71 }
  72
  73 type UploadDownloadPermission struct {
  74         Upload   bool
  75         Download bool
  76 }
  77
  78 type UploadDownloadRolePermissions struct {
  79         User  UploadDownloadPermission
  80         Admin UploadDownloadPermission
  81 }
  82
  83 type ManagedProperties map[string]struct {
  84         Value     interface{}
  85         Function  string
  86         Protected bool
  87 }
  88
  89 type Cluster struct {
  90         ClusterID       string `json:"-"`
  91         ManagementToken string
  92         SystemRootToken string
  93         Services        Services
  94         InstanceTypes   InstanceTypeMap
  95         Containers      ContainersConfig
  96         RemoteClusters  map[string]RemoteCluster
  97         PostgreSQL      PostgreSQL
  98
  99         API struct {
 100                 AsyncPermissionsUpdateInterval   Duration
 101                 DisabledAPIs                     StringSet
 102                 MaxIndexDatabaseRead             int
 103                 MaxItemsPerResponse              int
 104                 MaxConcurrentRequests            int
 105                 MaxKeepBlobBuffers               int
 106                 MaxRequestAmplification          int
 107                 MaxRequestSize                   int
 108                 MaxTokenLifetime                 Duration
 109                 RequestTimeout                   Duration
 110                 SendTimeout                      Duration
 111                 WebsocketClientEventQueue        int
 112                 WebsocketServerEventQueue        int
 113                 KeepServiceRequestTimeout        Duration
 114                 VocabularyPath                   string
 115                 FreezeProjectRequiresDescription bool
 116                 FreezeProjectRequiresProperties  StringSet
 117                 UnfreezeProjectRequiresAdmin     bool
 118         }
 119         AuditLogs struct {
 120                 MaxAge             Duration
 121                 MaxDeleteBatch     int
 122                 UnloggedAttributes StringSet
 123         }
 124         Collections struct {
 125                 BlobSigning                  bool
 126                 BlobSigningKey               string
 127                 BlobSigningTTL               Duration
 128                 BlobTrash                    bool
 129                 BlobTrashLifetime            Duration
 130                 BlobTrashCheckInterval       Duration
 131                 BlobTrashConcurrency         int
 132                 BlobDeleteConcurrency        int
 133                 BlobReplicateConcurrency     int
 134                 CollectionVersioning         bool
 135                 DefaultTrashLifetime         Duration
 136                 DefaultReplication           int
 137                 ManagedProperties            ManagedProperties
 138                 PreserveVersionIfIdle        Duration
 139                 TrashSweepInterval           Duration
 140                 TrustAllContent              bool
 141                 ForwardSlashNameSubstitution string
 142                 S3FolderObjects              bool
 143
 144                 BlobMissingReport        string
 145                 BalancePeriod            Duration
 146                 BalanceCollectionBatch   int
 147                 BalanceCollectionBuffers int
 148                 BalanceTimeout           Duration
 149                 BalanceUpdateLimit       int
 150
 151                 WebDAVCache WebDAVCacheConfig
 152
 153                 KeepproxyPermission UploadDownloadRolePermissions
 154                 WebDAVPermission    UploadDownloadRolePermissions
 155                 WebDAVLogEvents     bool
 156         }
 157         Git struct {
 158                 GitCommand   string
 159                 GitoliteHome string
 160                 Repositories string
 161         }
 162         Login struct {
 163                 LDAP struct {
 164                         Enable             bool
 165                         URL                URL
 166                         StartTLS           bool
 167                         InsecureTLS        bool
 168                         StripDomain        string
 169                         AppendDomain       string
 170                         SearchAttribute    string
 171                         SearchBindUser     string
 172                         SearchBindPassword string
 173                         SearchBase         string
 174                         SearchFilters      string
 175                         EmailAttribute     string
 176                         UsernameAttribute  string
 177                 }
 178                 Google struct {
 179                         Enable                          bool
 180                         ClientID                        string
 181                         ClientSecret                    string
 182                         AlternateEmailAddresses         bool
 183                         AuthenticationRequestParameters map[string]string
 184                 }
 185                 OpenIDConnect struct {
 186                         Enable                          bool
 187                         Issuer                          string
 188                         ClientID                        string
 189                         ClientSecret                    string
 190                         EmailClaim                      string
 191                         EmailVerifiedClaim              string
 192                         UsernameClaim                   string
 193                         AcceptAccessToken               bool
 194                         AcceptAccessTokenScope          string
 195                         AuthenticationRequestParameters map[string]string
 196                 }
 197                 PAM struct {
 198                         Enable             bool
 199                         Service            string
 200                         DefaultEmailDomain string
 201                 }
 202                 Test struct {
 203                         Enable bool
 204                         Users  map[string]TestUser
 205                 }
 206                 LoginCluster       string
 207                 RemoteTokenRefresh Duration
 208                 TokenLifetime      Duration
 209                 TrustedClients     map[string]struct{}
 210                 IssueTrustedTokens bool
 211         }
 212         Mail struct {
 213                 MailchimpAPIKey                string
 214                 MailchimpListID                string
 215                 SendUserSetupNotificationEmail bool
 216                 IssueReporterEmailFrom         string
 217                 IssueReporterEmailTo           string
 218                 SupportEmailAddress            string
 219                 EmailFrom                      string
 220         }
 221         SystemLogs struct {
 222                 LogLevel                string
 223                 Format                  string
 224                 MaxRequestLogParamsSize int
 225         }
 226         TLS struct {
 227                 Certificate string
 228                 Key         string
 229                 Insecure    bool
 230         }
 231         Users struct {
 232                 ActivatedUsersAreVisibleToOthers      bool
 233                 AnonymousUserToken                    string
 234                 AdminNotifierEmailFrom                string
 235                 AutoAdminFirstUser                    bool
 236                 AutoAdminUserWithEmail                string
 237                 AutoSetupNewUsers                     bool
 238                 AutoSetupNewUsersWithRepository       bool
 239                 AutoSetupNewUsersWithVmUUID           string
 240                 AutoSetupUsernameBlacklist            StringSet
 241                 EmailSubjectPrefix                    string
 242                 NewInactiveUserNotificationRecipients StringSet
 243                 NewUserNotificationRecipients         StringSet
 244                 NewUsersAreActive                     bool
 245                 UserNotifierEmailFrom                 string
 246                 UserNotifierEmailBcc                  StringSet
 247                 UserProfileNotificationAddress        string
 248                 PreferDomainForUsername               string
 249                 UserSetupMailText                     string
 250                 RoleGroupsVisibleToAll                bool
 251         }
 252         StorageClasses map[string]StorageClassConfig
 253         Volumes        map[string]Volume
 254         Workbench      struct {
 255                 ActivationContactLink            string
 256                 APIClientConnectTimeout          Duration
 257                 APIClientReceiveTimeout          Duration
 258                 APIResponseCompression           bool
 259                 ApplicationMimetypesWithViewIcon StringSet
 260                 ArvadosDocsite                   string
 261                 ArvadosPublicDataDocURL          string
 262                 DefaultOpenIdPrefix              string
 263                 EnableGettingStartedPopup        bool
 264                 EnablePublicProjectsPage         bool
 265                 FileViewersConfigURL             string
 266                 LogViewerMaxBytes                ByteSize
 267                 MultiSiteSearch                  string
 268                 ProfilingEnabled                 bool
 269                 Repositories                     bool
 270                 RepositoryCache                  string
 271                 RunningJobLogRecordsToFetch      int
 272                 SecretKeyBase                    string
 273                 ShowRecentCollectionsOnDashboard bool
 274                 ShowUserAgreementInline          bool
 275                 ShowUserNotifications            bool
 276                 SiteName                         string
 277                 Theme                            string
 278                 UserProfileFormFields            map[string]struct {
 279                         Type                 string
 280                         FormFieldTitle       string
 281                         FormFieldDescription string
 282                         Required             bool
 283                         Position             int
 284                         Options              map[string]struct{}
 285                 }
 286                 UserProfileFormMessage string
 287                 WelcomePageHTML        string
 288                 InactivePageHTML       string
 289                 SSHHelpPageHTML        string
 290                 SSHHelpHostSuffix      string
 291                 IdleTimeout            Duration
 292         }
 293 }
 294
 295 type StorageClassConfig struct {
 296         Default  bool
 297         Priority int
 298 }
 299
 300 type Volume struct {
 301         AccessViaHosts   map[URL]VolumeAccess
 302         ReadOnly         bool
 303         Replication      int
 304         StorageClasses   map[string]bool
 305         Driver           string
 306         DriverParameters json.RawMessage
 307 }
 308
 309 type S3VolumeDriverParameters struct {
 310         IAMRole            string
 311         AccessKeyID        string
 312         SecretAccessKey    string
 313         Endpoint           string
 314         Region             string
 315         Bucket             string
 316         LocationConstraint bool
 317         V2Signature        bool
 318         UseAWSS3v2Driver   bool
 319         IndexPageSize      int
 320         ConnectTimeout     Duration
 321         ReadTimeout        Duration
 322         RaceWindow         Duration
 323         UnsafeDelete       bool
 324         PrefixLength       int
 325 }
 326
 327 type AzureVolumeDriverParameters struct {
 328         StorageAccountName   string
 329         StorageAccountKey    string
 330         StorageBaseURL       string
 331         ContainerName        string
 332         RequestTimeout       Duration
 333         ListBlobsRetryDelay  Duration
 334         ListBlobsMaxAttempts int
 335 }
 336
 337 type DirectoryVolumeDriverParameters struct {
 338         Root      string
 339         Serialize bool
 340 }
 341
 342 type VolumeAccess struct {
 343         ReadOnly bool
 344 }
 345
 346 type Services struct {
 347         Composer       Service
 348         Controller     Service
 349         DispatchCloud  Service
 350         DispatchLSF    Service
 351         GitHTTP        Service
 352         GitSSH         Service
 353         Health         Service
 354         Keepbalance    Service
 355         Keepproxy      Service
 356         Keepstore      Service
 357         RailsAPI       Service
 358         WebDAVDownload Service
 359         WebDAV         Service
 360         WebShell       Service
 361         Websocket      Service
 362         Workbench1     Service
 363         Workbench2     Service
 364 }
 365
 366 type Service struct {
 367         InternalURLs map[URL]ServiceInstance
 368         ExternalURL  URL
 369 }
 370
 371 type TestUser struct {
 372         Email    string
 373         Password string
 374 }
 375
 376 // URL is a url.URL that is also usable as a JSON key/value.
 377 type URL url.URL
 378
 379 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 380 // used as a JSON key/value.
 381 func (su *URL) UnmarshalText(text []byte) error {
 382         u, err := url.Parse(string(text))
 383         if err == nil {
 384                 *su = URL(*u)
 385                 if su.Path == "" && su.Host != "" {
 386                         // http://example really means http://example/
 387                         su.Path = "/"
 388                 }
 389         }
 390         return err
 391 }
 392
 393 func (su URL) MarshalText() ([]byte, error) {
 394         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 395 }
 396
 397 func (su URL) String() string {
 398         return (*url.URL)(&su).String()
 399 }
 400
 401 type ServiceInstance struct {
 402         Rendezvous string `json:",omitempty"`
 403 }
 404
 405 type PostgreSQL struct {
 406         Connection     PostgreSQLConnection
 407         ConnectionPool int
 408 }
 409
 410 type PostgreSQLConnection map[string]string
 411
 412 type RemoteCluster struct {
 413         Host          string
 414         Proxy         bool
 415         Scheme        string
 416         Insecure      bool
 417         ActivateUsers bool
 418 }
 419
 420 type CUDAFeatures struct {
 421         DriverVersion      string
 422         HardwareCapability string
 423         DeviceCount        int
 424 }
 425
 426 type InstanceType struct {
 427         Name            string
 428         ProviderType    string
 429         VCPUs           int
 430         RAM             ByteSize
 431         Scratch         ByteSize
 432         IncludedScratch ByteSize
 433         AddedScratch    ByteSize
 434         Price           float64
 435         Preemptible     bool
 436         CUDA            CUDAFeatures
 437 }
 438
 439 type ContainersConfig struct {
 440         CloudVMs                      CloudVMsConfig
 441         CrunchRunCommand              string
 442         CrunchRunArgumentsList        []string
 443         DefaultKeepCacheRAM           ByteSize
 444         DispatchPrivateKey            string
 445         LogReuseDecisions             bool
 446         MaxComputeVMs                 int
 447         MaxDispatchAttempts           int
 448         MaxRetryAttempts              int
 449         MinRetryPeriod                Duration
 450         ReserveExtraRAM               ByteSize
 451         StaleLockTimeout              Duration
 452         SupportedDockerImageFormats   StringSet
 453         AlwaysUsePreemptibleInstances bool
 454         PreemptiblePriceFactor        float64
 455         RuntimeEngine                 string
 456         LocalKeepBlobBuffersPerVCPU   int
 457         LocalKeepLogsToContainerLog   string
 458
 459         JobsAPI struct {
 460                 Enable         string
 461                 GitInternalDir string
 462         }
 463         Logging struct {
 464                 MaxAge                       Duration
 465                 LogBytesPerEvent             int
 466                 LogSecondsBetweenEvents      Duration
 467                 LogThrottlePeriod            Duration
 468                 LogThrottleBytes             int
 469                 LogThrottleLines             int
 470                 LimitLogBytesPerJob          int
 471                 LogPartialLineThrottlePeriod Duration
 472                 LogUpdatePeriod              Duration
 473                 LogUpdateSize                ByteSize
 474         }
 475         ShellAccess struct {
 476                 Admin bool
 477                 User  bool
 478         }
 479         SLURM struct {
 480                 PrioritySpread             int64
 481                 SbatchArgumentsList        []string
 482                 SbatchEnvironmentVariables map[string]string
 483                 Managed                    struct {
 484                         DNSServerConfDir       string
 485                         DNSServerConfTemplate  string
 486                         DNSServerReloadCommand string
 487                         DNSServerUpdateCommand string
 488                         ComputeNodeDomain      string
 489                         ComputeNodeNameservers StringSet
 490                         AssignNodeHostname     string
 491                 }
 492         }
 493         LSF struct {
 494                 BsubSudoUser      string
 495                 BsubArgumentsList []string
 496                 BsubCUDAArguments []string
 497         }
 498 }
 499
 500 type CloudVMsConfig struct {
 501         Enable bool
 502
 503         BootProbeCommand               string
 504         DeployRunnerBinary             string
 505         ImageID                        string
 506         MaxCloudOpsPerSecond           int
 507         MaxProbesPerSecond             int
 508         MaxConcurrentInstanceCreateOps int
 509         PollInterval                   Duration
 510         ProbeInterval                  Duration
 511         SSHPort                        string
 512         SyncInterval                   Duration
 513         TimeoutBooting                 Duration
 514         TimeoutIdle                    Duration
 515         TimeoutProbe                   Duration
 516         TimeoutShutdown                Duration
 517         TimeoutSignal                  Duration
 518         TimeoutStaleRunLock            Duration
 519         TimeoutTERM                    Duration
 520         ResourceTags                   map[string]string
 521         TagKeyPrefix                   string
 522
 523         Driver           string
 524         DriverParameters json.RawMessage
 525 }
 526
 527 type InstanceTypeMap map[string]InstanceType
 528
 529 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 530
 531 // UnmarshalJSON handles old config files that provide an array of
 532 // instance types instead of a hash.
 533 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 534         fixup := func(t InstanceType) (InstanceType, error) {
 535                 if t.ProviderType == "" {
 536                         t.ProviderType = t.Name
 537                 }
 538                 if t.Scratch == 0 {
 539                         t.Scratch = t.IncludedScratch + t.AddedScratch
 540                 } else if t.AddedScratch == 0 {
 541                         t.AddedScratch = t.Scratch - t.IncludedScratch
 542                 } else if t.IncludedScratch == 0 {
 543                         t.IncludedScratch = t.Scratch - t.AddedScratch
 544                 }
 545
 546                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 547                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 548                 }
 549                 return t, nil
 550         }
 551
 552         if len(data) > 0 && data[0] == '[' {
 553                 var arr []InstanceType
 554                 err := json.Unmarshal(data, &arr)
 555                 if err != nil {
 556                         return err
 557                 }
 558                 if len(arr) == 0 {
 559                         *it = nil
 560                         return nil
 561                 }
 562                 *it = make(map[string]InstanceType, len(arr))
 563                 for _, t := range arr {
 564                         if _, ok := (*it)[t.Name]; ok {
 565                                 return errDuplicateInstanceTypeName
 566                         }
 567                         t, err := fixup(t)
 568                         if err != nil {
 569                                 return err
 570                         }
 571                         (*it)[t.Name] = t
 572                 }
 573                 return nil
 574         }
 575         var hash map[string]InstanceType
 576         err := json.Unmarshal(data, &hash)
 577         if err != nil {
 578                 return err
 579         }
 580         // Fill in Name field (and ProviderType field, if not
 581         // specified) using hash key.
 582         *it = InstanceTypeMap(hash)
 583         for name, t := range *it {
 584                 t.Name = name
 585                 t, err := fixup(t)
 586                 if err != nil {
 587                         return err
 588                 }
 589                 (*it)[name] = t
 590         }
 591         return nil
 592 }
 593
 594 type StringSet map[string]struct{}
 595
 596 // UnmarshalJSON handles old config files that provide an array of
 597 // instance types instead of a hash.
 598 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 599         if len(data) > 0 && data[0] == '[' {
 600                 var arr []string
 601                 err := json.Unmarshal(data, &arr)
 602                 if err != nil {
 603                         return err
 604                 }
 605                 if len(arr) == 0 {
 606                         *ss = nil
 607                         return nil
 608                 }
 609                 *ss = make(map[string]struct{}, len(arr))
 610                 for _, t := range arr {
 611                         (*ss)[t] = struct{}{}
 612                 }
 613                 return nil
 614         }
 615         var hash map[string]struct{}
 616         err := json.Unmarshal(data, &hash)
 617         if err != nil {
 618                 return err
 619         }
 620         *ss = make(map[string]struct{}, len(hash))
 621         for t := range hash {
 622                 (*ss)[t] = struct{}{}
 623         }
 624
 625         return nil
 626 }
 627
 628 type ServiceName string
 629
 630 const (
 631         ServiceNameController    ServiceName = "arvados-controller"
 632         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 633         ServiceNameDispatchLSF   ServiceName = "arvados-dispatch-lsf"
 634         ServiceNameGitHTTP       ServiceName = "arvados-git-httpd"
 635         ServiceNameHealth        ServiceName = "arvados-health"
 636         ServiceNameKeepbalance   ServiceName = "keep-balance"
 637         ServiceNameKeepproxy     ServiceName = "keepproxy"
 638         ServiceNameKeepstore     ServiceName = "keepstore"
 639         ServiceNameKeepweb       ServiceName = "keep-web"
 640         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 641         ServiceNameWebsocket     ServiceName = "arvados-ws"
 642         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 643         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 644 )
 645
 646 // Map returns all services as a map, suitable for iterating over all
 647 // services or looking up a service by name.
 648 func (svcs Services) Map() map[ServiceName]Service {
 649         return map[ServiceName]Service{
 650                 ServiceNameController:    svcs.Controller,
 651                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 652                 ServiceNameDispatchLSF:   svcs.DispatchLSF,
 653                 ServiceNameGitHTTP:       svcs.GitHTTP,
 654                 ServiceNameHealth:        svcs.Health,
 655                 ServiceNameKeepbalance:   svcs.Keepbalance,
 656                 ServiceNameKeepproxy:     svcs.Keepproxy,
 657                 ServiceNameKeepstore:     svcs.Keepstore,
 658                 ServiceNameKeepweb:       svcs.WebDAV,
 659                 ServiceNameRailsAPI:      svcs.RailsAPI,
 660                 ServiceNameWebsocket:     svcs.Websocket,
 661                 ServiceNameWorkbench1:    svcs.Workbench1,
 662                 ServiceNameWorkbench2:    svcs.Workbench2,
 663         }
 664 }