sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.arvados.org/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters         map[string]Cluster
  27         AutoReloadConfig bool
  28 }
  29
  30 // GetConfig returns the current system config, loading it from
  31 // configFile if needed.
  32 func GetConfig(configFile string) (*Config, error) {
  33         var cfg Config
  34         err := config.LoadFile(&cfg, configFile)
  35         return &cfg, err
  36 }
  37
  38 // GetCluster returns the cluster ID and config for the given
  39 // cluster, or the default/only configured cluster if clusterID is "".
  40 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  41         if clusterID == "" {
  42                 if len(sc.Clusters) == 0 {
  43                         return nil, fmt.Errorf("no clusters configured")
  44                 } else if len(sc.Clusters) > 1 {
  45                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  46                 } else {
  47                         for id, cc := range sc.Clusters {
  48                                 cc.ClusterID = id
  49                                 return &cc, nil
  50                         }
  51                 }
  52         }
  53         if cc, ok := sc.Clusters[clusterID]; !ok {
  54                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  55         } else {
  56                 cc.ClusterID = clusterID
  57                 return &cc, nil
  58         }
  59 }
  60
  61 type WebDAVCacheConfig struct {
  62         TTL                  Duration
  63         UUIDTTL              Duration
  64         MaxBlockEntries      int
  65         MaxCollectionEntries int
  66         MaxCollectionBytes   int64
  67         MaxPermissionEntries int
  68         MaxUUIDEntries       int
  69 }
  70
  71 type Cluster struct {
  72         ClusterID       string `json:"-"`
  73         ManagementToken string
  74         SystemRootToken string
  75         Services        Services
  76         InstanceTypes   InstanceTypeMap
  77         Containers      ContainersConfig
  78         RemoteClusters  map[string]RemoteCluster
  79         PostgreSQL      PostgreSQL
  80
  81         API struct {
  82                 AsyncPermissionsUpdateInterval Duration
  83                 DisabledAPIs                   StringSet
  84                 MaxIndexDatabaseRead           int
  85                 MaxItemsPerResponse            int
  86                 MaxConcurrentRequests          int
  87                 MaxKeepBlobBuffers             int
  88                 MaxRequestAmplification        int
  89                 MaxRequestSize                 int
  90                 RailsSessionSecretToken        string
  91                 RequestTimeout                 Duration
  92                 SendTimeout                    Duration
  93                 WebsocketClientEventQueue      int
  94                 WebsocketServerEventQueue      int
  95                 KeepServiceRequestTimeout      Duration
  96         }
  97         AuditLogs struct {
  98                 MaxAge             Duration
  99                 MaxDeleteBatch     int
 100                 UnloggedAttributes StringSet
 101         }
 102         Collections struct {
 103                 BlobSigning              bool
 104                 BlobSigningKey           string
 105                 BlobSigningTTL           Duration
 106                 BlobTrash                bool
 107                 BlobTrashLifetime        Duration
 108                 BlobTrashCheckInterval   Duration
 109                 BlobTrashConcurrency     int
 110                 BlobDeleteConcurrency    int
 111                 BlobReplicateConcurrency int
 112                 CollectionVersioning     bool
 113                 DefaultTrashLifetime     Duration
 114                 DefaultReplication       int
 115                 ManagedProperties        map[string]struct {
 116                         Value     interface{}
 117                         Function  string
 118                         Protected bool
 119                 }
 120                 PreserveVersionIfIdle        Duration
 121                 TrashSweepInterval           Duration
 122                 TrustAllContent              bool
 123                 ForwardSlashNameSubstitution string
 124                 S3FolderObjects              bool
 125
 126                 BlobMissingReport        string
 127                 BalancePeriod            Duration
 128                 BalanceCollectionBatch   int
 129                 BalanceCollectionBuffers int
 130                 BalanceTimeout           Duration
 131
 132                 WebDAVCache WebDAVCacheConfig
 133         }
 134         Git struct {
 135                 GitCommand   string
 136                 GitoliteHome string
 137                 Repositories string
 138         }
 139         Login struct {
 140                 LDAP struct {
 141                         Enable             bool
 142                         URL                URL
 143                         StartTLS           bool
 144                         InsecureTLS        bool
 145                         StripDomain        string
 146                         AppendDomain       string
 147                         SearchAttribute    string
 148                         SearchBindUser     string
 149                         SearchBindPassword string
 150                         SearchBase         string
 151                         SearchFilters      string
 152                         EmailAttribute     string
 153                         UsernameAttribute  string
 154                 }
 155                 Google struct {
 156                         Enable                  bool
 157                         ClientID                string
 158                         ClientSecret            string
 159                         AlternateEmailAddresses bool
 160                 }
 161                 OpenIDConnect struct {
 162                         Enable             bool
 163                         Issuer             string
 164                         ClientID           string
 165                         ClientSecret       string
 166                         EmailClaim         string
 167                         EmailVerifiedClaim string
 168                         UsernameClaim      string
 169                 }
 170                 PAM struct {
 171                         Enable             bool
 172                         Service            string
 173                         DefaultEmailDomain string
 174                 }
 175                 SSO struct {
 176                         Enable            bool
 177                         ProviderAppID     string
 178                         ProviderAppSecret string
 179                 }
 180                 Test struct {
 181                         Enable bool
 182                         Users  map[string]TestUser
 183                 }
 184                 LoginCluster       string
 185                 RemoteTokenRefresh Duration
 186         }
 187         Mail struct {
 188                 MailchimpAPIKey                string
 189                 MailchimpListID                string
 190                 SendUserSetupNotificationEmail bool
 191                 IssueReporterEmailFrom         string
 192                 IssueReporterEmailTo           string
 193                 SupportEmailAddress            string
 194                 EmailFrom                      string
 195         }
 196         SystemLogs struct {
 197                 LogLevel                string
 198                 Format                  string
 199                 MaxRequestLogParamsSize int
 200         }
 201         TLS struct {
 202                 Certificate string
 203                 Key         string
 204                 Insecure    bool
 205         }
 206         Users struct {
 207                 AnonymousUserToken                    string
 208                 AdminNotifierEmailFrom                string
 209                 AutoAdminFirstUser                    bool
 210                 AutoAdminUserWithEmail                string
 211                 AutoSetupNewUsers                     bool
 212                 AutoSetupNewUsersWithRepository       bool
 213                 AutoSetupNewUsersWithVmUUID           string
 214                 AutoSetupUsernameBlacklist            StringSet
 215                 EmailSubjectPrefix                    string
 216                 NewInactiveUserNotificationRecipients StringSet
 217                 NewUserNotificationRecipients         StringSet
 218                 NewUsersAreActive                     bool
 219                 UserNotifierEmailFrom                 string
 220                 UserProfileNotificationAddress        string
 221                 PreferDomainForUsername               string
 222         }
 223         Volumes   map[string]Volume
 224         Workbench struct {
 225                 ActivationContactLink            string
 226                 APIClientConnectTimeout          Duration
 227                 APIClientReceiveTimeout          Duration
 228                 APIResponseCompression           bool
 229                 ApplicationMimetypesWithViewIcon StringSet
 230                 ArvadosDocsite                   string
 231                 ArvadosPublicDataDocURL          string
 232                 DefaultOpenIdPrefix              string
 233                 EnableGettingStartedPopup        bool
 234                 EnablePublicProjectsPage         bool
 235                 FileViewersConfigURL             string
 236                 LogViewerMaxBytes                ByteSize
 237                 MultiSiteSearch                  string
 238                 ProfilingEnabled                 bool
 239                 Repositories                     bool
 240                 RepositoryCache                  string
 241                 RunningJobLogRecordsToFetch      int
 242                 SecretKeyBase                    string
 243                 ShowRecentCollectionsOnDashboard bool
 244                 ShowUserAgreementInline          bool
 245                 ShowUserNotifications            bool
 246                 SiteName                         string
 247                 Theme                            string
 248                 UserProfileFormFields            map[string]struct {
 249                         Type                 string
 250                         FormFieldTitle       string
 251                         FormFieldDescription string
 252                         Required             bool
 253                         Position             int
 254                         Options              map[string]struct{}
 255                 }
 256                 UserProfileFormMessage string
 257                 VocabularyURL          string
 258                 WelcomePageHTML        string
 259                 InactivePageHTML       string
 260                 SSHHelpPageHTML        string
 261                 SSHHelpHostSuffix      string
 262         }
 263
 264         ForceLegacyAPI14 bool
 265 }
 266
 267 type Volume struct {
 268         AccessViaHosts   map[URL]VolumeAccess
 269         ReadOnly         bool
 270         Replication      int
 271         StorageClasses   map[string]bool
 272         Driver           string
 273         DriverParameters json.RawMessage
 274 }
 275
 276 type S3VolumeDriverParameters struct {
 277         IAMRole            string
 278         AccessKey          string
 279         SecretKey          string
 280         Endpoint           string
 281         Region             string
 282         Bucket             string
 283         LocationConstraint bool
 284         V2Signature        bool
 285         UseAWSS3v2Driver   bool
 286         IndexPageSize      int
 287         ConnectTimeout     Duration
 288         ReadTimeout        Duration
 289         RaceWindow         Duration
 290         UnsafeDelete       bool
 291 }
 292
 293 type AzureVolumeDriverParameters struct {
 294         StorageAccountName   string
 295         StorageAccountKey    string
 296         StorageBaseURL       string
 297         ContainerName        string
 298         RequestTimeout       Duration
 299         ListBlobsRetryDelay  Duration
 300         ListBlobsMaxAttempts int
 301 }
 302
 303 type DirectoryVolumeDriverParameters struct {
 304         Root      string
 305         Serialize bool
 306 }
 307
 308 type VolumeAccess struct {
 309         ReadOnly bool
 310 }
 311
 312 type Services struct {
 313         Composer       Service
 314         Controller     Service
 315         DispatchCloud  Service
 316         GitHTTP        Service
 317         GitSSH         Service
 318         Health         Service
 319         Keepbalance    Service
 320         Keepproxy      Service
 321         Keepstore      Service
 322         RailsAPI       Service
 323         SSO            Service
 324         WebDAVDownload Service
 325         WebDAV         Service
 326         WebShell       Service
 327         Websocket      Service
 328         Workbench1     Service
 329         Workbench2     Service
 330 }
 331
 332 type Service struct {
 333         InternalURLs map[URL]ServiceInstance
 334         ExternalURL  URL
 335 }
 336
 337 type TestUser struct {
 338         Email    string
 339         Password string
 340 }
 341
 342 // URL is a url.URL that is also usable as a JSON key/value.
 343 type URL url.URL
 344
 345 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 346 // used as a JSON key/value.
 347 func (su *URL) UnmarshalText(text []byte) error {
 348         u, err := url.Parse(string(text))
 349         if err == nil {
 350                 *su = URL(*u)
 351                 if su.Path == "" && su.Host != "" {
 352                         // http://example really means http://example/
 353                         su.Path = "/"
 354                 }
 355         }
 356         return err
 357 }
 358
 359 func (su URL) MarshalText() ([]byte, error) {
 360         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 361 }
 362
 363 func (su URL) String() string {
 364         return (*url.URL)(&su).String()
 365 }
 366
 367 type ServiceInstance struct {
 368         Rendezvous string `json:",omitempty"`
 369 }
 370
 371 type PostgreSQL struct {
 372         Connection     PostgreSQLConnection
 373         ConnectionPool int
 374 }
 375
 376 type PostgreSQLConnection map[string]string
 377
 378 type RemoteCluster struct {
 379         Host          string
 380         Proxy         bool
 381         Scheme        string
 382         Insecure      bool
 383         ActivateUsers bool
 384 }
 385
 386 type InstanceType struct {
 387         Name            string
 388         ProviderType    string
 389         VCPUs           int
 390         RAM             ByteSize
 391         Scratch         ByteSize
 392         IncludedScratch ByteSize
 393         AddedScratch    ByteSize
 394         Price           float64
 395         Preemptible     bool
 396 }
 397
 398 type ContainersConfig struct {
 399         CloudVMs                    CloudVMsConfig
 400         CrunchRunCommand            string
 401         CrunchRunArgumentsList      []string
 402         DefaultKeepCacheRAM         ByteSize
 403         DispatchPrivateKey          string
 404         LogReuseDecisions           bool
 405         MaxComputeVMs               int
 406         MaxDispatchAttempts         int
 407         MaxRetryAttempts            int
 408         MinRetryPeriod              Duration
 409         ReserveExtraRAM             ByteSize
 410         StaleLockTimeout            Duration
 411         SupportedDockerImageFormats StringSet
 412         UsePreemptibleInstances     bool
 413
 414         JobsAPI struct {
 415                 Enable         string
 416                 GitInternalDir string
 417         }
 418         Logging struct {
 419                 MaxAge                       Duration
 420                 LogBytesPerEvent             int
 421                 LogSecondsBetweenEvents      Duration
 422                 LogThrottlePeriod            Duration
 423                 LogThrottleBytes             int
 424                 LogThrottleLines             int
 425                 LimitLogBytesPerJob          int
 426                 LogPartialLineThrottlePeriod Duration
 427                 LogUpdatePeriod              Duration
 428                 LogUpdateSize                ByteSize
 429         }
 430         SLURM struct {
 431                 PrioritySpread             int64
 432                 SbatchArgumentsList        []string
 433                 SbatchEnvironmentVariables map[string]string
 434                 Managed                    struct {
 435                         DNSServerConfDir       string
 436                         DNSServerConfTemplate  string
 437                         DNSServerReloadCommand string
 438                         DNSServerUpdateCommand string
 439                         ComputeNodeDomain      string
 440                         ComputeNodeNameservers StringSet
 441                         AssignNodeHostname     string
 442                 }
 443         }
 444 }
 445
 446 type CloudVMsConfig struct {
 447         Enable bool
 448
 449         BootProbeCommand     string
 450         DeployRunnerBinary   string
 451         ImageID              string
 452         MaxCloudOpsPerSecond int
 453         MaxProbesPerSecond   int
 454         PollInterval         Duration
 455         ProbeInterval        Duration
 456         SSHPort              string
 457         SyncInterval         Duration
 458         TimeoutBooting       Duration
 459         TimeoutIdle          Duration
 460         TimeoutProbe         Duration
 461         TimeoutShutdown      Duration
 462         TimeoutSignal        Duration
 463         TimeoutTERM          Duration
 464         ResourceTags         map[string]string
 465         TagKeyPrefix         string
 466
 467         Driver           string
 468         DriverParameters json.RawMessage
 469 }
 470
 471 type InstanceTypeMap map[string]InstanceType
 472
 473 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 474
 475 // UnmarshalJSON handles old config files that provide an array of
 476 // instance types instead of a hash.
 477 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 478         fixup := func(t InstanceType) (InstanceType, error) {
 479                 if t.ProviderType == "" {
 480                         t.ProviderType = t.Name
 481                 }
 482                 if t.Scratch == 0 {
 483                         t.Scratch = t.IncludedScratch + t.AddedScratch
 484                 } else if t.AddedScratch == 0 {
 485                         t.AddedScratch = t.Scratch - t.IncludedScratch
 486                 } else if t.IncludedScratch == 0 {
 487                         t.IncludedScratch = t.Scratch - t.AddedScratch
 488                 }
 489
 490                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 491                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 492                 }
 493                 return t, nil
 494         }
 495
 496         if len(data) > 0 && data[0] == '[' {
 497                 var arr []InstanceType
 498                 err := json.Unmarshal(data, &arr)
 499                 if err != nil {
 500                         return err
 501                 }
 502                 if len(arr) == 0 {
 503                         *it = nil
 504                         return nil
 505                 }
 506                 *it = make(map[string]InstanceType, len(arr))
 507                 for _, t := range arr {
 508                         if _, ok := (*it)[t.Name]; ok {
 509                                 return errDuplicateInstanceTypeName
 510                         }
 511                         t, err := fixup(t)
 512                         if err != nil {
 513                                 return err
 514                         }
 515                         (*it)[t.Name] = t
 516                 }
 517                 return nil
 518         }
 519         var hash map[string]InstanceType
 520         err := json.Unmarshal(data, &hash)
 521         if err != nil {
 522                 return err
 523         }
 524         // Fill in Name field (and ProviderType field, if not
 525         // specified) using hash key.
 526         *it = InstanceTypeMap(hash)
 527         for name, t := range *it {
 528                 t.Name = name
 529                 t, err := fixup(t)
 530                 if err != nil {
 531                         return err
 532                 }
 533                 (*it)[name] = t
 534         }
 535         return nil
 536 }
 537
 538 type StringSet map[string]struct{}
 539
 540 // UnmarshalJSON handles old config files that provide an array of
 541 // instance types instead of a hash.
 542 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 543         if len(data) > 0 && data[0] == '[' {
 544                 var arr []string
 545                 err := json.Unmarshal(data, &arr)
 546                 if err != nil {
 547                         return err
 548                 }
 549                 if len(arr) == 0 {
 550                         *ss = nil
 551                         return nil
 552                 }
 553                 *ss = make(map[string]struct{}, len(arr))
 554                 for _, t := range arr {
 555                         (*ss)[t] = struct{}{}
 556                 }
 557                 return nil
 558         }
 559         var hash map[string]struct{}
 560         err := json.Unmarshal(data, &hash)
 561         if err != nil {
 562                 return err
 563         }
 564         *ss = make(map[string]struct{}, len(hash))
 565         for t := range hash {
 566                 (*ss)[t] = struct{}{}
 567         }
 568
 569         return nil
 570 }
 571
 572 type ServiceName string
 573
 574 const (
 575         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 576         ServiceNameController    ServiceName = "arvados-controller"
 577         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 578         ServiceNameHealth        ServiceName = "arvados-health"
 579         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 580         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 581         ServiceNameWebsocket     ServiceName = "arvados-ws"
 582         ServiceNameKeepbalance   ServiceName = "keep-balance"
 583         ServiceNameKeepweb       ServiceName = "keep-web"
 584         ServiceNameKeepproxy     ServiceName = "keepproxy"
 585         ServiceNameKeepstore     ServiceName = "keepstore"
 586 )
 587
 588 // Map returns all services as a map, suitable for iterating over all
 589 // services or looking up a service by name.
 590 func (svcs Services) Map() map[ServiceName]Service {
 591         return map[ServiceName]Service{
 592                 ServiceNameRailsAPI:      svcs.RailsAPI,
 593                 ServiceNameController:    svcs.Controller,
 594                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 595                 ServiceNameHealth:        svcs.Health,
 596                 ServiceNameWorkbench1:    svcs.Workbench1,
 597                 ServiceNameWorkbench2:    svcs.Workbench2,
 598                 ServiceNameWebsocket:     svcs.Websocket,
 599                 ServiceNameKeepbalance:   svcs.Keepbalance,
 600                 ServiceNameKeepweb:       svcs.WebDAV,
 601                 ServiceNameKeepproxy:     svcs.Keepproxy,
 602                 ServiceNameKeepstore:     svcs.Keepstore,
 603         }
 604 }