sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.arvados.org/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters         map[string]Cluster
  27         AutoReloadConfig bool
  28 }
  29
  30 // GetConfig returns the current system config, loading it from
  31 // configFile if needed.
  32 func GetConfig(configFile string) (*Config, error) {
  33         var cfg Config
  34         err := config.LoadFile(&cfg, configFile)
  35         return &cfg, err
  36 }
  37
  38 // GetCluster returns the cluster ID and config for the given
  39 // cluster, or the default/only configured cluster if clusterID is "".
  40 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  41         if clusterID == "" {
  42                 if len(sc.Clusters) == 0 {
  43                         return nil, fmt.Errorf("no clusters configured")
  44                 } else if len(sc.Clusters) > 1 {
  45                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  46                 } else {
  47                         for id, cc := range sc.Clusters {
  48                                 cc.ClusterID = id
  49                                 return &cc, nil
  50                         }
  51                 }
  52         }
  53         if cc, ok := sc.Clusters[clusterID]; !ok {
  54                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  55         } else {
  56                 cc.ClusterID = clusterID
  57                 return &cc, nil
  58         }
  59 }
  60
  61 type WebDAVCacheConfig struct {
  62         TTL                  Duration
  63         UUIDTTL              Duration
  64         MaxBlockEntries      int
  65         MaxCollectionEntries int
  66         MaxCollectionBytes   int64
  67         MaxPermissionEntries int
  68         MaxUUIDEntries       int
  69 }
  70
  71 type Cluster struct {
  72         ClusterID       string `json:"-"`
  73         ManagementToken string
  74         SystemRootToken string
  75         Services        Services
  76         InstanceTypes   InstanceTypeMap
  77         Containers      ContainersConfig
  78         RemoteClusters  map[string]RemoteCluster
  79         PostgreSQL      PostgreSQL
  80
  81         API struct {
  82                 AsyncPermissionsUpdateInterval Duration
  83                 DisabledAPIs                   StringSet
  84                 MaxIndexDatabaseRead           int
  85                 MaxItemsPerResponse            int
  86                 MaxConcurrentRequests          int
  87                 MaxKeepBlobBuffers             int
  88                 MaxRequestAmplification        int
  89                 MaxRequestSize                 int
  90                 RailsSessionSecretToken        string
  91                 RequestTimeout                 Duration
  92                 SendTimeout                    Duration
  93                 WebsocketClientEventQueue      int
  94                 WebsocketServerEventQueue      int
  95                 KeepServiceRequestTimeout      Duration
  96         }
  97         AuditLogs struct {
  98                 MaxAge             Duration
  99                 MaxDeleteBatch     int
 100                 UnloggedAttributes StringSet
 101         }
 102         Collections struct {
 103                 BlobSigning              bool
 104                 BlobSigningKey           string
 105                 BlobSigningTTL           Duration
 106                 BlobTrash                bool
 107                 BlobTrashLifetime        Duration
 108                 BlobTrashCheckInterval   Duration
 109                 BlobTrashConcurrency     int
 110                 BlobDeleteConcurrency    int
 111                 BlobReplicateConcurrency int
 112                 CollectionVersioning     bool
 113                 DefaultTrashLifetime     Duration
 114                 DefaultReplication       int
 115                 ManagedProperties        map[string]struct {
 116                         Value     interface{}
 117                         Function  string
 118                         Protected bool
 119                 }
 120                 PreserveVersionIfIdle        Duration
 121                 TrashSweepInterval           Duration
 122                 TrustAllContent              bool
 123                 ForwardSlashNameSubstitution string
 124                 S3FolderObjects              bool
 125
 126                 BlobMissingReport        string
 127                 BalancePeriod            Duration
 128                 BalanceCollectionBatch   int
 129                 BalanceCollectionBuffers int
 130                 BalanceTimeout           Duration
 131
 132                 WebDAVCache WebDAVCacheConfig
 133         }
 134         Git struct {
 135                 GitCommand   string
 136                 GitoliteHome string
 137                 Repositories string
 138         }
 139         Login struct {
 140                 LDAP struct {
 141                         Enable             bool
 142                         URL                URL
 143                         StartTLS           bool
 144                         InsecureTLS        bool
 145                         StripDomain        string
 146                         AppendDomain       string
 147                         SearchAttribute    string
 148                         SearchBindUser     string
 149                         SearchBindPassword string
 150                         SearchBase         string
 151                         SearchFilters      string
 152                         EmailAttribute     string
 153                         UsernameAttribute  string
 154                 }
 155                 Google struct {
 156                         Enable                  bool
 157                         ClientID                string
 158                         ClientSecret            string
 159                         AlternateEmailAddresses bool
 160                 }
 161                 OpenIDConnect struct {
 162                         Enable             bool
 163                         Issuer             string
 164                         ClientID           string
 165                         ClientSecret       string
 166                         EmailClaim         string
 167                         EmailVerifiedClaim string
 168                         UsernameClaim      string
 169                 }
 170                 PAM struct {
 171                         Enable             bool
 172                         Service            string
 173                         DefaultEmailDomain string
 174                 }
 175                 SSO struct {
 176                         Enable            bool
 177                         ProviderAppID     string
 178                         ProviderAppSecret string
 179                 }
 180                 Test struct {
 181                         Enable bool
 182                         Users  map[string]TestUser
 183                 }
 184                 LoginCluster       string
 185                 RemoteTokenRefresh Duration
 186                 TokenLifetime      Duration
 187         }
 188         Mail struct {
 189                 MailchimpAPIKey                string
 190                 MailchimpListID                string
 191                 SendUserSetupNotificationEmail bool
 192                 IssueReporterEmailFrom         string
 193                 IssueReporterEmailTo           string
 194                 SupportEmailAddress            string
 195                 EmailFrom                      string
 196         }
 197         SystemLogs struct {
 198                 LogLevel                string
 199                 Format                  string
 200                 MaxRequestLogParamsSize int
 201         }
 202         TLS struct {
 203                 Certificate string
 204                 Key         string
 205                 Insecure    bool
 206         }
 207         Users struct {
 208                 AnonymousUserToken                    string
 209                 AdminNotifierEmailFrom                string
 210                 AutoAdminFirstUser                    bool
 211                 AutoAdminUserWithEmail                string
 212                 AutoSetupNewUsers                     bool
 213                 AutoSetupNewUsersWithRepository       bool
 214                 AutoSetupNewUsersWithVmUUID           string
 215                 AutoSetupUsernameBlacklist            StringSet
 216                 EmailSubjectPrefix                    string
 217                 NewInactiveUserNotificationRecipients StringSet
 218                 NewUserNotificationRecipients         StringSet
 219                 NewUsersAreActive                     bool
 220                 UserNotifierEmailFrom                 string
 221                 UserProfileNotificationAddress        string
 222                 PreferDomainForUsername               string
 223         }
 224         Volumes   map[string]Volume
 225         Workbench struct {
 226                 ActivationContactLink            string
 227                 APIClientConnectTimeout          Duration
 228                 APIClientReceiveTimeout          Duration
 229                 APIResponseCompression           bool
 230                 ApplicationMimetypesWithViewIcon StringSet
 231                 ArvadosDocsite                   string
 232                 ArvadosPublicDataDocURL          string
 233                 DefaultOpenIdPrefix              string
 234                 EnableGettingStartedPopup        bool
 235                 EnablePublicProjectsPage         bool
 236                 FileViewersConfigURL             string
 237                 LogViewerMaxBytes                ByteSize
 238                 MultiSiteSearch                  string
 239                 ProfilingEnabled                 bool
 240                 Repositories                     bool
 241                 RepositoryCache                  string
 242                 RunningJobLogRecordsToFetch      int
 243                 SecretKeyBase                    string
 244                 ShowRecentCollectionsOnDashboard bool
 245                 ShowUserAgreementInline          bool
 246                 ShowUserNotifications            bool
 247                 SiteName                         string
 248                 Theme                            string
 249                 UserProfileFormFields            map[string]struct {
 250                         Type                 string
 251                         FormFieldTitle       string
 252                         FormFieldDescription string
 253                         Required             bool
 254                         Position             int
 255                         Options              map[string]struct{}
 256                 }
 257                 UserProfileFormMessage string
 258                 VocabularyURL          string
 259                 WelcomePageHTML        string
 260                 InactivePageHTML       string
 261                 SSHHelpPageHTML        string
 262                 SSHHelpHostSuffix      string
 263         }
 264
 265         ForceLegacyAPI14 bool
 266 }
 267
 268 type Volume struct {
 269         AccessViaHosts   map[URL]VolumeAccess
 270         ReadOnly         bool
 271         Replication      int
 272         StorageClasses   map[string]bool
 273         Driver           string
 274         DriverParameters json.RawMessage
 275 }
 276
 277 type S3VolumeDriverParameters struct {
 278         IAMRole            string
 279         AccessKey          string
 280         SecretKey          string
 281         Endpoint           string
 282         Region             string
 283         Bucket             string
 284         LocationConstraint bool
 285         V2Signature        bool
 286         UseAWSS3v2Driver   bool
 287         IndexPageSize      int
 288         ConnectTimeout     Duration
 289         ReadTimeout        Duration
 290         RaceWindow         Duration
 291         UnsafeDelete       bool
 292 }
 293
 294 type AzureVolumeDriverParameters struct {
 295         StorageAccountName   string
 296         StorageAccountKey    string
 297         StorageBaseURL       string
 298         ContainerName        string
 299         RequestTimeout       Duration
 300         ListBlobsRetryDelay  Duration
 301         ListBlobsMaxAttempts int
 302 }
 303
 304 type DirectoryVolumeDriverParameters struct {
 305         Root      string
 306         Serialize bool
 307 }
 308
 309 type VolumeAccess struct {
 310         ReadOnly bool
 311 }
 312
 313 type Services struct {
 314         Composer       Service
 315         Controller     Service
 316         DispatchCloud  Service
 317         GitHTTP        Service
 318         GitSSH         Service
 319         Health         Service
 320         Keepbalance    Service
 321         Keepproxy      Service
 322         Keepstore      Service
 323         RailsAPI       Service
 324         SSO            Service
 325         WebDAVDownload Service
 326         WebDAV         Service
 327         WebShell       Service
 328         Websocket      Service
 329         Workbench1     Service
 330         Workbench2     Service
 331 }
 332
 333 type Service struct {
 334         InternalURLs map[URL]ServiceInstance
 335         ExternalURL  URL
 336 }
 337
 338 type TestUser struct {
 339         Email    string
 340         Password string
 341 }
 342
 343 // URL is a url.URL that is also usable as a JSON key/value.
 344 type URL url.URL
 345
 346 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 347 // used as a JSON key/value.
 348 func (su *URL) UnmarshalText(text []byte) error {
 349         u, err := url.Parse(string(text))
 350         if err == nil {
 351                 *su = URL(*u)
 352                 if su.Path == "" && su.Host != "" {
 353                         // http://example really means http://example/
 354                         su.Path = "/"
 355                 }
 356         }
 357         return err
 358 }
 359
 360 func (su URL) MarshalText() ([]byte, error) {
 361         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 362 }
 363
 364 func (su URL) String() string {
 365         return (*url.URL)(&su).String()
 366 }
 367
 368 type ServiceInstance struct {
 369         Rendezvous string `json:",omitempty"`
 370 }
 371
 372 type PostgreSQL struct {
 373         Connection     PostgreSQLConnection
 374         ConnectionPool int
 375 }
 376
 377 type PostgreSQLConnection map[string]string
 378
 379 type RemoteCluster struct {
 380         Host          string
 381         Proxy         bool
 382         Scheme        string
 383         Insecure      bool
 384         ActivateUsers bool
 385 }
 386
 387 type InstanceType struct {
 388         Name            string
 389         ProviderType    string
 390         VCPUs           int
 391         RAM             ByteSize
 392         Scratch         ByteSize
 393         IncludedScratch ByteSize
 394         AddedScratch    ByteSize
 395         Price           float64
 396         Preemptible     bool
 397 }
 398
 399 type ContainersConfig struct {
 400         CloudVMs                    CloudVMsConfig
 401         CrunchRunCommand            string
 402         CrunchRunArgumentsList      []string
 403         DefaultKeepCacheRAM         ByteSize
 404         DispatchPrivateKey          string
 405         LogReuseDecisions           bool
 406         MaxComputeVMs               int
 407         MaxDispatchAttempts         int
 408         MaxRetryAttempts            int
 409         MinRetryPeriod              Duration
 410         ReserveExtraRAM             ByteSize
 411         StaleLockTimeout            Duration
 412         SupportedDockerImageFormats StringSet
 413         UsePreemptibleInstances     bool
 414
 415         JobsAPI struct {
 416                 Enable         string
 417                 GitInternalDir string
 418         }
 419         Logging struct {
 420                 MaxAge                       Duration
 421                 LogBytesPerEvent             int
 422                 LogSecondsBetweenEvents      Duration
 423                 LogThrottlePeriod            Duration
 424                 LogThrottleBytes             int
 425                 LogThrottleLines             int
 426                 LimitLogBytesPerJob          int
 427                 LogPartialLineThrottlePeriod Duration
 428                 LogUpdatePeriod              Duration
 429                 LogUpdateSize                ByteSize
 430         }
 431         SLURM struct {
 432                 PrioritySpread             int64
 433                 SbatchArgumentsList        []string
 434                 SbatchEnvironmentVariables map[string]string
 435                 Managed                    struct {
 436                         DNSServerConfDir       string
 437                         DNSServerConfTemplate  string
 438                         DNSServerReloadCommand string
 439                         DNSServerUpdateCommand string
 440                         ComputeNodeDomain      string
 441                         ComputeNodeNameservers StringSet
 442                         AssignNodeHostname     string
 443                 }
 444         }
 445 }
 446
 447 type CloudVMsConfig struct {
 448         Enable bool
 449
 450         BootProbeCommand     string
 451         DeployRunnerBinary   string
 452         ImageID              string
 453         MaxCloudOpsPerSecond int
 454         MaxProbesPerSecond   int
 455         PollInterval         Duration
 456         ProbeInterval        Duration
 457         SSHPort              string
 458         SyncInterval         Duration
 459         TimeoutBooting       Duration
 460         TimeoutIdle          Duration
 461         TimeoutProbe         Duration
 462         TimeoutShutdown      Duration
 463         TimeoutSignal        Duration
 464         TimeoutTERM          Duration
 465         ResourceTags         map[string]string
 466         TagKeyPrefix         string
 467
 468         Driver           string
 469         DriverParameters json.RawMessage
 470 }
 471
 472 type InstanceTypeMap map[string]InstanceType
 473
 474 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 475
 476 // UnmarshalJSON handles old config files that provide an array of
 477 // instance types instead of a hash.
 478 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 479         fixup := func(t InstanceType) (InstanceType, error) {
 480                 if t.ProviderType == "" {
 481                         t.ProviderType = t.Name
 482                 }
 483                 if t.Scratch == 0 {
 484                         t.Scratch = t.IncludedScratch + t.AddedScratch
 485                 } else if t.AddedScratch == 0 {
 486                         t.AddedScratch = t.Scratch - t.IncludedScratch
 487                 } else if t.IncludedScratch == 0 {
 488                         t.IncludedScratch = t.Scratch - t.AddedScratch
 489                 }
 490
 491                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 492                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 493                 }
 494                 return t, nil
 495         }
 496
 497         if len(data) > 0 && data[0] == '[' {
 498                 var arr []InstanceType
 499                 err := json.Unmarshal(data, &arr)
 500                 if err != nil {
 501                         return err
 502                 }
 503                 if len(arr) == 0 {
 504                         *it = nil
 505                         return nil
 506                 }
 507                 *it = make(map[string]InstanceType, len(arr))
 508                 for _, t := range arr {
 509                         if _, ok := (*it)[t.Name]; ok {
 510                                 return errDuplicateInstanceTypeName
 511                         }
 512                         t, err := fixup(t)
 513                         if err != nil {
 514                                 return err
 515                         }
 516                         (*it)[t.Name] = t
 517                 }
 518                 return nil
 519         }
 520         var hash map[string]InstanceType
 521         err := json.Unmarshal(data, &hash)
 522         if err != nil {
 523                 return err
 524         }
 525         // Fill in Name field (and ProviderType field, if not
 526         // specified) using hash key.
 527         *it = InstanceTypeMap(hash)
 528         for name, t := range *it {
 529                 t.Name = name
 530                 t, err := fixup(t)
 531                 if err != nil {
 532                         return err
 533                 }
 534                 (*it)[name] = t
 535         }
 536         return nil
 537 }
 538
 539 type StringSet map[string]struct{}
 540
 541 // UnmarshalJSON handles old config files that provide an array of
 542 // instance types instead of a hash.
 543 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 544         if len(data) > 0 && data[0] == '[' {
 545                 var arr []string
 546                 err := json.Unmarshal(data, &arr)
 547                 if err != nil {
 548                         return err
 549                 }
 550                 if len(arr) == 0 {
 551                         *ss = nil
 552                         return nil
 553                 }
 554                 *ss = make(map[string]struct{}, len(arr))
 555                 for _, t := range arr {
 556                         (*ss)[t] = struct{}{}
 557                 }
 558                 return nil
 559         }
 560         var hash map[string]struct{}
 561         err := json.Unmarshal(data, &hash)
 562         if err != nil {
 563                 return err
 564         }
 565         *ss = make(map[string]struct{}, len(hash))
 566         for t := range hash {
 567                 (*ss)[t] = struct{}{}
 568         }
 569
 570         return nil
 571 }
 572
 573 type ServiceName string
 574
 575 const (
 576         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 577         ServiceNameController    ServiceName = "arvados-controller"
 578         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 579         ServiceNameHealth        ServiceName = "arvados-health"
 580         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 581         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 582         ServiceNameWebsocket     ServiceName = "arvados-ws"
 583         ServiceNameKeepbalance   ServiceName = "keep-balance"
 584         ServiceNameKeepweb       ServiceName = "keep-web"
 585         ServiceNameKeepproxy     ServiceName = "keepproxy"
 586         ServiceNameKeepstore     ServiceName = "keepstore"
 587 )
 588
 589 // Map returns all services as a map, suitable for iterating over all
 590 // services or looking up a service by name.
 591 func (svcs Services) Map() map[ServiceName]Service {
 592         return map[ServiceName]Service{
 593                 ServiceNameRailsAPI:      svcs.RailsAPI,
 594                 ServiceNameController:    svcs.Controller,
 595                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 596                 ServiceNameHealth:        svcs.Health,
 597                 ServiceNameWorkbench1:    svcs.Workbench1,
 598                 ServiceNameWorkbench2:    svcs.Workbench2,
 599                 ServiceNameWebsocket:     svcs.Websocket,
 600                 ServiceNameKeepbalance:   svcs.Keepbalance,
 601                 ServiceNameKeepweb:       svcs.WebDAV,
 602                 ServiceNameKeepproxy:     svcs.Keepproxy,
 603                 ServiceNameKeepstore:     svcs.Keepstore,
 604         }
 605 }