sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.arvados.org/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters         map[string]Cluster
  27         AutoReloadConfig bool
  28 }
  29
  30 // GetConfig returns the current system config, loading it from
  31 // configFile if needed.
  32 func GetConfig(configFile string) (*Config, error) {
  33         var cfg Config
  34         err := config.LoadFile(&cfg, configFile)
  35         return &cfg, err
  36 }
  37
  38 // GetCluster returns the cluster ID and config for the given
  39 // cluster, or the default/only configured cluster if clusterID is "".
  40 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  41         if clusterID == "" {
  42                 if len(sc.Clusters) == 0 {
  43                         return nil, fmt.Errorf("no clusters configured")
  44                 } else if len(sc.Clusters) > 1 {
  45                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  46                 } else {
  47                         for id, cc := range sc.Clusters {
  48                                 cc.ClusterID = id
  49                                 return &cc, nil
  50                         }
  51                 }
  52         }
  53         if cc, ok := sc.Clusters[clusterID]; !ok {
  54                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  55         } else {
  56                 cc.ClusterID = clusterID
  57                 return &cc, nil
  58         }
  59 }
  60
  61 type WebDAVCacheConfig struct {
  62         TTL                  Duration
  63         UUIDTTL              Duration
  64         MaxBlockEntries      int
  65         MaxCollectionEntries int
  66         MaxCollectionBytes   int64
  67         MaxPermissionEntries int
  68         MaxUUIDEntries       int
  69 }
  70
  71 type Cluster struct {
  72         ClusterID       string `json:"-"`
  73         ManagementToken string
  74         SystemRootToken string
  75         Services        Services
  76         InstanceTypes   InstanceTypeMap
  77         Containers      ContainersConfig
  78         RemoteClusters  map[string]RemoteCluster
  79         PostgreSQL      PostgreSQL
  80
  81         API struct {
  82                 AsyncPermissionsUpdateInterval Duration
  83                 DisabledAPIs                   StringSet
  84                 MaxIndexDatabaseRead           int
  85                 MaxItemsPerResponse            int
  86                 MaxConcurrentRequests          int
  87                 MaxKeepBlobBuffers             int
  88                 MaxRequestAmplification        int
  89                 MaxRequestSize                 int
  90                 RailsSessionSecretToken        string
  91                 RequestTimeout                 Duration
  92                 SendTimeout                    Duration
  93                 WebsocketClientEventQueue      int
  94                 WebsocketServerEventQueue      int
  95                 KeepServiceRequestTimeout      Duration
  96         }
  97         AuditLogs struct {
  98                 MaxAge             Duration
  99                 MaxDeleteBatch     int
 100                 UnloggedAttributes StringSet
 101         }
 102         Collections struct {
 103                 BlobSigning              bool
 104                 BlobSigningKey           string
 105                 BlobSigningTTL           Duration
 106                 BlobTrash                bool
 107                 BlobTrashLifetime        Duration
 108                 BlobTrashCheckInterval   Duration
 109                 BlobTrashConcurrency     int
 110                 BlobDeleteConcurrency    int
 111                 BlobReplicateConcurrency int
 112                 CollectionVersioning     bool
 113                 DefaultTrashLifetime     Duration
 114                 DefaultReplication       int
 115                 ManagedProperties        map[string]struct {
 116                         Value     interface{}
 117                         Function  string
 118                         Protected bool
 119                 }
 120                 PreserveVersionIfIdle        Duration
 121                 TrashSweepInterval           Duration
 122                 TrustAllContent              bool
 123                 ForwardSlashNameSubstitution string
 124
 125                 BlobMissingReport        string
 126                 BalancePeriod            Duration
 127                 BalanceCollectionBatch   int
 128                 BalanceCollectionBuffers int
 129                 BalanceTimeout           Duration
 130
 131                 WebDAVCache WebDAVCacheConfig
 132         }
 133         Git struct {
 134                 GitCommand   string
 135                 GitoliteHome string
 136                 Repositories string
 137         }
 138         Login struct {
 139                 LDAP struct {
 140                         Enable             bool
 141                         URL                URL
 142                         StartTLS           bool
 143                         InsecureTLS        bool
 144                         StripDomain        string
 145                         AppendDomain       string
 146                         SearchAttribute    string
 147                         SearchBindUser     string
 148                         SearchBindPassword string
 149                         SearchBase         string
 150                         SearchFilters      string
 151                         EmailAttribute     string
 152                         UsernameAttribute  string
 153                 }
 154                 Google struct {
 155                         Enable                  bool
 156                         ClientID                string
 157                         ClientSecret            string
 158                         AlternateEmailAddresses bool
 159                 }
 160                 OpenIDConnect struct {
 161                         Enable             bool
 162                         Issuer             string
 163                         ClientID           string
 164                         ClientSecret       string
 165                         EmailClaim         string
 166                         EmailVerifiedClaim string
 167                         UsernameClaim      string
 168                 }
 169                 PAM struct {
 170                         Enable             bool
 171                         Service            string
 172                         DefaultEmailDomain string
 173                 }
 174                 SSO struct {
 175                         Enable            bool
 176                         ProviderAppID     string
 177                         ProviderAppSecret string
 178                 }
 179                 LoginCluster       string
 180                 RemoteTokenRefresh Duration
 181                 TokenLifetime      Duration
 182         }
 183         Mail struct {
 184                 MailchimpAPIKey                string
 185                 MailchimpListID                string
 186                 SendUserSetupNotificationEmail bool
 187                 IssueReporterEmailFrom         string
 188                 IssueReporterEmailTo           string
 189                 SupportEmailAddress            string
 190                 EmailFrom                      string
 191         }
 192         SystemLogs struct {
 193                 LogLevel                string
 194                 Format                  string
 195                 MaxRequestLogParamsSize int
 196         }
 197         TLS struct {
 198                 Certificate string
 199                 Key         string
 200                 Insecure    bool
 201         }
 202         Users struct {
 203                 AnonymousUserToken                    string
 204                 AdminNotifierEmailFrom                string
 205                 AutoAdminFirstUser                    bool
 206                 AutoAdminUserWithEmail                string
 207                 AutoSetupNewUsers                     bool
 208                 AutoSetupNewUsersWithRepository       bool
 209                 AutoSetupNewUsersWithVmUUID           string
 210                 AutoSetupUsernameBlacklist            StringSet
 211                 EmailSubjectPrefix                    string
 212                 NewInactiveUserNotificationRecipients StringSet
 213                 NewUserNotificationRecipients         StringSet
 214                 NewUsersAreActive                     bool
 215                 UserNotifierEmailFrom                 string
 216                 UserProfileNotificationAddress        string
 217                 PreferDomainForUsername               string
 218         }
 219         Volumes   map[string]Volume
 220         Workbench struct {
 221                 ActivationContactLink            string
 222                 APIClientConnectTimeout          Duration
 223                 APIClientReceiveTimeout          Duration
 224                 APIResponseCompression           bool
 225                 ApplicationMimetypesWithViewIcon StringSet
 226                 ArvadosDocsite                   string
 227                 ArvadosPublicDataDocURL          string
 228                 DefaultOpenIdPrefix              string
 229                 EnableGettingStartedPopup        bool
 230                 EnablePublicProjectsPage         bool
 231                 FileViewersConfigURL             string
 232                 LogViewerMaxBytes                ByteSize
 233                 MultiSiteSearch                  string
 234                 ProfilingEnabled                 bool
 235                 Repositories                     bool
 236                 RepositoryCache                  string
 237                 RunningJobLogRecordsToFetch      int
 238                 SecretKeyBase                    string
 239                 ShowRecentCollectionsOnDashboard bool
 240                 ShowUserAgreementInline          bool
 241                 ShowUserNotifications            bool
 242                 SiteName                         string
 243                 Theme                            string
 244                 UserProfileFormFields            map[string]struct {
 245                         Type                 string
 246                         FormFieldTitle       string
 247                         FormFieldDescription string
 248                         Required             bool
 249                         Position             int
 250                         Options              map[string]struct{}
 251                 }
 252                 UserProfileFormMessage string
 253                 VocabularyURL          string
 254                 WelcomePageHTML        string
 255                 InactivePageHTML       string
 256                 SSHHelpPageHTML        string
 257                 SSHHelpHostSuffix      string
 258         }
 259
 260         ForceLegacyAPI14 bool
 261 }
 262
 263 type Volume struct {
 264         AccessViaHosts   map[URL]VolumeAccess
 265         ReadOnly         bool
 266         Replication      int
 267         StorageClasses   map[string]bool
 268         Driver           string
 269         DriverParameters json.RawMessage
 270 }
 271
 272 type S3VolumeDriverParameters struct {
 273         IAMRole            string
 274         AccessKey          string
 275         SecretKey          string
 276         Endpoint           string
 277         Region             string
 278         Bucket             string
 279         LocationConstraint bool
 280         V2Signature        bool
 281         UseAWSS3v2Driver   bool
 282         IndexPageSize      int
 283         ConnectTimeout     Duration
 284         ReadTimeout        Duration
 285         RaceWindow         Duration
 286         UnsafeDelete       bool
 287 }
 288
 289 type AzureVolumeDriverParameters struct {
 290         StorageAccountName   string
 291         StorageAccountKey    string
 292         StorageBaseURL       string
 293         ContainerName        string
 294         RequestTimeout       Duration
 295         ListBlobsRetryDelay  Duration
 296         ListBlobsMaxAttempts int
 297 }
 298
 299 type DirectoryVolumeDriverParameters struct {
 300         Root      string
 301         Serialize bool
 302 }
 303
 304 type VolumeAccess struct {
 305         ReadOnly bool
 306 }
 307
 308 type Services struct {
 309         Composer       Service
 310         Controller     Service
 311         DispatchCloud  Service
 312         GitHTTP        Service
 313         GitSSH         Service
 314         Health         Service
 315         Keepbalance    Service
 316         Keepproxy      Service
 317         Keepstore      Service
 318         Nodemanager    Service
 319         RailsAPI       Service
 320         SSO            Service
 321         WebDAVDownload Service
 322         WebDAV         Service
 323         WebShell       Service
 324         Websocket      Service
 325         Workbench1     Service
 326         Workbench2     Service
 327 }
 328
 329 type Service struct {
 330         InternalURLs map[URL]ServiceInstance
 331         ExternalURL  URL
 332 }
 333
 334 // URL is a url.URL that is also usable as a JSON key/value.
 335 type URL url.URL
 336
 337 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 338 // used as a JSON key/value.
 339 func (su *URL) UnmarshalText(text []byte) error {
 340         u, err := url.Parse(string(text))
 341         if err == nil {
 342                 *su = URL(*u)
 343                 if su.Path == "" && su.Host != "" {
 344                         // http://example really means http://example/
 345                         su.Path = "/"
 346                 }
 347         }
 348         return err
 349 }
 350
 351 func (su URL) MarshalText() ([]byte, error) {
 352         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 353 }
 354
 355 func (su URL) String() string {
 356         return (*url.URL)(&su).String()
 357 }
 358
 359 type ServiceInstance struct {
 360         Rendezvous string `json:",omitempty"`
 361 }
 362
 363 type PostgreSQL struct {
 364         Connection     PostgreSQLConnection
 365         ConnectionPool int
 366 }
 367
 368 type PostgreSQLConnection map[string]string
 369
 370 type RemoteCluster struct {
 371         Host          string
 372         Proxy         bool
 373         Scheme        string
 374         Insecure      bool
 375         ActivateUsers bool
 376 }
 377
 378 type InstanceType struct {
 379         Name            string
 380         ProviderType    string
 381         VCPUs           int
 382         RAM             ByteSize
 383         Scratch         ByteSize
 384         IncludedScratch ByteSize
 385         AddedScratch    ByteSize
 386         Price           float64
 387         Preemptible     bool
 388 }
 389
 390 type ContainersConfig struct {
 391         CloudVMs                    CloudVMsConfig
 392         CrunchRunCommand            string
 393         CrunchRunArgumentsList      []string
 394         DefaultKeepCacheRAM         ByteSize
 395         DispatchPrivateKey          string
 396         LogReuseDecisions           bool
 397         MaxComputeVMs               int
 398         MaxDispatchAttempts         int
 399         MaxRetryAttempts            int
 400         MinRetryPeriod              Duration
 401         ReserveExtraRAM             ByteSize
 402         StaleLockTimeout            Duration
 403         SupportedDockerImageFormats StringSet
 404         UsePreemptibleInstances     bool
 405
 406         JobsAPI struct {
 407                 Enable         string
 408                 GitInternalDir string
 409         }
 410         Logging struct {
 411                 MaxAge                       Duration
 412                 LogBytesPerEvent             int
 413                 LogSecondsBetweenEvents      Duration
 414                 LogThrottlePeriod            Duration
 415                 LogThrottleBytes             int
 416                 LogThrottleLines             int
 417                 LimitLogBytesPerJob          int
 418                 LogPartialLineThrottlePeriod Duration
 419                 LogUpdatePeriod              Duration
 420                 LogUpdateSize                ByteSize
 421         }
 422         SLURM struct {
 423                 PrioritySpread             int64
 424                 SbatchArgumentsList        []string
 425                 SbatchEnvironmentVariables map[string]string
 426                 Managed                    struct {
 427                         DNSServerConfDir       string
 428                         DNSServerConfTemplate  string
 429                         DNSServerReloadCommand string
 430                         DNSServerUpdateCommand string
 431                         ComputeNodeDomain      string
 432                         ComputeNodeNameservers StringSet
 433                         AssignNodeHostname     string
 434                 }
 435         }
 436 }
 437
 438 type CloudVMsConfig struct {
 439         Enable bool
 440
 441         BootProbeCommand     string
 442         DeployRunnerBinary   string
 443         ImageID              string
 444         MaxCloudOpsPerSecond int
 445         MaxProbesPerSecond   int
 446         PollInterval         Duration
 447         ProbeInterval        Duration
 448         SSHPort              string
 449         SyncInterval         Duration
 450         TimeoutBooting       Duration
 451         TimeoutIdle          Duration
 452         TimeoutProbe         Duration
 453         TimeoutShutdown      Duration
 454         TimeoutSignal        Duration
 455         TimeoutTERM          Duration
 456         ResourceTags         map[string]string
 457         TagKeyPrefix         string
 458
 459         Driver           string
 460         DriverParameters json.RawMessage
 461 }
 462
 463 type InstanceTypeMap map[string]InstanceType
 464
 465 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 466
 467 // UnmarshalJSON handles old config files that provide an array of
 468 // instance types instead of a hash.
 469 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 470         fixup := func(t InstanceType) (InstanceType, error) {
 471                 if t.ProviderType == "" {
 472                         t.ProviderType = t.Name
 473                 }
 474                 if t.Scratch == 0 {
 475                         t.Scratch = t.IncludedScratch + t.AddedScratch
 476                 } else if t.AddedScratch == 0 {
 477                         t.AddedScratch = t.Scratch - t.IncludedScratch
 478                 } else if t.IncludedScratch == 0 {
 479                         t.IncludedScratch = t.Scratch - t.AddedScratch
 480                 }
 481
 482                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 483                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 484                 }
 485                 return t, nil
 486         }
 487
 488         if len(data) > 0 && data[0] == '[' {
 489                 var arr []InstanceType
 490                 err := json.Unmarshal(data, &arr)
 491                 if err != nil {
 492                         return err
 493                 }
 494                 if len(arr) == 0 {
 495                         *it = nil
 496                         return nil
 497                 }
 498                 *it = make(map[string]InstanceType, len(arr))
 499                 for _, t := range arr {
 500                         if _, ok := (*it)[t.Name]; ok {
 501                                 return errDuplicateInstanceTypeName
 502                         }
 503                         t, err := fixup(t)
 504                         if err != nil {
 505                                 return err
 506                         }
 507                         (*it)[t.Name] = t
 508                 }
 509                 return nil
 510         }
 511         var hash map[string]InstanceType
 512         err := json.Unmarshal(data, &hash)
 513         if err != nil {
 514                 return err
 515         }
 516         // Fill in Name field (and ProviderType field, if not
 517         // specified) using hash key.
 518         *it = InstanceTypeMap(hash)
 519         for name, t := range *it {
 520                 t.Name = name
 521                 t, err := fixup(t)
 522                 if err != nil {
 523                         return err
 524                 }
 525                 (*it)[name] = t
 526         }
 527         return nil
 528 }
 529
 530 type StringSet map[string]struct{}
 531
 532 // UnmarshalJSON handles old config files that provide an array of
 533 // instance types instead of a hash.
 534 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 535         if len(data) > 0 && data[0] == '[' {
 536                 var arr []string
 537                 err := json.Unmarshal(data, &arr)
 538                 if err != nil {
 539                         return err
 540                 }
 541                 if len(arr) == 0 {
 542                         *ss = nil
 543                         return nil
 544                 }
 545                 *ss = make(map[string]struct{}, len(arr))
 546                 for _, t := range arr {
 547                         (*ss)[t] = struct{}{}
 548                 }
 549                 return nil
 550         }
 551         var hash map[string]struct{}
 552         err := json.Unmarshal(data, &hash)
 553         if err != nil {
 554                 return err
 555         }
 556         *ss = make(map[string]struct{}, len(hash))
 557         for t := range hash {
 558                 (*ss)[t] = struct{}{}
 559         }
 560
 561         return nil
 562 }
 563
 564 type ServiceName string
 565
 566 const (
 567         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 568         ServiceNameController    ServiceName = "arvados-controller"
 569         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 570         ServiceNameHealth        ServiceName = "arvados-health"
 571         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 572         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 573         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 574         ServiceNameWebsocket     ServiceName = "arvados-ws"
 575         ServiceNameKeepbalance   ServiceName = "keep-balance"
 576         ServiceNameKeepweb       ServiceName = "keep-web"
 577         ServiceNameKeepproxy     ServiceName = "keepproxy"
 578         ServiceNameKeepstore     ServiceName = "keepstore"
 579 )
 580
 581 // Map returns all services as a map, suitable for iterating over all
 582 // services or looking up a service by name.
 583 func (svcs Services) Map() map[ServiceName]Service {
 584         return map[ServiceName]Service{
 585                 ServiceNameRailsAPI:      svcs.RailsAPI,
 586                 ServiceNameController:    svcs.Controller,
 587                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 588                 ServiceNameHealth:        svcs.Health,
 589                 ServiceNameNodemanager:   svcs.Nodemanager,
 590                 ServiceNameWorkbench1:    svcs.Workbench1,
 591                 ServiceNameWorkbench2:    svcs.Workbench2,
 592                 ServiceNameWebsocket:     svcs.Websocket,
 593                 ServiceNameKeepbalance:   svcs.Keepbalance,
 594                 ServiceNameKeepweb:       svcs.WebDAV,
 595                 ServiceNameKeepproxy:     svcs.Keepproxy,
 596                 ServiceNameKeepstore:     svcs.Keepstore,
 597         }
 598 }