sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.arvados.org/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters         map[string]Cluster
  27         AutoReloadConfig bool
  28 }
  29
  30 // GetConfig returns the current system config, loading it from
  31 // configFile if needed.
  32 func GetConfig(configFile string) (*Config, error) {
  33         var cfg Config
  34         err := config.LoadFile(&cfg, configFile)
  35         return &cfg, err
  36 }
  37
  38 // GetCluster returns the cluster ID and config for the given
  39 // cluster, or the default/only configured cluster if clusterID is "".
  40 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  41         if clusterID == "" {
  42                 if len(sc.Clusters) == 0 {
  43                         return nil, fmt.Errorf("no clusters configured")
  44                 } else if len(sc.Clusters) > 1 {
  45                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  46                 } else {
  47                         for id, cc := range sc.Clusters {
  48                                 cc.ClusterID = id
  49                                 return &cc, nil
  50                         }
  51                 }
  52         }
  53         if cc, ok := sc.Clusters[clusterID]; !ok {
  54                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  55         } else {
  56                 cc.ClusterID = clusterID
  57                 return &cc, nil
  58         }
  59 }
  60
  61 type WebDAVCacheConfig struct {
  62         TTL                  Duration
  63         UUIDTTL              Duration
  64         MaxBlockEntries      int
  65         MaxCollectionEntries int
  66         MaxCollectionBytes   int64
  67         MaxPermissionEntries int
  68         MaxUUIDEntries       int
  69 }
  70
  71 type Cluster struct {
  72         ClusterID       string `json:"-"`
  73         ManagementToken string
  74         SystemRootToken string
  75         Services        Services
  76         InstanceTypes   InstanceTypeMap
  77         Containers      ContainersConfig
  78         RemoteClusters  map[string]RemoteCluster
  79         PostgreSQL      PostgreSQL
  80
  81         API struct {
  82                 AsyncPermissionsUpdateInterval Duration
  83                 DisabledAPIs                   StringSet
  84                 MaxIndexDatabaseRead           int
  85                 MaxItemsPerResponse            int
  86                 MaxConcurrentRequests          int
  87                 MaxKeepBlobBuffers             int
  88                 MaxRequestAmplification        int
  89                 MaxRequestSize                 int
  90                 RailsSessionSecretToken        string
  91                 RequestTimeout                 Duration
  92                 SendTimeout                    Duration
  93                 WebsocketClientEventQueue      int
  94                 WebsocketServerEventQueue      int
  95                 KeepServiceRequestTimeout      Duration
  96         }
  97         AuditLogs struct {
  98                 MaxAge             Duration
  99                 MaxDeleteBatch     int
 100                 UnloggedAttributes StringSet
 101         }
 102         Collections struct {
 103                 BlobSigning              bool
 104                 BlobSigningKey           string
 105                 BlobSigningTTL           Duration
 106                 BlobTrash                bool
 107                 BlobTrashLifetime        Duration
 108                 BlobTrashCheckInterval   Duration
 109                 BlobTrashConcurrency     int
 110                 BlobDeleteConcurrency    int
 111                 BlobReplicateConcurrency int
 112                 CollectionVersioning     bool
 113                 DefaultTrashLifetime     Duration
 114                 DefaultReplication       int
 115                 ManagedProperties        map[string]struct {
 116                         Value     interface{}
 117                         Function  string
 118                         Protected bool
 119                 }
 120                 PreserveVersionIfIdle        Duration
 121                 TrashSweepInterval           Duration
 122                 TrustAllContent              bool
 123                 ForwardSlashNameSubstitution string
 124                 S3FolderObjects              bool
 125
 126                 BlobMissingReport        string
 127                 BalancePeriod            Duration
 128                 BalanceCollectionBatch   int
 129                 BalanceCollectionBuffers int
 130                 BalanceTimeout           Duration
 131
 132                 WebDAVCache WebDAVCacheConfig
 133         }
 134         Git struct {
 135                 GitCommand   string
 136                 GitoliteHome string
 137                 Repositories string
 138         }
 139         Login struct {
 140                 LDAP struct {
 141                         Enable             bool
 142                         URL                URL
 143                         StartTLS           bool
 144                         InsecureTLS        bool
 145                         StripDomain        string
 146                         AppendDomain       string
 147                         SearchAttribute    string
 148                         SearchBindUser     string
 149                         SearchBindPassword string
 150                         SearchBase         string
 151                         SearchFilters      string
 152                         EmailAttribute     string
 153                         UsernameAttribute  string
 154                 }
 155                 Google struct {
 156                         Enable                  bool
 157                         ClientID                string
 158                         ClientSecret            string
 159                         AlternateEmailAddresses bool
 160                 }
 161                 OpenIDConnect struct {
 162                         Enable             bool
 163                         Issuer             string
 164                         ClientID           string
 165                         ClientSecret       string
 166                         EmailClaim         string
 167                         EmailVerifiedClaim string
 168                         UsernameClaim      string
 169                 }
 170                 PAM struct {
 171                         Enable             bool
 172                         Service            string
 173                         DefaultEmailDomain string
 174                 }
 175                 SSO struct {
 176                         Enable            bool
 177                         ProviderAppID     string
 178                         ProviderAppSecret string
 179                 }
 180                 LoginCluster       string
 181                 RemoteTokenRefresh Duration
 182                 TokenLifetime      Duration
 183         }
 184         Mail struct {
 185                 MailchimpAPIKey                string
 186                 MailchimpListID                string
 187                 SendUserSetupNotificationEmail bool
 188                 IssueReporterEmailFrom         string
 189                 IssueReporterEmailTo           string
 190                 SupportEmailAddress            string
 191                 EmailFrom                      string
 192         }
 193         SystemLogs struct {
 194                 LogLevel                string
 195                 Format                  string
 196                 MaxRequestLogParamsSize int
 197         }
 198         TLS struct {
 199                 Certificate string
 200                 Key         string
 201                 Insecure    bool
 202         }
 203         Users struct {
 204                 AnonymousUserToken                    string
 205                 AdminNotifierEmailFrom                string
 206                 AutoAdminFirstUser                    bool
 207                 AutoAdminUserWithEmail                string
 208                 AutoSetupNewUsers                     bool
 209                 AutoSetupNewUsersWithRepository       bool
 210                 AutoSetupNewUsersWithVmUUID           string
 211                 AutoSetupUsernameBlacklist            StringSet
 212                 EmailSubjectPrefix                    string
 213                 NewInactiveUserNotificationRecipients StringSet
 214                 NewUserNotificationRecipients         StringSet
 215                 NewUsersAreActive                     bool
 216                 UserNotifierEmailFrom                 string
 217                 UserProfileNotificationAddress        string
 218                 PreferDomainForUsername               string
 219         }
 220         Volumes   map[string]Volume
 221         Workbench struct {
 222                 ActivationContactLink            string
 223                 APIClientConnectTimeout          Duration
 224                 APIClientReceiveTimeout          Duration
 225                 APIResponseCompression           bool
 226                 ApplicationMimetypesWithViewIcon StringSet
 227                 ArvadosDocsite                   string
 228                 ArvadosPublicDataDocURL          string
 229                 DefaultOpenIdPrefix              string
 230                 EnableGettingStartedPopup        bool
 231                 EnablePublicProjectsPage         bool
 232                 FileViewersConfigURL             string
 233                 LogViewerMaxBytes                ByteSize
 234                 MultiSiteSearch                  string
 235                 ProfilingEnabled                 bool
 236                 Repositories                     bool
 237                 RepositoryCache                  string
 238                 RunningJobLogRecordsToFetch      int
 239                 SecretKeyBase                    string
 240                 ShowRecentCollectionsOnDashboard bool
 241                 ShowUserAgreementInline          bool
 242                 ShowUserNotifications            bool
 243                 SiteName                         string
 244                 Theme                            string
 245                 UserProfileFormFields            map[string]struct {
 246                         Type                 string
 247                         FormFieldTitle       string
 248                         FormFieldDescription string
 249                         Required             bool
 250                         Position             int
 251                         Options              map[string]struct{}
 252                 }
 253                 UserProfileFormMessage string
 254                 VocabularyURL          string
 255                 WelcomePageHTML        string
 256                 InactivePageHTML       string
 257                 SSHHelpPageHTML        string
 258                 SSHHelpHostSuffix      string
 259         }
 260
 261         ForceLegacyAPI14 bool
 262 }
 263
 264 type Volume struct {
 265         AccessViaHosts   map[URL]VolumeAccess
 266         ReadOnly         bool
 267         Replication      int
 268         StorageClasses   map[string]bool
 269         Driver           string
 270         DriverParameters json.RawMessage
 271 }
 272
 273 type S3VolumeDriverParameters struct {
 274         IAMRole            string
 275         AccessKey          string
 276         SecretKey          string
 277         Endpoint           string
 278         Region             string
 279         Bucket             string
 280         LocationConstraint bool
 281         V2Signature        bool
 282         UseAWSS3v2Driver   bool
 283         IndexPageSize      int
 284         ConnectTimeout     Duration
 285         ReadTimeout        Duration
 286         RaceWindow         Duration
 287         UnsafeDelete       bool
 288 }
 289
 290 type AzureVolumeDriverParameters struct {
 291         StorageAccountName   string
 292         StorageAccountKey    string
 293         StorageBaseURL       string
 294         ContainerName        string
 295         RequestTimeout       Duration
 296         ListBlobsRetryDelay  Duration
 297         ListBlobsMaxAttempts int
 298 }
 299
 300 type DirectoryVolumeDriverParameters struct {
 301         Root      string
 302         Serialize bool
 303 }
 304
 305 type VolumeAccess struct {
 306         ReadOnly bool
 307 }
 308
 309 type Services struct {
 310         Composer       Service
 311         Controller     Service
 312         DispatchCloud  Service
 313         GitHTTP        Service
 314         GitSSH         Service
 315         Health         Service
 316         Keepbalance    Service
 317         Keepproxy      Service
 318         Keepstore      Service
 319         RailsAPI       Service
 320         SSO            Service
 321         WebDAVDownload Service
 322         WebDAV         Service
 323         WebShell       Service
 324         Websocket      Service
 325         Workbench1     Service
 326         Workbench2     Service
 327 }
 328
 329 type Service struct {
 330         InternalURLs map[URL]ServiceInstance
 331         ExternalURL  URL
 332 }
 333
 334 // URL is a url.URL that is also usable as a JSON key/value.
 335 type URL url.URL
 336
 337 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 338 // used as a JSON key/value.
 339 func (su *URL) UnmarshalText(text []byte) error {
 340         u, err := url.Parse(string(text))
 341         if err == nil {
 342                 *su = URL(*u)
 343                 if su.Path == "" && su.Host != "" {
 344                         // http://example really means http://example/
 345                         su.Path = "/"
 346                 }
 347         }
 348         return err
 349 }
 350
 351 func (su URL) MarshalText() ([]byte, error) {
 352         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 353 }
 354
 355 func (su URL) String() string {
 356         return (*url.URL)(&su).String()
 357 }
 358
 359 type ServiceInstance struct {
 360         Rendezvous string `json:",omitempty"`
 361 }
 362
 363 type PostgreSQL struct {
 364         Connection     PostgreSQLConnection
 365         ConnectionPool int
 366 }
 367
 368 type PostgreSQLConnection map[string]string
 369
 370 type RemoteCluster struct {
 371         Host          string
 372         Proxy         bool
 373         Scheme        string
 374         Insecure      bool
 375         ActivateUsers bool
 376 }
 377
 378 type InstanceType struct {
 379         Name            string
 380         ProviderType    string
 381         VCPUs           int
 382         RAM             ByteSize
 383         Scratch         ByteSize
 384         IncludedScratch ByteSize
 385         AddedScratch    ByteSize
 386         Price           float64
 387         Preemptible     bool
 388 }
 389
 390 type ContainersConfig struct {
 391         CloudVMs                    CloudVMsConfig
 392         CrunchRunCommand            string
 393         CrunchRunArgumentsList      []string
 394         DefaultKeepCacheRAM         ByteSize
 395         DispatchPrivateKey          string
 396         LogReuseDecisions           bool
 397         MaxComputeVMs               int
 398         MaxDispatchAttempts         int
 399         MaxRetryAttempts            int
 400         MinRetryPeriod              Duration
 401         ReserveExtraRAM             ByteSize
 402         StaleLockTimeout            Duration
 403         SupportedDockerImageFormats StringSet
 404         UsePreemptibleInstances     bool
 405
 406         JobsAPI struct {
 407                 Enable         string
 408                 GitInternalDir string
 409         }
 410         Logging struct {
 411                 MaxAge                       Duration
 412                 LogBytesPerEvent             int
 413                 LogSecondsBetweenEvents      Duration
 414                 LogThrottlePeriod            Duration
 415                 LogThrottleBytes             int
 416                 LogThrottleLines             int
 417                 LimitLogBytesPerJob          int
 418                 LogPartialLineThrottlePeriod Duration
 419                 LogUpdatePeriod              Duration
 420                 LogUpdateSize                ByteSize
 421         }
 422         SLURM struct {
 423                 PrioritySpread             int64
 424                 SbatchArgumentsList        []string
 425                 SbatchEnvironmentVariables map[string]string
 426                 Managed                    struct {
 427                         DNSServerConfDir       string
 428                         DNSServerConfTemplate  string
 429                         DNSServerReloadCommand string
 430                         DNSServerUpdateCommand string
 431                         ComputeNodeDomain      string
 432                         ComputeNodeNameservers StringSet
 433                         AssignNodeHostname     string
 434                 }
 435         }
 436 }
 437
 438 type CloudVMsConfig struct {
 439         Enable bool
 440
 441         BootProbeCommand     string
 442         DeployRunnerBinary   string
 443         ImageID              string
 444         MaxCloudOpsPerSecond int
 445         MaxProbesPerSecond   int
 446         PollInterval         Duration
 447         ProbeInterval        Duration
 448         SSHPort              string
 449         SyncInterval         Duration
 450         TimeoutBooting       Duration
 451         TimeoutIdle          Duration
 452         TimeoutProbe         Duration
 453         TimeoutShutdown      Duration
 454         TimeoutSignal        Duration
 455         TimeoutTERM          Duration
 456         ResourceTags         map[string]string
 457         TagKeyPrefix         string
 458
 459         Driver           string
 460         DriverParameters json.RawMessage
 461 }
 462
 463 type InstanceTypeMap map[string]InstanceType
 464
 465 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 466
 467 // UnmarshalJSON handles old config files that provide an array of
 468 // instance types instead of a hash.
 469 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 470         fixup := func(t InstanceType) (InstanceType, error) {
 471                 if t.ProviderType == "" {
 472                         t.ProviderType = t.Name
 473                 }
 474                 if t.Scratch == 0 {
 475                         t.Scratch = t.IncludedScratch + t.AddedScratch
 476                 } else if t.AddedScratch == 0 {
 477                         t.AddedScratch = t.Scratch - t.IncludedScratch
 478                 } else if t.IncludedScratch == 0 {
 479                         t.IncludedScratch = t.Scratch - t.AddedScratch
 480                 }
 481
 482                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 483                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 484                 }
 485                 return t, nil
 486         }
 487
 488         if len(data) > 0 && data[0] == '[' {
 489                 var arr []InstanceType
 490                 err := json.Unmarshal(data, &arr)
 491                 if err != nil {
 492                         return err
 493                 }
 494                 if len(arr) == 0 {
 495                         *it = nil
 496                         return nil
 497                 }
 498                 *it = make(map[string]InstanceType, len(arr))
 499                 for _, t := range arr {
 500                         if _, ok := (*it)[t.Name]; ok {
 501                                 return errDuplicateInstanceTypeName
 502                         }
 503                         t, err := fixup(t)
 504                         if err != nil {
 505                                 return err
 506                         }
 507                         (*it)[t.Name] = t
 508                 }
 509                 return nil
 510         }
 511         var hash map[string]InstanceType
 512         err := json.Unmarshal(data, &hash)
 513         if err != nil {
 514                 return err
 515         }
 516         // Fill in Name field (and ProviderType field, if not
 517         // specified) using hash key.
 518         *it = InstanceTypeMap(hash)
 519         for name, t := range *it {
 520                 t.Name = name
 521                 t, err := fixup(t)
 522                 if err != nil {
 523                         return err
 524                 }
 525                 (*it)[name] = t
 526         }
 527         return nil
 528 }
 529
 530 type StringSet map[string]struct{}
 531
 532 // UnmarshalJSON handles old config files that provide an array of
 533 // instance types instead of a hash.
 534 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 535         if len(data) > 0 && data[0] == '[' {
 536                 var arr []string
 537                 err := json.Unmarshal(data, &arr)
 538                 if err != nil {
 539                         return err
 540                 }
 541                 if len(arr) == 0 {
 542                         *ss = nil
 543                         return nil
 544                 }
 545                 *ss = make(map[string]struct{}, len(arr))
 546                 for _, t := range arr {
 547                         (*ss)[t] = struct{}{}
 548                 }
 549                 return nil
 550         }
 551         var hash map[string]struct{}
 552         err := json.Unmarshal(data, &hash)
 553         if err != nil {
 554                 return err
 555         }
 556         *ss = make(map[string]struct{}, len(hash))
 557         for t := range hash {
 558                 (*ss)[t] = struct{}{}
 559         }
 560
 561         return nil
 562 }
 563
 564 type ServiceName string
 565
 566 const (
 567         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 568         ServiceNameController    ServiceName = "arvados-controller"
 569         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 570         ServiceNameHealth        ServiceName = "arvados-health"
 571         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 572         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 573         ServiceNameWebsocket     ServiceName = "arvados-ws"
 574         ServiceNameKeepbalance   ServiceName = "keep-balance"
 575         ServiceNameKeepweb       ServiceName = "keep-web"
 576         ServiceNameKeepproxy     ServiceName = "keepproxy"
 577         ServiceNameKeepstore     ServiceName = "keepstore"
 578 )
 579
 580 // Map returns all services as a map, suitable for iterating over all
 581 // services or looking up a service by name.
 582 func (svcs Services) Map() map[ServiceName]Service {
 583         return map[ServiceName]Service{
 584                 ServiceNameRailsAPI:      svcs.RailsAPI,
 585                 ServiceNameController:    svcs.Controller,
 586                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 587                 ServiceNameHealth:        svcs.Health,
 588                 ServiceNameWorkbench1:    svcs.Workbench1,
 589                 ServiceNameWorkbench2:    svcs.Workbench2,
 590                 ServiceNameWebsocket:     svcs.Websocket,
 591                 ServiceNameKeepbalance:   svcs.Keepbalance,
 592                 ServiceNameKeepweb:       svcs.WebDAV,
 593                 ServiceNameKeepproxy:     svcs.Keepproxy,
 594                 ServiceNameKeepstore:     svcs.Keepstore,
 595         }
 596 }