sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.arvados.org/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters         map[string]Cluster
  27         AutoReloadConfig bool
  28 }
  29
  30 // GetConfig returns the current system config, loading it from
  31 // configFile if needed.
  32 func GetConfig(configFile string) (*Config, error) {
  33         var cfg Config
  34         err := config.LoadFile(&cfg, configFile)
  35         return &cfg, err
  36 }
  37
  38 // GetCluster returns the cluster ID and config for the given
  39 // cluster, or the default/only configured cluster if clusterID is "".
  40 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  41         if clusterID == "" {
  42                 if len(sc.Clusters) == 0 {
  43                         return nil, fmt.Errorf("no clusters configured")
  44                 } else if len(sc.Clusters) > 1 {
  45                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  46                 } else {
  47                         for id, cc := range sc.Clusters {
  48                                 cc.ClusterID = id
  49                                 return &cc, nil
  50                         }
  51                 }
  52         }
  53         if cc, ok := sc.Clusters[clusterID]; !ok {
  54                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  55         } else {
  56                 cc.ClusterID = clusterID
  57                 return &cc, nil
  58         }
  59 }
  60
  61 type WebDAVCacheConfig struct {
  62         TTL                  Duration
  63         UUIDTTL              Duration
  64         MaxBlockEntries      int
  65         MaxCollectionEntries int
  66         MaxCollectionBytes   int64
  67         MaxPermissionEntries int
  68         MaxUUIDEntries       int
  69 }
  70
  71 type Cluster struct {
  72         ClusterID       string `json:"-"`
  73         ManagementToken string
  74         SystemRootToken string
  75         Services        Services
  76         InstanceTypes   InstanceTypeMap
  77         Containers      ContainersConfig
  78         RemoteClusters  map[string]RemoteCluster
  79         PostgreSQL      PostgreSQL
  80
  81         API struct {
  82                 AsyncPermissionsUpdateInterval Duration
  83                 DisabledAPIs                   StringSet
  84                 MaxIndexDatabaseRead           int
  85                 MaxItemsPerResponse            int
  86                 MaxConcurrentRequests          int
  87                 MaxKeepBlobBuffers             int
  88                 MaxRequestAmplification        int
  89                 MaxRequestSize                 int
  90                 RailsSessionSecretToken        string
  91                 RequestTimeout                 Duration
  92                 SendTimeout                    Duration
  93                 WebsocketClientEventQueue      int
  94                 WebsocketServerEventQueue      int
  95                 KeepServiceRequestTimeout      Duration
  96         }
  97         AuditLogs struct {
  98                 MaxAge             Duration
  99                 MaxDeleteBatch     int
 100                 UnloggedAttributes StringSet
 101         }
 102         Collections struct {
 103                 BlobSigning              bool
 104                 BlobSigningKey           string
 105                 BlobSigningTTL           Duration
 106                 BlobTrash                bool
 107                 BlobTrashLifetime        Duration
 108                 BlobTrashCheckInterval   Duration
 109                 BlobTrashConcurrency     int
 110                 BlobDeleteConcurrency    int
 111                 BlobReplicateConcurrency int
 112                 CollectionVersioning     bool
 113                 DefaultTrashLifetime     Duration
 114                 DefaultReplication       int
 115                 ManagedProperties        map[string]struct {
 116                         Value     interface{}
 117                         Function  string
 118                         Protected bool
 119                 }
 120                 PreserveVersionIfIdle        Duration
 121                 TrashSweepInterval           Duration
 122                 TrustAllContent              bool
 123                 ForwardSlashNameSubstitution string
 124
 125                 BlobMissingReport        string
 126                 BalancePeriod            Duration
 127                 BalanceCollectionBatch   int
 128                 BalanceCollectionBuffers int
 129                 BalanceTimeout           Duration
 130
 131                 WebDAVCache WebDAVCacheConfig
 132         }
 133         Git struct {
 134                 GitCommand   string
 135                 GitoliteHome string
 136                 Repositories string
 137         }
 138         Login struct {
 139                 LDAP struct {
 140                         Enable             bool
 141                         URL                URL
 142                         StartTLS           bool
 143                         InsecureTLS        bool
 144                         StripDomain        string
 145                         AppendDomain       string
 146                         SearchAttribute    string
 147                         SearchBindUser     string
 148                         SearchBindPassword string
 149                         SearchBase         string
 150                         SearchFilters      string
 151                         EmailAttribute     string
 152                         UsernameAttribute  string
 153                 }
 154                 Google struct {
 155                         Enable                  bool
 156                         ClientID                string
 157                         ClientSecret            string
 158                         AlternateEmailAddresses bool
 159                 }
 160                 OpenIDConnect struct {
 161                         Enable             bool
 162                         Issuer             string
 163                         ClientID           string
 164                         ClientSecret       string
 165                         EmailClaim         string
 166                         EmailVerifiedClaim string
 167                         UsernameClaim      string
 168                 }
 169                 PAM struct {
 170                         Enable             bool
 171                         Service            string
 172                         DefaultEmailDomain string
 173                 }
 174                 SSO struct {
 175                         Enable            bool
 176                         ProviderAppID     string
 177                         ProviderAppSecret string
 178                 }
 179                 LoginCluster       string
 180                 RemoteTokenRefresh Duration
 181         }
 182         Mail struct {
 183                 MailchimpAPIKey                string
 184                 MailchimpListID                string
 185                 SendUserSetupNotificationEmail bool
 186                 IssueReporterEmailFrom         string
 187                 IssueReporterEmailTo           string
 188                 SupportEmailAddress            string
 189                 EmailFrom                      string
 190         }
 191         SystemLogs struct {
 192                 LogLevel                string
 193                 Format                  string
 194                 MaxRequestLogParamsSize int
 195         }
 196         TLS struct {
 197                 Certificate string
 198                 Key         string
 199                 Insecure    bool
 200         }
 201         Users struct {
 202                 AnonymousUserToken                    string
 203                 AdminNotifierEmailFrom                string
 204                 AutoAdminFirstUser                    bool
 205                 AutoAdminUserWithEmail                string
 206                 AutoSetupNewUsers                     bool
 207                 AutoSetupNewUsersWithRepository       bool
 208                 AutoSetupNewUsersWithVmUUID           string
 209                 AutoSetupUsernameBlacklist            StringSet
 210                 EmailSubjectPrefix                    string
 211                 NewInactiveUserNotificationRecipients StringSet
 212                 NewUserNotificationRecipients         StringSet
 213                 NewUsersAreActive                     bool
 214                 UserNotifierEmailFrom                 string
 215                 UserProfileNotificationAddress        string
 216                 PreferDomainForUsername               string
 217         }
 218         Volumes   map[string]Volume
 219         Workbench struct {
 220                 ActivationContactLink            string
 221                 APIClientConnectTimeout          Duration
 222                 APIClientReceiveTimeout          Duration
 223                 APIResponseCompression           bool
 224                 ApplicationMimetypesWithViewIcon StringSet
 225                 ArvadosDocsite                   string
 226                 ArvadosPublicDataDocURL          string
 227                 DefaultOpenIdPrefix              string
 228                 EnableGettingStartedPopup        bool
 229                 EnablePublicProjectsPage         bool
 230                 FileViewersConfigURL             string
 231                 LogViewerMaxBytes                ByteSize
 232                 MultiSiteSearch                  string
 233                 ProfilingEnabled                 bool
 234                 Repositories                     bool
 235                 RepositoryCache                  string
 236                 RunningJobLogRecordsToFetch      int
 237                 SecretKeyBase                    string
 238                 ShowRecentCollectionsOnDashboard bool
 239                 ShowUserAgreementInline          bool
 240                 ShowUserNotifications            bool
 241                 SiteName                         string
 242                 Theme                            string
 243                 UserProfileFormFields            map[string]struct {
 244                         Type                 string
 245                         FormFieldTitle       string
 246                         FormFieldDescription string
 247                         Required             bool
 248                         Position             int
 249                         Options              map[string]struct{}
 250                 }
 251                 UserProfileFormMessage string
 252                 VocabularyURL          string
 253                 WelcomePageHTML        string
 254                 InactivePageHTML       string
 255                 SSHHelpPageHTML        string
 256                 SSHHelpHostSuffix      string
 257         }
 258
 259         ForceLegacyAPI14 bool
 260 }
 261
 262 type Volume struct {
 263         AccessViaHosts   map[URL]VolumeAccess
 264         ReadOnly         bool
 265         Replication      int
 266         StorageClasses   map[string]bool
 267         Driver           string
 268         DriverParameters json.RawMessage
 269 }
 270
 271 type S3VolumeDriverParameters struct {
 272         IAMRole            string
 273         AccessKey          string
 274         SecretKey          string
 275         Endpoint           string
 276         Region             string
 277         Bucket             string
 278         LocationConstraint bool
 279         V2Signature        bool
 280         UseAWSS3v2Driver   bool
 281         IndexPageSize      int
 282         ConnectTimeout     Duration
 283         ReadTimeout        Duration
 284         RaceWindow         Duration
 285         UnsafeDelete       bool
 286 }
 287
 288 type AzureVolumeDriverParameters struct {
 289         StorageAccountName   string
 290         StorageAccountKey    string
 291         StorageBaseURL       string
 292         ContainerName        string
 293         RequestTimeout       Duration
 294         ListBlobsRetryDelay  Duration
 295         ListBlobsMaxAttempts int
 296 }
 297
 298 type DirectoryVolumeDriverParameters struct {
 299         Root      string
 300         Serialize bool
 301 }
 302
 303 type VolumeAccess struct {
 304         ReadOnly bool
 305 }
 306
 307 type Services struct {
 308         Composer       Service
 309         Controller     Service
 310         DispatchCloud  Service
 311         GitHTTP        Service
 312         GitSSH         Service
 313         Health         Service
 314         Keepbalance    Service
 315         Keepproxy      Service
 316         Keepstore      Service
 317         RailsAPI       Service
 318         SSO            Service
 319         WebDAVDownload Service
 320         WebDAV         Service
 321         WebShell       Service
 322         Websocket      Service
 323         Workbench1     Service
 324         Workbench2     Service
 325 }
 326
 327 type Service struct {
 328         InternalURLs map[URL]ServiceInstance
 329         ExternalURL  URL
 330 }
 331
 332 // URL is a url.URL that is also usable as a JSON key/value.
 333 type URL url.URL
 334
 335 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 336 // used as a JSON key/value.
 337 func (su *URL) UnmarshalText(text []byte) error {
 338         u, err := url.Parse(string(text))
 339         if err == nil {
 340                 *su = URL(*u)
 341                 if su.Path == "" && su.Host != "" {
 342                         // http://example really means http://example/
 343                         su.Path = "/"
 344                 }
 345         }
 346         return err
 347 }
 348
 349 func (su URL) MarshalText() ([]byte, error) {
 350         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 351 }
 352
 353 func (su URL) String() string {
 354         return (*url.URL)(&su).String()
 355 }
 356
 357 type ServiceInstance struct {
 358         Rendezvous string `json:",omitempty"`
 359 }
 360
 361 type PostgreSQL struct {
 362         Connection     PostgreSQLConnection
 363         ConnectionPool int
 364 }
 365
 366 type PostgreSQLConnection map[string]string
 367
 368 type RemoteCluster struct {
 369         Host          string
 370         Proxy         bool
 371         Scheme        string
 372         Insecure      bool
 373         ActivateUsers bool
 374 }
 375
 376 type InstanceType struct {
 377         Name            string
 378         ProviderType    string
 379         VCPUs           int
 380         RAM             ByteSize
 381         Scratch         ByteSize
 382         IncludedScratch ByteSize
 383         AddedScratch    ByteSize
 384         Price           float64
 385         Preemptible     bool
 386 }
 387
 388 type ContainersConfig struct {
 389         CloudVMs                    CloudVMsConfig
 390         CrunchRunCommand            string
 391         CrunchRunArgumentsList      []string
 392         DefaultKeepCacheRAM         ByteSize
 393         DispatchPrivateKey          string
 394         LogReuseDecisions           bool
 395         MaxComputeVMs               int
 396         MaxDispatchAttempts         int
 397         MaxRetryAttempts            int
 398         MinRetryPeriod              Duration
 399         ReserveExtraRAM             ByteSize
 400         StaleLockTimeout            Duration
 401         SupportedDockerImageFormats StringSet
 402         UsePreemptibleInstances     bool
 403
 404         JobsAPI struct {
 405                 Enable         string
 406                 GitInternalDir string
 407         }
 408         Logging struct {
 409                 MaxAge                       Duration
 410                 LogBytesPerEvent             int
 411                 LogSecondsBetweenEvents      Duration
 412                 LogThrottlePeriod            Duration
 413                 LogThrottleBytes             int
 414                 LogThrottleLines             int
 415                 LimitLogBytesPerJob          int
 416                 LogPartialLineThrottlePeriod Duration
 417                 LogUpdatePeriod              Duration
 418                 LogUpdateSize                ByteSize
 419         }
 420         SLURM struct {
 421                 PrioritySpread             int64
 422                 SbatchArgumentsList        []string
 423                 SbatchEnvironmentVariables map[string]string
 424                 Managed                    struct {
 425                         DNSServerConfDir       string
 426                         DNSServerConfTemplate  string
 427                         DNSServerReloadCommand string
 428                         DNSServerUpdateCommand string
 429                         ComputeNodeDomain      string
 430                         ComputeNodeNameservers StringSet
 431                         AssignNodeHostname     string
 432                 }
 433         }
 434 }
 435
 436 type CloudVMsConfig struct {
 437         Enable bool
 438
 439         BootProbeCommand     string
 440         DeployRunnerBinary   string
 441         ImageID              string
 442         MaxCloudOpsPerSecond int
 443         MaxProbesPerSecond   int
 444         PollInterval         Duration
 445         ProbeInterval        Duration
 446         SSHPort              string
 447         SyncInterval         Duration
 448         TimeoutBooting       Duration
 449         TimeoutIdle          Duration
 450         TimeoutProbe         Duration
 451         TimeoutShutdown      Duration
 452         TimeoutSignal        Duration
 453         TimeoutTERM          Duration
 454         ResourceTags         map[string]string
 455         TagKeyPrefix         string
 456
 457         Driver           string
 458         DriverParameters json.RawMessage
 459 }
 460
 461 type InstanceTypeMap map[string]InstanceType
 462
 463 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 464
 465 // UnmarshalJSON handles old config files that provide an array of
 466 // instance types instead of a hash.
 467 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 468         fixup := func(t InstanceType) (InstanceType, error) {
 469                 if t.ProviderType == "" {
 470                         t.ProviderType = t.Name
 471                 }
 472                 if t.Scratch == 0 {
 473                         t.Scratch = t.IncludedScratch + t.AddedScratch
 474                 } else if t.AddedScratch == 0 {
 475                         t.AddedScratch = t.Scratch - t.IncludedScratch
 476                 } else if t.IncludedScratch == 0 {
 477                         t.IncludedScratch = t.Scratch - t.AddedScratch
 478                 }
 479
 480                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 481                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 482                 }
 483                 return t, nil
 484         }
 485
 486         if len(data) > 0 && data[0] == '[' {
 487                 var arr []InstanceType
 488                 err := json.Unmarshal(data, &arr)
 489                 if err != nil {
 490                         return err
 491                 }
 492                 if len(arr) == 0 {
 493                         *it = nil
 494                         return nil
 495                 }
 496                 *it = make(map[string]InstanceType, len(arr))
 497                 for _, t := range arr {
 498                         if _, ok := (*it)[t.Name]; ok {
 499                                 return errDuplicateInstanceTypeName
 500                         }
 501                         t, err := fixup(t)
 502                         if err != nil {
 503                                 return err
 504                         }
 505                         (*it)[t.Name] = t
 506                 }
 507                 return nil
 508         }
 509         var hash map[string]InstanceType
 510         err := json.Unmarshal(data, &hash)
 511         if err != nil {
 512                 return err
 513         }
 514         // Fill in Name field (and ProviderType field, if not
 515         // specified) using hash key.
 516         *it = InstanceTypeMap(hash)
 517         for name, t := range *it {
 518                 t.Name = name
 519                 t, err := fixup(t)
 520                 if err != nil {
 521                         return err
 522                 }
 523                 (*it)[name] = t
 524         }
 525         return nil
 526 }
 527
 528 type StringSet map[string]struct{}
 529
 530 // UnmarshalJSON handles old config files that provide an array of
 531 // instance types instead of a hash.
 532 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 533         if len(data) > 0 && data[0] == '[' {
 534                 var arr []string
 535                 err := json.Unmarshal(data, &arr)
 536                 if err != nil {
 537                         return err
 538                 }
 539                 if len(arr) == 0 {
 540                         *ss = nil
 541                         return nil
 542                 }
 543                 *ss = make(map[string]struct{}, len(arr))
 544                 for _, t := range arr {
 545                         (*ss)[t] = struct{}{}
 546                 }
 547                 return nil
 548         }
 549         var hash map[string]struct{}
 550         err := json.Unmarshal(data, &hash)
 551         if err != nil {
 552                 return err
 553         }
 554         *ss = make(map[string]struct{}, len(hash))
 555         for t := range hash {
 556                 (*ss)[t] = struct{}{}
 557         }
 558
 559         return nil
 560 }
 561
 562 type ServiceName string
 563
 564 const (
 565         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 566         ServiceNameController    ServiceName = "arvados-controller"
 567         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 568         ServiceNameHealth        ServiceName = "arvados-health"
 569         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 570         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 571         ServiceNameWebsocket     ServiceName = "arvados-ws"
 572         ServiceNameKeepbalance   ServiceName = "keep-balance"
 573         ServiceNameKeepweb       ServiceName = "keep-web"
 574         ServiceNameKeepproxy     ServiceName = "keepproxy"
 575         ServiceNameKeepstore     ServiceName = "keepstore"
 576 )
 577
 578 // Map returns all services as a map, suitable for iterating over all
 579 // services or looking up a service by name.
 580 func (svcs Services) Map() map[ServiceName]Service {
 581         return map[ServiceName]Service{
 582                 ServiceNameRailsAPI:      svcs.RailsAPI,
 583                 ServiceNameController:    svcs.Controller,
 584                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 585                 ServiceNameHealth:        svcs.Health,
 586                 ServiceNameWorkbench1:    svcs.Workbench1,
 587                 ServiceNameWorkbench2:    svcs.Workbench2,
 588                 ServiceNameWebsocket:     svcs.Websocket,
 589                 ServiceNameKeepbalance:   svcs.Keepbalance,
 590                 ServiceNameKeepweb:       svcs.WebDAV,
 591                 ServiceNameKeepproxy:     svcs.Keepproxy,
 592                 ServiceNameKeepstore:     svcs.Keepstore,
 593         }
 594 }