sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.arvados.org/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters         map[string]Cluster
  27         AutoReloadConfig bool
  28 }
  29
  30 // GetConfig returns the current system config, loading it from
  31 // configFile if needed.
  32 func GetConfig(configFile string) (*Config, error) {
  33         var cfg Config
  34         err := config.LoadFile(&cfg, configFile)
  35         return &cfg, err
  36 }
  37
  38 // GetCluster returns the cluster ID and config for the given
  39 // cluster, or the default/only configured cluster if clusterID is "".
  40 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  41         if clusterID == "" {
  42                 if len(sc.Clusters) == 0 {
  43                         return nil, fmt.Errorf("no clusters configured")
  44                 } else if len(sc.Clusters) > 1 {
  45                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  46                 } else {
  47                         for id, cc := range sc.Clusters {
  48                                 cc.ClusterID = id
  49                                 return &cc, nil
  50                         }
  51                 }
  52         }
  53         if cc, ok := sc.Clusters[clusterID]; !ok {
  54                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  55         } else {
  56                 cc.ClusterID = clusterID
  57                 return &cc, nil
  58         }
  59 }
  60
  61 type WebDAVCacheConfig struct {
  62         TTL                  Duration
  63         UUIDTTL              Duration
  64         MaxBlockEntries      int
  65         MaxCollectionEntries int
  66         MaxCollectionBytes   int64
  67         MaxPermissionEntries int
  68         MaxUUIDEntries       int
  69 }
  70
  71 type Cluster struct {
  72         ClusterID       string `json:"-"`
  73         ManagementToken string
  74         SystemRootToken string
  75         Services        Services
  76         InstanceTypes   InstanceTypeMap
  77         Containers      ContainersConfig
  78         RemoteClusters  map[string]RemoteCluster
  79         PostgreSQL      PostgreSQL
  80
  81         API struct {
  82                 AsyncPermissionsUpdateInterval Duration
  83                 DisabledAPIs                   StringSet
  84                 MaxIndexDatabaseRead           int
  85                 MaxItemsPerResponse            int
  86                 MaxConcurrentRequests          int
  87                 MaxKeepBlobBuffers             int
  88                 MaxRequestAmplification        int
  89                 MaxRequestSize                 int
  90                 RailsSessionSecretToken        string
  91                 RequestTimeout                 Duration
  92                 SendTimeout                    Duration
  93                 WebsocketClientEventQueue      int
  94                 WebsocketServerEventQueue      int
  95                 KeepServiceRequestTimeout      Duration
  96         }
  97         AuditLogs struct {
  98                 MaxAge             Duration
  99                 MaxDeleteBatch     int
 100                 UnloggedAttributes StringSet
 101         }
 102         Collections struct {
 103                 BlobSigning              bool
 104                 BlobSigningKey           string
 105                 BlobSigningTTL           Duration
 106                 BlobTrash                bool
 107                 BlobTrashLifetime        Duration
 108                 BlobTrashCheckInterval   Duration
 109                 BlobTrashConcurrency     int
 110                 BlobDeleteConcurrency    int
 111                 BlobReplicateConcurrency int
 112                 CollectionVersioning     bool
 113                 DefaultTrashLifetime     Duration
 114                 DefaultReplication       int
 115                 ManagedProperties        map[string]struct {
 116                         Value     interface{}
 117                         Function  string
 118                         Protected bool
 119                 }
 120                 PreserveVersionIfIdle        Duration
 121                 TrashSweepInterval           Duration
 122                 TrustAllContent              bool
 123                 ForwardSlashNameSubstitution string
 124
 125                 BlobMissingReport        string
 126                 BalancePeriod            Duration
 127                 BalanceCollectionBatch   int
 128                 BalanceCollectionBuffers int
 129                 BalanceTimeout           Duration
 130
 131                 WebDAVCache WebDAVCacheConfig
 132         }
 133         Git struct {
 134                 GitCommand   string
 135                 GitoliteHome string
 136                 Repositories string
 137         }
 138         Login struct {
 139                 LDAP struct {
 140                         Enable             bool
 141                         URL                URL
 142                         StartTLS           bool
 143                         InsecureTLS        bool
 144                         StripDomain        string
 145                         AppendDomain       string
 146                         SearchAttribute    string
 147                         SearchBindUser     string
 148                         SearchBindPassword string
 149                         SearchBase         string
 150                         SearchFilters      string
 151                         EmailAttribute     string
 152                         UsernameAttribute  string
 153                 }
 154                 Google struct {
 155                         Enable                  bool
 156                         ClientID                string
 157                         ClientSecret            string
 158                         AlternateEmailAddresses bool
 159                 }
 160                 OpenIDConnect struct {
 161                         Enable             bool
 162                         Issuer             string
 163                         ClientID           string
 164                         ClientSecret       string
 165                         EmailClaim         string
 166                         EmailVerifiedClaim string
 167                         UsernameClaim      string
 168                 }
 169                 PAM struct {
 170                         Enable             bool
 171                         Service            string
 172                         DefaultEmailDomain string
 173                 }
 174                 SSO struct {
 175                         Enable            bool
 176                         ProviderAppID     string
 177                         ProviderAppSecret string
 178                 }
 179                 LoginCluster       string
 180                 RemoteTokenRefresh Duration
 181         }
 182         Mail struct {
 183                 MailchimpAPIKey                string
 184                 MailchimpListID                string
 185                 SendUserSetupNotificationEmail bool
 186                 IssueReporterEmailFrom         string
 187                 IssueReporterEmailTo           string
 188                 SupportEmailAddress            string
 189                 EmailFrom                      string
 190         }
 191         SystemLogs struct {
 192                 LogLevel                string
 193                 Format                  string
 194                 MaxRequestLogParamsSize int
 195         }
 196         TLS struct {
 197                 Certificate string
 198                 Key         string
 199                 Insecure    bool
 200         }
 201         Users struct {
 202                 AnonymousUserToken                    string
 203                 AdminNotifierEmailFrom                string
 204                 AutoAdminFirstUser                    bool
 205                 AutoAdminUserWithEmail                string
 206                 AutoSetupNewUsers                     bool
 207                 AutoSetupNewUsersWithRepository       bool
 208                 AutoSetupNewUsersWithVmUUID           string
 209                 AutoSetupUsernameBlacklist            StringSet
 210                 EmailSubjectPrefix                    string
 211                 NewInactiveUserNotificationRecipients StringSet
 212                 NewUserNotificationRecipients         StringSet
 213                 NewUsersAreActive                     bool
 214                 UserNotifierEmailFrom                 string
 215                 UserProfileNotificationAddress        string
 216                 PreferDomainForUsername               string
 217         }
 218         Volumes   map[string]Volume
 219         Workbench struct {
 220                 ActivationContactLink            string
 221                 APIClientConnectTimeout          Duration
 222                 APIClientReceiveTimeout          Duration
 223                 APIResponseCompression           bool
 224                 ApplicationMimetypesWithViewIcon StringSet
 225                 ArvadosDocsite                   string
 226                 ArvadosPublicDataDocURL          string
 227                 DefaultOpenIdPrefix              string
 228                 EnableGettingStartedPopup        bool
 229                 EnablePublicProjectsPage         bool
 230                 FileViewersConfigURL             string
 231                 LogViewerMaxBytes                ByteSize
 232                 MultiSiteSearch                  string
 233                 ProfilingEnabled                 bool
 234                 Repositories                     bool
 235                 RepositoryCache                  string
 236                 RunningJobLogRecordsToFetch      int
 237                 SecretKeyBase                    string
 238                 ShowRecentCollectionsOnDashboard bool
 239                 ShowUserAgreementInline          bool
 240                 ShowUserNotifications            bool
 241                 SiteName                         string
 242                 Theme                            string
 243                 UserProfileFormFields            map[string]struct {
 244                         Type                 string
 245                         FormFieldTitle       string
 246                         FormFieldDescription string
 247                         Required             bool
 248                         Position             int
 249                         Options              map[string]struct{}
 250                 }
 251                 UserProfileFormMessage string
 252                 VocabularyURL          string
 253                 WelcomePageHTML        string
 254                 InactivePageHTML       string
 255                 SSHHelpPageHTML        string
 256                 SSHHelpHostSuffix      string
 257         }
 258
 259         ForceLegacyAPI14 bool
 260 }
 261
 262 type Volume struct {
 263         AccessViaHosts   map[URL]VolumeAccess
 264         ReadOnly         bool
 265         Replication      int
 266         StorageClasses   map[string]bool
 267         Driver           string
 268         DriverParameters json.RawMessage
 269 }
 270
 271 type S3VolumeDriverParameters struct {
 272         IAMRole            string
 273         AccessKey          string
 274         SecretKey          string
 275         Endpoint           string
 276         Region             string
 277         Bucket             string
 278         LocationConstraint bool
 279         V2Signature        bool
 280         UseAWSS3v2Driver   bool
 281         IndexPageSize      int
 282         ConnectTimeout     Duration
 283         ReadTimeout        Duration
 284         RaceWindow         Duration
 285         UnsafeDelete       bool
 286 }
 287
 288 type AzureVolumeDriverParameters struct {
 289         StorageAccountName   string
 290         StorageAccountKey    string
 291         StorageBaseURL       string
 292         ContainerName        string
 293         RequestTimeout       Duration
 294         ListBlobsRetryDelay  Duration
 295         ListBlobsMaxAttempts int
 296 }
 297
 298 type DirectoryVolumeDriverParameters struct {
 299         Root      string
 300         Serialize bool
 301 }
 302
 303 type VolumeAccess struct {
 304         ReadOnly bool
 305 }
 306
 307 type Services struct {
 308         Composer       Service
 309         Controller     Service
 310         DispatchCloud  Service
 311         GitHTTP        Service
 312         GitSSH         Service
 313         Health         Service
 314         Keepbalance    Service
 315         Keepproxy      Service
 316         Keepstore      Service
 317         Nodemanager    Service
 318         RailsAPI       Service
 319         SSO            Service
 320         WebDAVDownload Service
 321         WebDAV         Service
 322         WebShell       Service
 323         Websocket      Service
 324         Workbench1     Service
 325         Workbench2     Service
 326 }
 327
 328 type Service struct {
 329         InternalURLs map[URL]ServiceInstance
 330         ExternalURL  URL
 331 }
 332
 333 // URL is a url.URL that is also usable as a JSON key/value.
 334 type URL url.URL
 335
 336 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 337 // used as a JSON key/value.
 338 func (su *URL) UnmarshalText(text []byte) error {
 339         u, err := url.Parse(string(text))
 340         if err == nil {
 341                 *su = URL(*u)
 342                 if su.Path == "" && su.Host != "" {
 343                         // http://example really means http://example/
 344                         su.Path = "/"
 345                 }
 346         }
 347         return err
 348 }
 349
 350 func (su URL) MarshalText() ([]byte, error) {
 351         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 352 }
 353
 354 func (su URL) String() string {
 355         return (*url.URL)(&su).String()
 356 }
 357
 358 type ServiceInstance struct {
 359         Rendezvous string `json:",omitempty"`
 360 }
 361
 362 type PostgreSQL struct {
 363         Connection     PostgreSQLConnection
 364         ConnectionPool int
 365 }
 366
 367 type PostgreSQLConnection map[string]string
 368
 369 type RemoteCluster struct {
 370         Host          string
 371         Proxy         bool
 372         Scheme        string
 373         Insecure      bool
 374         ActivateUsers bool
 375 }
 376
 377 type InstanceType struct {
 378         Name            string
 379         ProviderType    string
 380         VCPUs           int
 381         RAM             ByteSize
 382         Scratch         ByteSize
 383         IncludedScratch ByteSize
 384         AddedScratch    ByteSize
 385         Price           float64
 386         Preemptible     bool
 387 }
 388
 389 type ContainersConfig struct {
 390         CloudVMs                    CloudVMsConfig
 391         CrunchRunCommand            string
 392         CrunchRunArgumentsList      []string
 393         DefaultKeepCacheRAM         ByteSize
 394         DispatchPrivateKey          string
 395         LogReuseDecisions           bool
 396         MaxComputeVMs               int
 397         MaxDispatchAttempts         int
 398         MaxRetryAttempts            int
 399         MinRetryPeriod              Duration
 400         ReserveExtraRAM             ByteSize
 401         StaleLockTimeout            Duration
 402         SupportedDockerImageFormats StringSet
 403         UsePreemptibleInstances     bool
 404
 405         JobsAPI struct {
 406                 Enable         string
 407                 GitInternalDir string
 408         }
 409         Logging struct {
 410                 MaxAge                       Duration
 411                 LogBytesPerEvent             int
 412                 LogSecondsBetweenEvents      Duration
 413                 LogThrottlePeriod            Duration
 414                 LogThrottleBytes             int
 415                 LogThrottleLines             int
 416                 LimitLogBytesPerJob          int
 417                 LogPartialLineThrottlePeriod Duration
 418                 LogUpdatePeriod              Duration
 419                 LogUpdateSize                ByteSize
 420         }
 421         SLURM struct {
 422                 PrioritySpread             int64
 423                 SbatchArgumentsList        []string
 424                 SbatchEnvironmentVariables map[string]string
 425                 Managed                    struct {
 426                         DNSServerConfDir       string
 427                         DNSServerConfTemplate  string
 428                         DNSServerReloadCommand string
 429                         DNSServerUpdateCommand string
 430                         ComputeNodeDomain      string
 431                         ComputeNodeNameservers StringSet
 432                         AssignNodeHostname     string
 433                 }
 434         }
 435 }
 436
 437 type CloudVMsConfig struct {
 438         Enable bool
 439
 440         BootProbeCommand     string
 441         DeployRunnerBinary   string
 442         ImageID              string
 443         MaxCloudOpsPerSecond int
 444         MaxProbesPerSecond   int
 445         PollInterval         Duration
 446         ProbeInterval        Duration
 447         SSHPort              string
 448         SyncInterval         Duration
 449         TimeoutBooting       Duration
 450         TimeoutIdle          Duration
 451         TimeoutProbe         Duration
 452         TimeoutShutdown      Duration
 453         TimeoutSignal        Duration
 454         TimeoutTERM          Duration
 455         ResourceTags         map[string]string
 456         TagKeyPrefix         string
 457
 458         Driver           string
 459         DriverParameters json.RawMessage
 460 }
 461
 462 type InstanceTypeMap map[string]InstanceType
 463
 464 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 465
 466 // UnmarshalJSON handles old config files that provide an array of
 467 // instance types instead of a hash.
 468 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 469         fixup := func(t InstanceType) (InstanceType, error) {
 470                 if t.ProviderType == "" {
 471                         t.ProviderType = t.Name
 472                 }
 473                 if t.Scratch == 0 {
 474                         t.Scratch = t.IncludedScratch + t.AddedScratch
 475                 } else if t.AddedScratch == 0 {
 476                         t.AddedScratch = t.Scratch - t.IncludedScratch
 477                 } else if t.IncludedScratch == 0 {
 478                         t.IncludedScratch = t.Scratch - t.AddedScratch
 479                 }
 480
 481                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 482                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 483                 }
 484                 return t, nil
 485         }
 486
 487         if len(data) > 0 && data[0] == '[' {
 488                 var arr []InstanceType
 489                 err := json.Unmarshal(data, &arr)
 490                 if err != nil {
 491                         return err
 492                 }
 493                 if len(arr) == 0 {
 494                         *it = nil
 495                         return nil
 496                 }
 497                 *it = make(map[string]InstanceType, len(arr))
 498                 for _, t := range arr {
 499                         if _, ok := (*it)[t.Name]; ok {
 500                                 return errDuplicateInstanceTypeName
 501                         }
 502                         t, err := fixup(t)
 503                         if err != nil {
 504                                 return err
 505                         }
 506                         (*it)[t.Name] = t
 507                 }
 508                 return nil
 509         }
 510         var hash map[string]InstanceType
 511         err := json.Unmarshal(data, &hash)
 512         if err != nil {
 513                 return err
 514         }
 515         // Fill in Name field (and ProviderType field, if not
 516         // specified) using hash key.
 517         *it = InstanceTypeMap(hash)
 518         for name, t := range *it {
 519                 t.Name = name
 520                 t, err := fixup(t)
 521                 if err != nil {
 522                         return err
 523                 }
 524                 (*it)[name] = t
 525         }
 526         return nil
 527 }
 528
 529 type StringSet map[string]struct{}
 530
 531 // UnmarshalJSON handles old config files that provide an array of
 532 // instance types instead of a hash.
 533 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 534         if len(data) > 0 && data[0] == '[' {
 535                 var arr []string
 536                 err := json.Unmarshal(data, &arr)
 537                 if err != nil {
 538                         return err
 539                 }
 540                 if len(arr) == 0 {
 541                         *ss = nil
 542                         return nil
 543                 }
 544                 *ss = make(map[string]struct{}, len(arr))
 545                 for _, t := range arr {
 546                         (*ss)[t] = struct{}{}
 547                 }
 548                 return nil
 549         }
 550         var hash map[string]struct{}
 551         err := json.Unmarshal(data, &hash)
 552         if err != nil {
 553                 return err
 554         }
 555         *ss = make(map[string]struct{}, len(hash))
 556         for t := range hash {
 557                 (*ss)[t] = struct{}{}
 558         }
 559
 560         return nil
 561 }
 562
 563 type ServiceName string
 564
 565 const (
 566         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 567         ServiceNameController    ServiceName = "arvados-controller"
 568         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 569         ServiceNameHealth        ServiceName = "arvados-health"
 570         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 571         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 572         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 573         ServiceNameWebsocket     ServiceName = "arvados-ws"
 574         ServiceNameKeepbalance   ServiceName = "keep-balance"
 575         ServiceNameKeepweb       ServiceName = "keep-web"
 576         ServiceNameKeepproxy     ServiceName = "keepproxy"
 577         ServiceNameKeepstore     ServiceName = "keepstore"
 578 )
 579
 580 // Map returns all services as a map, suitable for iterating over all
 581 // services or looking up a service by name.
 582 func (svcs Services) Map() map[ServiceName]Service {
 583         return map[ServiceName]Service{
 584                 ServiceNameRailsAPI:      svcs.RailsAPI,
 585                 ServiceNameController:    svcs.Controller,
 586                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 587                 ServiceNameHealth:        svcs.Health,
 588                 ServiceNameNodemanager:   svcs.Nodemanager,
 589                 ServiceNameWorkbench1:    svcs.Workbench1,
 590                 ServiceNameWorkbench2:    svcs.Workbench2,
 591                 ServiceNameWebsocket:     svcs.Websocket,
 592                 ServiceNameKeepbalance:   svcs.Keepbalance,
 593                 ServiceNameKeepweb:       svcs.WebDAV,
 594                 ServiceNameKeepproxy:     svcs.Keepproxy,
 595                 ServiceNameKeepstore:     svcs.Keepstore,
 596         }
 597 }