sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.arvados.org/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters         map[string]Cluster
  27         AutoReloadConfig bool
  28 }
  29
  30 // GetConfig returns the current system config, loading it from
  31 // configFile if needed.
  32 func GetConfig(configFile string) (*Config, error) {
  33         var cfg Config
  34         err := config.LoadFile(&cfg, configFile)
  35         return &cfg, err
  36 }
  37
  38 // GetCluster returns the cluster ID and config for the given
  39 // cluster, or the default/only configured cluster if clusterID is "".
  40 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  41         if clusterID == "" {
  42                 if len(sc.Clusters) == 0 {
  43                         return nil, fmt.Errorf("no clusters configured")
  44                 } else if len(sc.Clusters) > 1 {
  45                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  46                 } else {
  47                         for id, cc := range sc.Clusters {
  48                                 cc.ClusterID = id
  49                                 return &cc, nil
  50                         }
  51                 }
  52         }
  53         if cc, ok := sc.Clusters[clusterID]; !ok {
  54                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  55         } else {
  56                 cc.ClusterID = clusterID
  57                 return &cc, nil
  58         }
  59 }
  60
  61 type WebDAVCacheConfig struct {
  62         TTL                  Duration
  63         UUIDTTL              Duration
  64         MaxBlockEntries      int
  65         MaxCollectionEntries int
  66         MaxCollectionBytes   int64
  67         MaxPermissionEntries int
  68         MaxUUIDEntries       int
  69 }
  70
  71 type Cluster struct {
  72         ClusterID       string `json:"-"`
  73         ManagementToken string
  74         SystemRootToken string
  75         Services        Services
  76         InstanceTypes   InstanceTypeMap
  77         Containers      ContainersConfig
  78         RemoteClusters  map[string]RemoteCluster
  79         PostgreSQL      PostgreSQL
  80
  81         API struct {
  82                 AsyncPermissionsUpdateInterval Duration
  83                 DisabledAPIs                   StringSet
  84                 MaxIndexDatabaseRead           int
  85                 MaxItemsPerResponse            int
  86                 MaxConcurrentRequests          int
  87                 MaxKeepBlobBuffers             int
  88                 MaxRequestAmplification        int
  89                 MaxRequestSize                 int
  90                 RailsSessionSecretToken        string
  91                 RequestTimeout                 Duration
  92                 SendTimeout                    Duration
  93                 WebsocketClientEventQueue      int
  94                 WebsocketServerEventQueue      int
  95                 KeepServiceRequestTimeout      Duration
  96         }
  97         AuditLogs struct {
  98                 MaxAge             Duration
  99                 MaxDeleteBatch     int
 100                 UnloggedAttributes StringSet
 101         }
 102         Collections struct {
 103                 BlobSigning              bool
 104                 BlobSigningKey           string
 105                 BlobSigningTTL           Duration
 106                 BlobTrash                bool
 107                 BlobTrashLifetime        Duration
 108                 BlobTrashCheckInterval   Duration
 109                 BlobTrashConcurrency     int
 110                 BlobDeleteConcurrency    int
 111                 BlobReplicateConcurrency int
 112                 CollectionVersioning     bool
 113                 DefaultTrashLifetime     Duration
 114                 DefaultReplication       int
 115                 ManagedProperties        map[string]struct {
 116                         Value     interface{}
 117                         Function  string
 118                         Protected bool
 119                 }
 120                 PreserveVersionIfIdle        Duration
 121                 TrashSweepInterval           Duration
 122                 TrustAllContent              bool
 123                 ForwardSlashNameSubstitution string
 124                 S3FolderObjects              bool
 125
 126                 BlobMissingReport        string
 127                 BalancePeriod            Duration
 128                 BalanceCollectionBatch   int
 129                 BalanceCollectionBuffers int
 130                 BalanceTimeout           Duration
 131
 132                 WebDAVCache WebDAVCacheConfig
 133         }
 134         Git struct {
 135                 GitCommand   string
 136                 GitoliteHome string
 137                 Repositories string
 138         }
 139         Login struct {
 140                 LDAP struct {
 141                         Enable             bool
 142                         URL                URL
 143                         StartTLS           bool
 144                         InsecureTLS        bool
 145                         StripDomain        string
 146                         AppendDomain       string
 147                         SearchAttribute    string
 148                         SearchBindUser     string
 149                         SearchBindPassword string
 150                         SearchBase         string
 151                         SearchFilters      string
 152                         EmailAttribute     string
 153                         UsernameAttribute  string
 154                 }
 155                 Google struct {
 156                         Enable                  bool
 157                         ClientID                string
 158                         ClientSecret            string
 159                         AlternateEmailAddresses bool
 160                 }
 161                 OpenIDConnect struct {
 162                         Enable             bool
 163                         Issuer             string
 164                         ClientID           string
 165                         ClientSecret       string
 166                         EmailClaim         string
 167                         EmailVerifiedClaim string
 168                         UsernameClaim      string
 169                 }
 170                 PAM struct {
 171                         Enable             bool
 172                         Service            string
 173                         DefaultEmailDomain string
 174                 }
 175                 SSO struct {
 176                         Enable            bool
 177                         ProviderAppID     string
 178                         ProviderAppSecret string
 179                 }
 180                 LoginCluster       string
 181                 RemoteTokenRefresh Duration
 182         }
 183         Mail struct {
 184                 MailchimpAPIKey                string
 185                 MailchimpListID                string
 186                 SendUserSetupNotificationEmail bool
 187                 IssueReporterEmailFrom         string
 188                 IssueReporterEmailTo           string
 189                 SupportEmailAddress            string
 190                 EmailFrom                      string
 191         }
 192         SystemLogs struct {
 193                 LogLevel                string
 194                 Format                  string
 195                 MaxRequestLogParamsSize int
 196         }
 197         TLS struct {
 198                 Certificate string
 199                 Key         string
 200                 Insecure    bool
 201         }
 202         Users struct {
 203                 AnonymousUserToken                    string
 204                 AdminNotifierEmailFrom                string
 205                 AutoAdminFirstUser                    bool
 206                 AutoAdminUserWithEmail                string
 207                 AutoSetupNewUsers                     bool
 208                 AutoSetupNewUsersWithRepository       bool
 209                 AutoSetupNewUsersWithVmUUID           string
 210                 AutoSetupUsernameBlacklist            StringSet
 211                 EmailSubjectPrefix                    string
 212                 NewInactiveUserNotificationRecipients StringSet
 213                 NewUserNotificationRecipients         StringSet
 214                 NewUsersAreActive                     bool
 215                 UserNotifierEmailFrom                 string
 216                 UserProfileNotificationAddress        string
 217                 PreferDomainForUsername               string
 218         }
 219         Volumes   map[string]Volume
 220         Workbench struct {
 221                 ActivationContactLink            string
 222                 APIClientConnectTimeout          Duration
 223                 APIClientReceiveTimeout          Duration
 224                 APIResponseCompression           bool
 225                 ApplicationMimetypesWithViewIcon StringSet
 226                 ArvadosDocsite                   string
 227                 ArvadosPublicDataDocURL          string
 228                 DefaultOpenIdPrefix              string
 229                 EnableGettingStartedPopup        bool
 230                 EnablePublicProjectsPage         bool
 231                 FileViewersConfigURL             string
 232                 LogViewerMaxBytes                ByteSize
 233                 MultiSiteSearch                  string
 234                 ProfilingEnabled                 bool
 235                 Repositories                     bool
 236                 RepositoryCache                  string
 237                 RunningJobLogRecordsToFetch      int
 238                 SecretKeyBase                    string
 239                 ShowRecentCollectionsOnDashboard bool
 240                 ShowUserAgreementInline          bool
 241                 ShowUserNotifications            bool
 242                 SiteName                         string
 243                 Theme                            string
 244                 UserProfileFormFields            map[string]struct {
 245                         Type                 string
 246                         FormFieldTitle       string
 247                         FormFieldDescription string
 248                         Required             bool
 249                         Position             int
 250                         Options              map[string]struct{}
 251                 }
 252                 UserProfileFormMessage string
 253                 VocabularyURL          string
 254                 WelcomePageHTML        string
 255                 InactivePageHTML       string
 256                 SSHHelpPageHTML        string
 257                 SSHHelpHostSuffix      string
 258         }
 259
 260         ForceLegacyAPI14 bool
 261 }
 262
 263 type Volume struct {
 264         AccessViaHosts   map[URL]VolumeAccess
 265         ReadOnly         bool
 266         Replication      int
 267         StorageClasses   map[string]bool
 268         Driver           string
 269         DriverParameters json.RawMessage
 270 }
 271
 272 type S3VolumeDriverParameters struct {
 273         IAMRole            string
 274         AccessKey          string
 275         SecretKey          string
 276         Endpoint           string
 277         Region             string
 278         Bucket             string
 279         LocationConstraint bool
 280         V2Signature        bool
 281         UseAWSS3v2Driver   bool
 282         IndexPageSize      int
 283         ConnectTimeout     Duration
 284         ReadTimeout        Duration
 285         RaceWindow         Duration
 286         UnsafeDelete       bool
 287 }
 288
 289 type AzureVolumeDriverParameters struct {
 290         StorageAccountName   string
 291         StorageAccountKey    string
 292         StorageBaseURL       string
 293         ContainerName        string
 294         RequestTimeout       Duration
 295         ListBlobsRetryDelay  Duration
 296         ListBlobsMaxAttempts int
 297 }
 298
 299 type DirectoryVolumeDriverParameters struct {
 300         Root      string
 301         Serialize bool
 302 }
 303
 304 type VolumeAccess struct {
 305         ReadOnly bool
 306 }
 307
 308 type Services struct {
 309         Composer       Service
 310         Controller     Service
 311         DispatchCloud  Service
 312         GitHTTP        Service
 313         GitSSH         Service
 314         Health         Service
 315         Keepbalance    Service
 316         Keepproxy      Service
 317         Keepstore      Service
 318         RailsAPI       Service
 319         SSO            Service
 320         WebDAVDownload Service
 321         WebDAV         Service
 322         WebShell       Service
 323         Websocket      Service
 324         Workbench1     Service
 325         Workbench2     Service
 326 }
 327
 328 type Service struct {
 329         InternalURLs map[URL]ServiceInstance
 330         ExternalURL  URL
 331 }
 332
 333 // URL is a url.URL that is also usable as a JSON key/value.
 334 type URL url.URL
 335
 336 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 337 // used as a JSON key/value.
 338 func (su *URL) UnmarshalText(text []byte) error {
 339         u, err := url.Parse(string(text))
 340         if err == nil {
 341                 *su = URL(*u)
 342                 if su.Path == "" && su.Host != "" {
 343                         // http://example really means http://example/
 344                         su.Path = "/"
 345                 }
 346         }
 347         return err
 348 }
 349
 350 func (su URL) MarshalText() ([]byte, error) {
 351         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 352 }
 353
 354 func (su URL) String() string {
 355         return (*url.URL)(&su).String()
 356 }
 357
 358 type ServiceInstance struct {
 359         Rendezvous string `json:",omitempty"`
 360 }
 361
 362 type PostgreSQL struct {
 363         Connection     PostgreSQLConnection
 364         ConnectionPool int
 365 }
 366
 367 type PostgreSQLConnection map[string]string
 368
 369 type RemoteCluster struct {
 370         Host          string
 371         Proxy         bool
 372         Scheme        string
 373         Insecure      bool
 374         ActivateUsers bool
 375 }
 376
 377 type InstanceType struct {
 378         Name            string
 379         ProviderType    string
 380         VCPUs           int
 381         RAM             ByteSize
 382         Scratch         ByteSize
 383         IncludedScratch ByteSize
 384         AddedScratch    ByteSize
 385         Price           float64
 386         Preemptible     bool
 387 }
 388
 389 type ContainersConfig struct {
 390         CloudVMs                    CloudVMsConfig
 391         CrunchRunCommand            string
 392         CrunchRunArgumentsList      []string
 393         DefaultKeepCacheRAM         ByteSize
 394         DispatchPrivateKey          string
 395         LogReuseDecisions           bool
 396         MaxComputeVMs               int
 397         MaxDispatchAttempts         int
 398         MaxRetryAttempts            int
 399         MinRetryPeriod              Duration
 400         ReserveExtraRAM             ByteSize
 401         StaleLockTimeout            Duration
 402         SupportedDockerImageFormats StringSet
 403         UsePreemptibleInstances     bool
 404
 405         JobsAPI struct {
 406                 Enable         string
 407                 GitInternalDir string
 408         }
 409         Logging struct {
 410                 MaxAge                       Duration
 411                 LogBytesPerEvent             int
 412                 LogSecondsBetweenEvents      Duration
 413                 LogThrottlePeriod            Duration
 414                 LogThrottleBytes             int
 415                 LogThrottleLines             int
 416                 LimitLogBytesPerJob          int
 417                 LogPartialLineThrottlePeriod Duration
 418                 LogUpdatePeriod              Duration
 419                 LogUpdateSize                ByteSize
 420         }
 421         SLURM struct {
 422                 PrioritySpread             int64
 423                 SbatchArgumentsList        []string
 424                 SbatchEnvironmentVariables map[string]string
 425                 Managed                    struct {
 426                         DNSServerConfDir       string
 427                         DNSServerConfTemplate  string
 428                         DNSServerReloadCommand string
 429                         DNSServerUpdateCommand string
 430                         ComputeNodeDomain      string
 431                         ComputeNodeNameservers StringSet
 432                         AssignNodeHostname     string
 433                 }
 434         }
 435 }
 436
 437 type CloudVMsConfig struct {
 438         Enable bool
 439
 440         BootProbeCommand     string
 441         DeployRunnerBinary   string
 442         ImageID              string
 443         MaxCloudOpsPerSecond int
 444         MaxProbesPerSecond   int
 445         PollInterval         Duration
 446         ProbeInterval        Duration
 447         SSHPort              string
 448         SyncInterval         Duration
 449         TimeoutBooting       Duration
 450         TimeoutIdle          Duration
 451         TimeoutProbe         Duration
 452         TimeoutShutdown      Duration
 453         TimeoutSignal        Duration
 454         TimeoutTERM          Duration
 455         ResourceTags         map[string]string
 456         TagKeyPrefix         string
 457
 458         Driver           string
 459         DriverParameters json.RawMessage
 460 }
 461
 462 type InstanceTypeMap map[string]InstanceType
 463
 464 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 465
 466 // UnmarshalJSON handles old config files that provide an array of
 467 // instance types instead of a hash.
 468 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 469         fixup := func(t InstanceType) (InstanceType, error) {
 470                 if t.ProviderType == "" {
 471                         t.ProviderType = t.Name
 472                 }
 473                 if t.Scratch == 0 {
 474                         t.Scratch = t.IncludedScratch + t.AddedScratch
 475                 } else if t.AddedScratch == 0 {
 476                         t.AddedScratch = t.Scratch - t.IncludedScratch
 477                 } else if t.IncludedScratch == 0 {
 478                         t.IncludedScratch = t.Scratch - t.AddedScratch
 479                 }
 480
 481                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 482                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 483                 }
 484                 return t, nil
 485         }
 486
 487         if len(data) > 0 && data[0] == '[' {
 488                 var arr []InstanceType
 489                 err := json.Unmarshal(data, &arr)
 490                 if err != nil {
 491                         return err
 492                 }
 493                 if len(arr) == 0 {
 494                         *it = nil
 495                         return nil
 496                 }
 497                 *it = make(map[string]InstanceType, len(arr))
 498                 for _, t := range arr {
 499                         if _, ok := (*it)[t.Name]; ok {
 500                                 return errDuplicateInstanceTypeName
 501                         }
 502                         t, err := fixup(t)
 503                         if err != nil {
 504                                 return err
 505                         }
 506                         (*it)[t.Name] = t
 507                 }
 508                 return nil
 509         }
 510         var hash map[string]InstanceType
 511         err := json.Unmarshal(data, &hash)
 512         if err != nil {
 513                 return err
 514         }
 515         // Fill in Name field (and ProviderType field, if not
 516         // specified) using hash key.
 517         *it = InstanceTypeMap(hash)
 518         for name, t := range *it {
 519                 t.Name = name
 520                 t, err := fixup(t)
 521                 if err != nil {
 522                         return err
 523                 }
 524                 (*it)[name] = t
 525         }
 526         return nil
 527 }
 528
 529 type StringSet map[string]struct{}
 530
 531 // UnmarshalJSON handles old config files that provide an array of
 532 // instance types instead of a hash.
 533 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 534         if len(data) > 0 && data[0] == '[' {
 535                 var arr []string
 536                 err := json.Unmarshal(data, &arr)
 537                 if err != nil {
 538                         return err
 539                 }
 540                 if len(arr) == 0 {
 541                         *ss = nil
 542                         return nil
 543                 }
 544                 *ss = make(map[string]struct{}, len(arr))
 545                 for _, t := range arr {
 546                         (*ss)[t] = struct{}{}
 547                 }
 548                 return nil
 549         }
 550         var hash map[string]struct{}
 551         err := json.Unmarshal(data, &hash)
 552         if err != nil {
 553                 return err
 554         }
 555         *ss = make(map[string]struct{}, len(hash))
 556         for t := range hash {
 557                 (*ss)[t] = struct{}{}
 558         }
 559
 560         return nil
 561 }
 562
 563 type ServiceName string
 564
 565 const (
 566         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 567         ServiceNameController    ServiceName = "arvados-controller"
 568         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 569         ServiceNameHealth        ServiceName = "arvados-health"
 570         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 571         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 572         ServiceNameWebsocket     ServiceName = "arvados-ws"
 573         ServiceNameKeepbalance   ServiceName = "keep-balance"
 574         ServiceNameKeepweb       ServiceName = "keep-web"
 575         ServiceNameKeepproxy     ServiceName = "keepproxy"
 576         ServiceNameKeepstore     ServiceName = "keepstore"
 577 )
 578
 579 // Map returns all services as a map, suitable for iterating over all
 580 // services or looking up a service by name.
 581 func (svcs Services) Map() map[ServiceName]Service {
 582         return map[ServiceName]Service{
 583                 ServiceNameRailsAPI:      svcs.RailsAPI,
 584                 ServiceNameController:    svcs.Controller,
 585                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 586                 ServiceNameHealth:        svcs.Health,
 587                 ServiceNameWorkbench1:    svcs.Workbench1,
 588                 ServiceNameWorkbench2:    svcs.Workbench2,
 589                 ServiceNameWebsocket:     svcs.Websocket,
 590                 ServiceNameKeepbalance:   svcs.Keepbalance,
 591                 ServiceNameKeepweb:       svcs.WebDAV,
 592                 ServiceNameKeepproxy:     svcs.Keepproxy,
 593                 ServiceNameKeepstore:     svcs.Keepstore,
 594         }
 595 }