sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.arvados.org/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters         map[string]Cluster
  27         AutoReloadConfig bool
  28 }
  29
  30 // GetConfig returns the current system config, loading it from
  31 // configFile if needed.
  32 func GetConfig(configFile string) (*Config, error) {
  33         var cfg Config
  34         err := config.LoadFile(&cfg, configFile)
  35         return &cfg, err
  36 }
  37
  38 // GetCluster returns the cluster ID and config for the given
  39 // cluster, or the default/only configured cluster if clusterID is "".
  40 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  41         if clusterID == "" {
  42                 if len(sc.Clusters) == 0 {
  43                         return nil, fmt.Errorf("no clusters configured")
  44                 } else if len(sc.Clusters) > 1 {
  45                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  46                 } else {
  47                         for id, cc := range sc.Clusters {
  48                                 cc.ClusterID = id
  49                                 return &cc, nil
  50                         }
  51                 }
  52         }
  53         if cc, ok := sc.Clusters[clusterID]; !ok {
  54                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  55         } else {
  56                 cc.ClusterID = clusterID
  57                 return &cc, nil
  58         }
  59 }
  60
  61 type WebDAVCacheConfig struct {
  62         TTL                  Duration
  63         UUIDTTL              Duration
  64         MaxBlockEntries      int
  65         MaxCollectionEntries int
  66         MaxCollectionBytes   int64
  67         MaxPermissionEntries int
  68         MaxUUIDEntries       int
  69 }
  70
  71 type Cluster struct {
  72         ClusterID       string `json:"-"`
  73         ManagementToken string
  74         SystemRootToken string
  75         Services        Services
  76         InstanceTypes   InstanceTypeMap
  77         Containers      ContainersConfig
  78         RemoteClusters  map[string]RemoteCluster
  79         PostgreSQL      PostgreSQL
  80
  81         API struct {
  82                 AsyncPermissionsUpdateInterval Duration
  83                 DisabledAPIs                   StringSet
  84                 MaxIndexDatabaseRead           int
  85                 MaxItemsPerResponse            int
  86                 MaxConcurrentRequests          int
  87                 MaxKeepBlobBuffers             int
  88                 MaxRequestAmplification        int
  89                 MaxRequestSize                 int
  90                 RailsSessionSecretToken        string
  91                 RequestTimeout                 Duration
  92                 SendTimeout                    Duration
  93                 WebsocketClientEventQueue      int
  94                 WebsocketServerEventQueue      int
  95                 KeepServiceRequestTimeout      Duration
  96         }
  97         AuditLogs struct {
  98                 MaxAge             Duration
  99                 MaxDeleteBatch     int
 100                 UnloggedAttributes StringSet
 101         }
 102         Collections struct {
 103                 BlobSigning              bool
 104                 BlobSigningKey           string
 105                 BlobSigningTTL           Duration
 106                 BlobTrash                bool
 107                 BlobTrashLifetime        Duration
 108                 BlobTrashCheckInterval   Duration
 109                 BlobTrashConcurrency     int
 110                 BlobDeleteConcurrency    int
 111                 BlobReplicateConcurrency int
 112                 CollectionVersioning     bool
 113                 DefaultTrashLifetime     Duration
 114                 DefaultReplication       int
 115                 ManagedProperties        map[string]struct {
 116                         Value     interface{}
 117                         Function  string
 118                         Protected bool
 119                 }
 120                 PreserveVersionIfIdle        Duration
 121                 TrashSweepInterval           Duration
 122                 TrustAllContent              bool
 123                 ForwardSlashNameSubstitution string
 124
 125                 BlobMissingReport        string
 126                 BalancePeriod            Duration
 127                 BalanceCollectionBatch   int
 128                 BalanceCollectionBuffers int
 129
 130                 WebDAVCache WebDAVCacheConfig
 131         }
 132         Git struct {
 133                 GitCommand   string
 134                 GitoliteHome string
 135                 Repositories string
 136         }
 137         Login struct {
 138                 LDAP struct {
 139                         Enable             bool
 140                         URL                URL
 141                         StartTLS           bool
 142                         InsecureTLS        bool
 143                         StripDomain        string
 144                         AppendDomain       string
 145                         SearchAttribute    string
 146                         SearchBindUser     string
 147                         SearchBindPassword string
 148                         SearchBase         string
 149                         SearchFilters      string
 150                         EmailAttribute     string
 151                         UsernameAttribute  string
 152                 }
 153                 Google struct {
 154                         Enable                  bool
 155                         ClientID                string
 156                         ClientSecret            string
 157                         AlternateEmailAddresses bool
 158                 }
 159                 OpenIDConnect struct {
 160                         Enable       bool
 161                         Issuer       string
 162                         ClientID     string
 163                         ClientSecret string
 164                 }
 165                 PAM struct {
 166                         Enable             bool
 167                         Service            string
 168                         DefaultEmailDomain string
 169                 }
 170                 SSO struct {
 171                         Enable            bool
 172                         ProviderAppID     string
 173                         ProviderAppSecret string
 174                 }
 175                 LoginCluster       string
 176                 RemoteTokenRefresh Duration
 177         }
 178         Mail struct {
 179                 MailchimpAPIKey                string
 180                 MailchimpListID                string
 181                 SendUserSetupNotificationEmail bool
 182                 IssueReporterEmailFrom         string
 183                 IssueReporterEmailTo           string
 184                 SupportEmailAddress            string
 185                 EmailFrom                      string
 186         }
 187         SystemLogs struct {
 188                 LogLevel                string
 189                 Format                  string
 190                 MaxRequestLogParamsSize int
 191         }
 192         TLS struct {
 193                 Certificate string
 194                 Key         string
 195                 Insecure    bool
 196         }
 197         Users struct {
 198                 AnonymousUserToken                    string
 199                 AdminNotifierEmailFrom                string
 200                 AutoAdminFirstUser                    bool
 201                 AutoAdminUserWithEmail                string
 202                 AutoSetupNewUsers                     bool
 203                 AutoSetupNewUsersWithRepository       bool
 204                 AutoSetupNewUsersWithVmUUID           string
 205                 AutoSetupUsernameBlacklist            StringSet
 206                 EmailSubjectPrefix                    string
 207                 NewInactiveUserNotificationRecipients StringSet
 208                 NewUserNotificationRecipients         StringSet
 209                 NewUsersAreActive                     bool
 210                 UserNotifierEmailFrom                 string
 211                 UserProfileNotificationAddress        string
 212                 PreferDomainForUsername               string
 213         }
 214         Volumes   map[string]Volume
 215         Workbench struct {
 216                 ActivationContactLink            string
 217                 APIClientConnectTimeout          Duration
 218                 APIClientReceiveTimeout          Duration
 219                 APIResponseCompression           bool
 220                 ApplicationMimetypesWithViewIcon StringSet
 221                 ArvadosDocsite                   string
 222                 ArvadosPublicDataDocURL          string
 223                 DefaultOpenIdPrefix              string
 224                 EnableGettingStartedPopup        bool
 225                 EnablePublicProjectsPage         bool
 226                 FileViewersConfigURL             string
 227                 LogViewerMaxBytes                ByteSize
 228                 MultiSiteSearch                  string
 229                 ProfilingEnabled                 bool
 230                 Repositories                     bool
 231                 RepositoryCache                  string
 232                 RunningJobLogRecordsToFetch      int
 233                 SecretKeyBase                    string
 234                 ShowRecentCollectionsOnDashboard bool
 235                 ShowUserAgreementInline          bool
 236                 ShowUserNotifications            bool
 237                 SiteName                         string
 238                 Theme                            string
 239                 UserProfileFormFields            map[string]struct {
 240                         Type                 string
 241                         FormFieldTitle       string
 242                         FormFieldDescription string
 243                         Required             bool
 244                         Position             int
 245                         Options              map[string]struct{}
 246                 }
 247                 UserProfileFormMessage string
 248                 VocabularyURL          string
 249                 WelcomePageHTML        string
 250                 InactivePageHTML       string
 251                 SSHHelpPageHTML        string
 252                 SSHHelpHostSuffix      string
 253         }
 254
 255         ForceLegacyAPI14 bool
 256 }
 257
 258 type Volume struct {
 259         AccessViaHosts   map[URL]VolumeAccess
 260         ReadOnly         bool
 261         Replication      int
 262         StorageClasses   map[string]bool
 263         Driver           string
 264         DriverParameters json.RawMessage
 265 }
 266
 267 type S3VolumeDriverParameters struct {
 268         IAMRole            string
 269         AccessKey          string
 270         SecretKey          string
 271         Endpoint           string
 272         Region             string
 273         Bucket             string
 274         LocationConstraint bool
 275         V2Signature        bool
 276         IndexPageSize      int
 277         ConnectTimeout     Duration
 278         ReadTimeout        Duration
 279         RaceWindow         Duration
 280         UnsafeDelete       bool
 281 }
 282
 283 type AzureVolumeDriverParameters struct {
 284         StorageAccountName   string
 285         StorageAccountKey    string
 286         StorageBaseURL       string
 287         ContainerName        string
 288         RequestTimeout       Duration
 289         ListBlobsRetryDelay  Duration
 290         ListBlobsMaxAttempts int
 291 }
 292
 293 type DirectoryVolumeDriverParameters struct {
 294         Root      string
 295         Serialize bool
 296 }
 297
 298 type VolumeAccess struct {
 299         ReadOnly bool
 300 }
 301
 302 type Services struct {
 303         Composer       Service
 304         Controller     Service
 305         DispatchCloud  Service
 306         GitHTTP        Service
 307         GitSSH         Service
 308         Health         Service
 309         Keepbalance    Service
 310         Keepproxy      Service
 311         Keepstore      Service
 312         Nodemanager    Service
 313         RailsAPI       Service
 314         SSO            Service
 315         WebDAVDownload Service
 316         WebDAV         Service
 317         WebShell       Service
 318         Websocket      Service
 319         Workbench1     Service
 320         Workbench2     Service
 321 }
 322
 323 type Service struct {
 324         InternalURLs map[URL]ServiceInstance
 325         ExternalURL  URL
 326 }
 327
 328 // URL is a url.URL that is also usable as a JSON key/value.
 329 type URL url.URL
 330
 331 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 332 // used as a JSON key/value.
 333 func (su *URL) UnmarshalText(text []byte) error {
 334         u, err := url.Parse(string(text))
 335         if err == nil {
 336                 *su = URL(*u)
 337                 if su.Path == "" && su.Host != "" {
 338                         // http://example really means http://example/
 339                         su.Path = "/"
 340                 }
 341         }
 342         return err
 343 }
 344
 345 func (su URL) MarshalText() ([]byte, error) {
 346         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 347 }
 348
 349 func (su URL) String() string {
 350         return (*url.URL)(&su).String()
 351 }
 352
 353 type ServiceInstance struct {
 354         Rendezvous string `json:",omitempty"`
 355 }
 356
 357 type PostgreSQL struct {
 358         Connection     PostgreSQLConnection
 359         ConnectionPool int
 360 }
 361
 362 type PostgreSQLConnection map[string]string
 363
 364 type RemoteCluster struct {
 365         Host          string
 366         Proxy         bool
 367         Scheme        string
 368         Insecure      bool
 369         ActivateUsers bool
 370 }
 371
 372 type InstanceType struct {
 373         Name            string
 374         ProviderType    string
 375         VCPUs           int
 376         RAM             ByteSize
 377         Scratch         ByteSize
 378         IncludedScratch ByteSize
 379         AddedScratch    ByteSize
 380         Price           float64
 381         Preemptible     bool
 382 }
 383
 384 type ContainersConfig struct {
 385         CloudVMs                    CloudVMsConfig
 386         CrunchRunCommand            string
 387         CrunchRunArgumentsList      []string
 388         DefaultKeepCacheRAM         ByteSize
 389         DispatchPrivateKey          string
 390         LogReuseDecisions           bool
 391         MaxComputeVMs               int
 392         MaxDispatchAttempts         int
 393         MaxRetryAttempts            int
 394         MinRetryPeriod              Duration
 395         ReserveExtraRAM             ByteSize
 396         StaleLockTimeout            Duration
 397         SupportedDockerImageFormats StringSet
 398         UsePreemptibleInstances     bool
 399
 400         JobsAPI struct {
 401                 Enable         string
 402                 GitInternalDir string
 403         }
 404         Logging struct {
 405                 MaxAge                       Duration
 406                 LogBytesPerEvent             int
 407                 LogSecondsBetweenEvents      Duration
 408                 LogThrottlePeriod            Duration
 409                 LogThrottleBytes             int
 410                 LogThrottleLines             int
 411                 LimitLogBytesPerJob          int
 412                 LogPartialLineThrottlePeriod Duration
 413                 LogUpdatePeriod              Duration
 414                 LogUpdateSize                ByteSize
 415         }
 416         SLURM struct {
 417                 PrioritySpread             int64
 418                 SbatchArgumentsList        []string
 419                 SbatchEnvironmentVariables map[string]string
 420                 Managed                    struct {
 421                         DNSServerConfDir       string
 422                         DNSServerConfTemplate  string
 423                         DNSServerReloadCommand string
 424                         DNSServerUpdateCommand string
 425                         ComputeNodeDomain      string
 426                         ComputeNodeNameservers StringSet
 427                         AssignNodeHostname     string
 428                 }
 429         }
 430 }
 431
 432 type CloudVMsConfig struct {
 433         Enable bool
 434
 435         BootProbeCommand     string
 436         DeployRunnerBinary   string
 437         ImageID              string
 438         MaxCloudOpsPerSecond int
 439         MaxProbesPerSecond   int
 440         PollInterval         Duration
 441         ProbeInterval        Duration
 442         SSHPort              string
 443         SyncInterval         Duration
 444         TimeoutBooting       Duration
 445         TimeoutIdle          Duration
 446         TimeoutProbe         Duration
 447         TimeoutShutdown      Duration
 448         TimeoutSignal        Duration
 449         TimeoutTERM          Duration
 450         ResourceTags         map[string]string
 451         TagKeyPrefix         string
 452
 453         Driver           string
 454         DriverParameters json.RawMessage
 455 }
 456
 457 type InstanceTypeMap map[string]InstanceType
 458
 459 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 460
 461 // UnmarshalJSON handles old config files that provide an array of
 462 // instance types instead of a hash.
 463 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 464         fixup := func(t InstanceType) (InstanceType, error) {
 465                 if t.ProviderType == "" {
 466                         t.ProviderType = t.Name
 467                 }
 468                 if t.Scratch == 0 {
 469                         t.Scratch = t.IncludedScratch + t.AddedScratch
 470                 } else if t.AddedScratch == 0 {
 471                         t.AddedScratch = t.Scratch - t.IncludedScratch
 472                 } else if t.IncludedScratch == 0 {
 473                         t.IncludedScratch = t.Scratch - t.AddedScratch
 474                 }
 475
 476                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 477                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 478                 }
 479                 return t, nil
 480         }
 481
 482         if len(data) > 0 && data[0] == '[' {
 483                 var arr []InstanceType
 484                 err := json.Unmarshal(data, &arr)
 485                 if err != nil {
 486                         return err
 487                 }
 488                 if len(arr) == 0 {
 489                         *it = nil
 490                         return nil
 491                 }
 492                 *it = make(map[string]InstanceType, len(arr))
 493                 for _, t := range arr {
 494                         if _, ok := (*it)[t.Name]; ok {
 495                                 return errDuplicateInstanceTypeName
 496                         }
 497                         t, err := fixup(t)
 498                         if err != nil {
 499                                 return err
 500                         }
 501                         (*it)[t.Name] = t
 502                 }
 503                 return nil
 504         }
 505         var hash map[string]InstanceType
 506         err := json.Unmarshal(data, &hash)
 507         if err != nil {
 508                 return err
 509         }
 510         // Fill in Name field (and ProviderType field, if not
 511         // specified) using hash key.
 512         *it = InstanceTypeMap(hash)
 513         for name, t := range *it {
 514                 t.Name = name
 515                 t, err := fixup(t)
 516                 if err != nil {
 517                         return err
 518                 }
 519                 (*it)[name] = t
 520         }
 521         return nil
 522 }
 523
 524 type StringSet map[string]struct{}
 525
 526 // UnmarshalJSON handles old config files that provide an array of
 527 // instance types instead of a hash.
 528 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 529         if len(data) > 0 && data[0] == '[' {
 530                 var arr []string
 531                 err := json.Unmarshal(data, &arr)
 532                 if err != nil {
 533                         return err
 534                 }
 535                 if len(arr) == 0 {
 536                         *ss = nil
 537                         return nil
 538                 }
 539                 *ss = make(map[string]struct{}, len(arr))
 540                 for _, t := range arr {
 541                         (*ss)[t] = struct{}{}
 542                 }
 543                 return nil
 544         }
 545         var hash map[string]struct{}
 546         err := json.Unmarshal(data, &hash)
 547         if err != nil {
 548                 return err
 549         }
 550         *ss = make(map[string]struct{}, len(hash))
 551         for t, _ := range hash {
 552                 (*ss)[t] = struct{}{}
 553         }
 554
 555         return nil
 556 }
 557
 558 type ServiceName string
 559
 560 const (
 561         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 562         ServiceNameController    ServiceName = "arvados-controller"
 563         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 564         ServiceNameHealth        ServiceName = "arvados-health"
 565         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 566         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 567         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 568         ServiceNameWebsocket     ServiceName = "arvados-ws"
 569         ServiceNameKeepbalance   ServiceName = "keep-balance"
 570         ServiceNameKeepweb       ServiceName = "keep-web"
 571         ServiceNameKeepproxy     ServiceName = "keepproxy"
 572         ServiceNameKeepstore     ServiceName = "keepstore"
 573 )
 574
 575 // Map returns all services as a map, suitable for iterating over all
 576 // services or looking up a service by name.
 577 func (svcs Services) Map() map[ServiceName]Service {
 578         return map[ServiceName]Service{
 579                 ServiceNameRailsAPI:      svcs.RailsAPI,
 580                 ServiceNameController:    svcs.Controller,
 581                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 582                 ServiceNameHealth:        svcs.Health,
 583                 ServiceNameNodemanager:   svcs.Nodemanager,
 584                 ServiceNameWorkbench1:    svcs.Workbench1,
 585                 ServiceNameWorkbench2:    svcs.Workbench2,
 586                 ServiceNameWebsocket:     svcs.Websocket,
 587                 ServiceNameKeepbalance:   svcs.Keepbalance,
 588                 ServiceNameKeepweb:       svcs.WebDAV,
 589                 ServiceNameKeepproxy:     svcs.Keepproxy,
 590                 ServiceNameKeepstore:     svcs.Keepstore,
 591         }
 592 }