sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.arvados.org/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters         map[string]Cluster
  27         AutoReloadConfig bool
  28 }
  29
  30 // GetConfig returns the current system config, loading it from
  31 // configFile if needed.
  32 func GetConfig(configFile string) (*Config, error) {
  33         var cfg Config
  34         err := config.LoadFile(&cfg, configFile)
  35         return &cfg, err
  36 }
  37
  38 // GetCluster returns the cluster ID and config for the given
  39 // cluster, or the default/only configured cluster if clusterID is "".
  40 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  41         if clusterID == "" {
  42                 if len(sc.Clusters) == 0 {
  43                         return nil, fmt.Errorf("no clusters configured")
  44                 } else if len(sc.Clusters) > 1 {
  45                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  46                 } else {
  47                         for id, cc := range sc.Clusters {
  48                                 cc.ClusterID = id
  49                                 return &cc, nil
  50                         }
  51                 }
  52         }
  53         if cc, ok := sc.Clusters[clusterID]; !ok {
  54                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  55         } else {
  56                 cc.ClusterID = clusterID
  57                 return &cc, nil
  58         }
  59 }
  60
  61 type WebDAVCacheConfig struct {
  62         TTL                  Duration
  63         UUIDTTL              Duration
  64         MaxBlockEntries      int
  65         MaxCollectionEntries int
  66         MaxCollectionBytes   int64
  67         MaxPermissionEntries int
  68         MaxUUIDEntries       int
  69 }
  70
  71 type Cluster struct {
  72         ClusterID       string `json:"-"`
  73         ManagementToken string
  74         SystemRootToken string
  75         Services        Services
  76         InstanceTypes   InstanceTypeMap
  77         Containers      ContainersConfig
  78         RemoteClusters  map[string]RemoteCluster
  79         PostgreSQL      PostgreSQL
  80
  81         API struct {
  82                 AsyncPermissionsUpdateInterval Duration
  83                 DisabledAPIs                   StringSet
  84                 MaxIndexDatabaseRead           int
  85                 MaxItemsPerResponse            int
  86                 MaxConcurrentRequests          int
  87                 MaxKeepBlobBuffers             int
  88                 MaxRequestAmplification        int
  89                 MaxRequestSize                 int
  90                 RailsSessionSecretToken        string
  91                 RequestTimeout                 Duration
  92                 SendTimeout                    Duration
  93                 WebsocketClientEventQueue      int
  94                 WebsocketServerEventQueue      int
  95                 KeepServiceRequestTimeout      Duration
  96         }
  97         AuditLogs struct {
  98                 MaxAge             Duration
  99                 MaxDeleteBatch     int
 100                 UnloggedAttributes StringSet
 101         }
 102         Collections struct {
 103                 BlobSigning              bool
 104                 BlobSigningKey           string
 105                 BlobSigningTTL           Duration
 106                 BlobTrash                bool
 107                 BlobTrashLifetime        Duration
 108                 BlobTrashCheckInterval   Duration
 109                 BlobTrashConcurrency     int
 110                 BlobDeleteConcurrency    int
 111                 BlobReplicateConcurrency int
 112                 CollectionVersioning     bool
 113                 DefaultTrashLifetime     Duration
 114                 DefaultReplication       int
 115                 ManagedProperties        map[string]struct {
 116                         Value     interface{}
 117                         Function  string
 118                         Protected bool
 119                 }
 120                 PreserveVersionIfIdle        Duration
 121                 TrashSweepInterval           Duration
 122                 TrustAllContent              bool
 123                 ForwardSlashNameSubstitution string
 124
 125                 BlobMissingReport        string
 126                 BalancePeriod            Duration
 127                 BalanceCollectionBatch   int
 128                 BalanceCollectionBuffers int
 129                 BalanceTimeout           Duration
 130
 131                 WebDAVCache WebDAVCacheConfig
 132         }
 133         Git struct {
 134                 GitCommand   string
 135                 GitoliteHome string
 136                 Repositories string
 137         }
 138         Login struct {
 139                 LDAP struct {
 140                         Enable             bool
 141                         URL                URL
 142                         StartTLS           bool
 143                         InsecureTLS        bool
 144                         StripDomain        string
 145                         AppendDomain       string
 146                         SearchAttribute    string
 147                         SearchBindUser     string
 148                         SearchBindPassword string
 149                         SearchBase         string
 150                         SearchFilters      string
 151                         EmailAttribute     string
 152                         UsernameAttribute  string
 153                 }
 154                 Google struct {
 155                         Enable                  bool
 156                         ClientID                string
 157                         ClientSecret            string
 158                         AlternateEmailAddresses bool
 159                 }
 160                 OpenIDConnect struct {
 161                         Enable       bool
 162                         Issuer       string
 163                         ClientID     string
 164                         ClientSecret string
 165                 }
 166                 PAM struct {
 167                         Enable             bool
 168                         Service            string
 169                         DefaultEmailDomain string
 170                 }
 171                 SSO struct {
 172                         Enable            bool
 173                         ProviderAppID     string
 174                         ProviderAppSecret string
 175                 }
 176                 LoginCluster       string
 177                 RemoteTokenRefresh Duration
 178         }
 179         Mail struct {
 180                 MailchimpAPIKey                string
 181                 MailchimpListID                string
 182                 SendUserSetupNotificationEmail bool
 183                 IssueReporterEmailFrom         string
 184                 IssueReporterEmailTo           string
 185                 SupportEmailAddress            string
 186                 EmailFrom                      string
 187         }
 188         SystemLogs struct {
 189                 LogLevel                string
 190                 Format                  string
 191                 MaxRequestLogParamsSize int
 192         }
 193         TLS struct {
 194                 Certificate string
 195                 Key         string
 196                 Insecure    bool
 197         }
 198         Users struct {
 199                 AnonymousUserToken                    string
 200                 AdminNotifierEmailFrom                string
 201                 AutoAdminFirstUser                    bool
 202                 AutoAdminUserWithEmail                string
 203                 AutoSetupNewUsers                     bool
 204                 AutoSetupNewUsersWithRepository       bool
 205                 AutoSetupNewUsersWithVmUUID           string
 206                 AutoSetupUsernameBlacklist            StringSet
 207                 EmailSubjectPrefix                    string
 208                 NewInactiveUserNotificationRecipients StringSet
 209                 NewUserNotificationRecipients         StringSet
 210                 NewUsersAreActive                     bool
 211                 UserNotifierEmailFrom                 string
 212                 UserProfileNotificationAddress        string
 213                 PreferDomainForUsername               string
 214         }
 215         Volumes   map[string]Volume
 216         Workbench struct {
 217                 ActivationContactLink            string
 218                 APIClientConnectTimeout          Duration
 219                 APIClientReceiveTimeout          Duration
 220                 APIResponseCompression           bool
 221                 ApplicationMimetypesWithViewIcon StringSet
 222                 ArvadosDocsite                   string
 223                 ArvadosPublicDataDocURL          string
 224                 DefaultOpenIdPrefix              string
 225                 EnableGettingStartedPopup        bool
 226                 EnablePublicProjectsPage         bool
 227                 FileViewersConfigURL             string
 228                 LogViewerMaxBytes                ByteSize
 229                 MultiSiteSearch                  string
 230                 ProfilingEnabled                 bool
 231                 Repositories                     bool
 232                 RepositoryCache                  string
 233                 RunningJobLogRecordsToFetch      int
 234                 SecretKeyBase                    string
 235                 ShowRecentCollectionsOnDashboard bool
 236                 ShowUserAgreementInline          bool
 237                 ShowUserNotifications            bool
 238                 SiteName                         string
 239                 Theme                            string
 240                 UserProfileFormFields            map[string]struct {
 241                         Type                 string
 242                         FormFieldTitle       string
 243                         FormFieldDescription string
 244                         Required             bool
 245                         Position             int
 246                         Options              map[string]struct{}
 247                 }
 248                 UserProfileFormMessage string
 249                 VocabularyURL          string
 250                 WelcomePageHTML        string
 251                 InactivePageHTML       string
 252                 SSHHelpPageHTML        string
 253                 SSHHelpHostSuffix      string
 254         }
 255
 256         ForceLegacyAPI14 bool
 257 }
 258
 259 type Volume struct {
 260         AccessViaHosts   map[URL]VolumeAccess
 261         ReadOnly         bool
 262         Replication      int
 263         StorageClasses   map[string]bool
 264         Driver           string
 265         DriverParameters json.RawMessage
 266 }
 267
 268 type S3VolumeDriverParameters struct {
 269         IAMRole            string
 270         AccessKey          string
 271         SecretKey          string
 272         Endpoint           string
 273         Region             string
 274         Bucket             string
 275         LocationConstraint bool
 276         V2Signature        bool
 277         IndexPageSize      int
 278         ConnectTimeout     Duration
 279         ReadTimeout        Duration
 280         RaceWindow         Duration
 281         UnsafeDelete       bool
 282 }
 283
 284 type AzureVolumeDriverParameters struct {
 285         StorageAccountName   string
 286         StorageAccountKey    string
 287         StorageBaseURL       string
 288         ContainerName        string
 289         RequestTimeout       Duration
 290         ListBlobsRetryDelay  Duration
 291         ListBlobsMaxAttempts int
 292 }
 293
 294 type DirectoryVolumeDriverParameters struct {
 295         Root      string
 296         Serialize bool
 297 }
 298
 299 type VolumeAccess struct {
 300         ReadOnly bool
 301 }
 302
 303 type Services struct {
 304         Composer       Service
 305         Controller     Service
 306         DispatchCloud  Service
 307         GitHTTP        Service
 308         GitSSH         Service
 309         Health         Service
 310         Keepbalance    Service
 311         Keepproxy      Service
 312         Keepstore      Service
 313         Nodemanager    Service
 314         RailsAPI       Service
 315         SSO            Service
 316         WebDAVDownload Service
 317         WebDAV         Service
 318         WebShell       Service
 319         Websocket      Service
 320         Workbench1     Service
 321         Workbench2     Service
 322 }
 323
 324 type Service struct {
 325         InternalURLs map[URL]ServiceInstance
 326         ExternalURL  URL
 327 }
 328
 329 // URL is a url.URL that is also usable as a JSON key/value.
 330 type URL url.URL
 331
 332 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 333 // used as a JSON key/value.
 334 func (su *URL) UnmarshalText(text []byte) error {
 335         u, err := url.Parse(string(text))
 336         if err == nil {
 337                 *su = URL(*u)
 338                 if su.Path == "" && su.Host != "" {
 339                         // http://example really means http://example/
 340                         su.Path = "/"
 341                 }
 342         }
 343         return err
 344 }
 345
 346 func (su URL) MarshalText() ([]byte, error) {
 347         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 348 }
 349
 350 func (su URL) String() string {
 351         return (*url.URL)(&su).String()
 352 }
 353
 354 type ServiceInstance struct {
 355         Rendezvous string `json:",omitempty"`
 356 }
 357
 358 type PostgreSQL struct {
 359         Connection     PostgreSQLConnection
 360         ConnectionPool int
 361 }
 362
 363 type PostgreSQLConnection map[string]string
 364
 365 type RemoteCluster struct {
 366         Host          string
 367         Proxy         bool
 368         Scheme        string
 369         Insecure      bool
 370         ActivateUsers bool
 371 }
 372
 373 type InstanceType struct {
 374         Name            string
 375         ProviderType    string
 376         VCPUs           int
 377         RAM             ByteSize
 378         Scratch         ByteSize
 379         IncludedScratch ByteSize
 380         AddedScratch    ByteSize
 381         Price           float64
 382         Preemptible     bool
 383 }
 384
 385 type ContainersConfig struct {
 386         CloudVMs                    CloudVMsConfig
 387         CrunchRunCommand            string
 388         CrunchRunArgumentsList      []string
 389         DefaultKeepCacheRAM         ByteSize
 390         DispatchPrivateKey          string
 391         LogReuseDecisions           bool
 392         MaxComputeVMs               int
 393         MaxDispatchAttempts         int
 394         MaxRetryAttempts            int
 395         MinRetryPeriod              Duration
 396         ReserveExtraRAM             ByteSize
 397         StaleLockTimeout            Duration
 398         SupportedDockerImageFormats StringSet
 399         UsePreemptibleInstances     bool
 400
 401         JobsAPI struct {
 402                 Enable         string
 403                 GitInternalDir string
 404         }
 405         Logging struct {
 406                 MaxAge                       Duration
 407                 LogBytesPerEvent             int
 408                 LogSecondsBetweenEvents      Duration
 409                 LogThrottlePeriod            Duration
 410                 LogThrottleBytes             int
 411                 LogThrottleLines             int
 412                 LimitLogBytesPerJob          int
 413                 LogPartialLineThrottlePeriod Duration
 414                 LogUpdatePeriod              Duration
 415                 LogUpdateSize                ByteSize
 416         }
 417         SLURM struct {
 418                 PrioritySpread             int64
 419                 SbatchArgumentsList        []string
 420                 SbatchEnvironmentVariables map[string]string
 421                 Managed                    struct {
 422                         DNSServerConfDir       string
 423                         DNSServerConfTemplate  string
 424                         DNSServerReloadCommand string
 425                         DNSServerUpdateCommand string
 426                         ComputeNodeDomain      string
 427                         ComputeNodeNameservers StringSet
 428                         AssignNodeHostname     string
 429                 }
 430         }
 431 }
 432
 433 type CloudVMsConfig struct {
 434         Enable bool
 435
 436         BootProbeCommand     string
 437         DeployRunnerBinary   string
 438         ImageID              string
 439         MaxCloudOpsPerSecond int
 440         MaxProbesPerSecond   int
 441         PollInterval         Duration
 442         ProbeInterval        Duration
 443         SSHPort              string
 444         SyncInterval         Duration
 445         TimeoutBooting       Duration
 446         TimeoutIdle          Duration
 447         TimeoutProbe         Duration
 448         TimeoutShutdown      Duration
 449         TimeoutSignal        Duration
 450         TimeoutTERM          Duration
 451         ResourceTags         map[string]string
 452         TagKeyPrefix         string
 453
 454         Driver           string
 455         DriverParameters json.RawMessage
 456 }
 457
 458 type InstanceTypeMap map[string]InstanceType
 459
 460 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 461
 462 // UnmarshalJSON handles old config files that provide an array of
 463 // instance types instead of a hash.
 464 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 465         fixup := func(t InstanceType) (InstanceType, error) {
 466                 if t.ProviderType == "" {
 467                         t.ProviderType = t.Name
 468                 }
 469                 if t.Scratch == 0 {
 470                         t.Scratch = t.IncludedScratch + t.AddedScratch
 471                 } else if t.AddedScratch == 0 {
 472                         t.AddedScratch = t.Scratch - t.IncludedScratch
 473                 } else if t.IncludedScratch == 0 {
 474                         t.IncludedScratch = t.Scratch - t.AddedScratch
 475                 }
 476
 477                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 478                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 479                 }
 480                 return t, nil
 481         }
 482
 483         if len(data) > 0 && data[0] == '[' {
 484                 var arr []InstanceType
 485                 err := json.Unmarshal(data, &arr)
 486                 if err != nil {
 487                         return err
 488                 }
 489                 if len(arr) == 0 {
 490                         *it = nil
 491                         return nil
 492                 }
 493                 *it = make(map[string]InstanceType, len(arr))
 494                 for _, t := range arr {
 495                         if _, ok := (*it)[t.Name]; ok {
 496                                 return errDuplicateInstanceTypeName
 497                         }
 498                         t, err := fixup(t)
 499                         if err != nil {
 500                                 return err
 501                         }
 502                         (*it)[t.Name] = t
 503                 }
 504                 return nil
 505         }
 506         var hash map[string]InstanceType
 507         err := json.Unmarshal(data, &hash)
 508         if err != nil {
 509                 return err
 510         }
 511         // Fill in Name field (and ProviderType field, if not
 512         // specified) using hash key.
 513         *it = InstanceTypeMap(hash)
 514         for name, t := range *it {
 515                 t.Name = name
 516                 t, err := fixup(t)
 517                 if err != nil {
 518                         return err
 519                 }
 520                 (*it)[name] = t
 521         }
 522         return nil
 523 }
 524
 525 type StringSet map[string]struct{}
 526
 527 // UnmarshalJSON handles old config files that provide an array of
 528 // instance types instead of a hash.
 529 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 530         if len(data) > 0 && data[0] == '[' {
 531                 var arr []string
 532                 err := json.Unmarshal(data, &arr)
 533                 if err != nil {
 534                         return err
 535                 }
 536                 if len(arr) == 0 {
 537                         *ss = nil
 538                         return nil
 539                 }
 540                 *ss = make(map[string]struct{}, len(arr))
 541                 for _, t := range arr {
 542                         (*ss)[t] = struct{}{}
 543                 }
 544                 return nil
 545         }
 546         var hash map[string]struct{}
 547         err := json.Unmarshal(data, &hash)
 548         if err != nil {
 549                 return err
 550         }
 551         *ss = make(map[string]struct{}, len(hash))
 552         for t, _ := range hash {
 553                 (*ss)[t] = struct{}{}
 554         }
 555
 556         return nil
 557 }
 558
 559 type ServiceName string
 560
 561 const (
 562         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 563         ServiceNameController    ServiceName = "arvados-controller"
 564         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 565         ServiceNameHealth        ServiceName = "arvados-health"
 566         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 567         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 568         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 569         ServiceNameWebsocket     ServiceName = "arvados-ws"
 570         ServiceNameKeepbalance   ServiceName = "keep-balance"
 571         ServiceNameKeepweb       ServiceName = "keep-web"
 572         ServiceNameKeepproxy     ServiceName = "keepproxy"
 573         ServiceNameKeepstore     ServiceName = "keepstore"
 574 )
 575
 576 // Map returns all services as a map, suitable for iterating over all
 577 // services or looking up a service by name.
 578 func (svcs Services) Map() map[ServiceName]Service {
 579         return map[ServiceName]Service{
 580                 ServiceNameRailsAPI:      svcs.RailsAPI,
 581                 ServiceNameController:    svcs.Controller,
 582                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 583                 ServiceNameHealth:        svcs.Health,
 584                 ServiceNameNodemanager:   svcs.Nodemanager,
 585                 ServiceNameWorkbench1:    svcs.Workbench1,
 586                 ServiceNameWorkbench2:    svcs.Workbench2,
 587                 ServiceNameWebsocket:     svcs.Websocket,
 588                 ServiceNameKeepbalance:   svcs.Keepbalance,
 589                 ServiceNameKeepweb:       svcs.WebDAV,
 590                 ServiceNameKeepproxy:     svcs.Keepproxy,
 591                 ServiceNameKeepstore:     svcs.Keepstore,
 592         }
 593 }