sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.arvados.org/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters         map[string]Cluster
  27         AutoReloadConfig bool
  28 }
  29
  30 // GetConfig returns the current system config, loading it from
  31 // configFile if needed.
  32 func GetConfig(configFile string) (*Config, error) {
  33         var cfg Config
  34         err := config.LoadFile(&cfg, configFile)
  35         return &cfg, err
  36 }
  37
  38 // GetCluster returns the cluster ID and config for the given
  39 // cluster, or the default/only configured cluster if clusterID is "".
  40 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  41         if clusterID == "" {
  42                 if len(sc.Clusters) == 0 {
  43                         return nil, fmt.Errorf("no clusters configured")
  44                 } else if len(sc.Clusters) > 1 {
  45                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  46                 } else {
  47                         for id, cc := range sc.Clusters {
  48                                 cc.ClusterID = id
  49                                 return &cc, nil
  50                         }
  51                 }
  52         }
  53         if cc, ok := sc.Clusters[clusterID]; !ok {
  54                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  55         } else {
  56                 cc.ClusterID = clusterID
  57                 return &cc, nil
  58         }
  59 }
  60
  61 type WebDAVCacheConfig struct {
  62         TTL                  Duration
  63         UUIDTTL              Duration
  64         MaxBlockEntries      int
  65         MaxCollectionEntries int
  66         MaxCollectionBytes   int64
  67         MaxPermissionEntries int
  68         MaxUUIDEntries       int
  69 }
  70
  71 type Cluster struct {
  72         ClusterID       string `json:"-"`
  73         ManagementToken string
  74         SystemRootToken string
  75         Services        Services
  76         InstanceTypes   InstanceTypeMap
  77         Containers      ContainersConfig
  78         RemoteClusters  map[string]RemoteCluster
  79         PostgreSQL      PostgreSQL
  80
  81         API struct {
  82                 AsyncPermissionsUpdateInterval Duration
  83                 DisabledAPIs                   StringSet
  84                 MaxIndexDatabaseRead           int
  85                 MaxItemsPerResponse            int
  86                 MaxConcurrentRequests          int
  87                 MaxKeepBlobBuffers             int
  88                 MaxRequestAmplification        int
  89                 MaxRequestSize                 int
  90                 RailsSessionSecretToken        string
  91                 RequestTimeout                 Duration
  92                 SendTimeout                    Duration
  93                 WebsocketClientEventQueue      int
  94                 WebsocketServerEventQueue      int
  95                 KeepServiceRequestTimeout      Duration
  96         }
  97         AuditLogs struct {
  98                 MaxAge             Duration
  99                 MaxDeleteBatch     int
 100                 UnloggedAttributes StringSet
 101         }
 102         Collections struct {
 103                 BlobSigning              bool
 104                 BlobSigningKey           string
 105                 BlobSigningTTL           Duration
 106                 BlobTrash                bool
 107                 BlobTrashLifetime        Duration
 108                 BlobTrashCheckInterval   Duration
 109                 BlobTrashConcurrency     int
 110                 BlobDeleteConcurrency    int
 111                 BlobReplicateConcurrency int
 112                 CollectionVersioning     bool
 113                 DefaultTrashLifetime     Duration
 114                 DefaultReplication       int
 115                 ManagedProperties        map[string]struct {
 116                         Value     interface{}
 117                         Function  string
 118                         Protected bool
 119                 }
 120                 PreserveVersionIfIdle        Duration
 121                 TrashSweepInterval           Duration
 122                 TrustAllContent              bool
 123                 ForwardSlashNameSubstitution string
 124
 125                 BlobMissingReport        string
 126                 BalancePeriod            Duration
 127                 BalanceCollectionBatch   int
 128                 BalanceCollectionBuffers int
 129
 130                 WebDAVCache WebDAVCacheConfig
 131         }
 132         Git struct {
 133                 GitCommand   string
 134                 GitoliteHome string
 135                 Repositories string
 136         }
 137         Login struct {
 138                 LDAP struct {
 139                         Enable             bool
 140                         URL                URL
 141                         StartTLS           bool
 142                         InsecureTLS        bool
 143                         StripDomain        string
 144                         AppendDomain       string
 145                         SearchAttribute    string
 146                         SearchBindUser     string
 147                         SearchBindPassword string
 148                         SearchBase         string
 149                         SearchFilters      string
 150                         EmailAttribute     string
 151                         UsernameAttribute  string
 152                 }
 153                 Google struct {
 154                         Enable                  bool
 155                         ClientID                string
 156                         ClientSecret            string
 157                         AlternateEmailAddresses bool
 158                 }
 159                 OpenIDConnect struct {
 160                         Enable             bool
 161                         Issuer             string
 162                         ClientID           string
 163                         ClientSecret       string
 164                         EmailClaim         string
 165                         EmailVerifiedClaim string
 166                         UsernameClaim      string
 167                 }
 168                 PAM struct {
 169                         Enable             bool
 170                         Service            string
 171                         DefaultEmailDomain string
 172                 }
 173                 SSO struct {
 174                         Enable            bool
 175                         ProviderAppID     string
 176                         ProviderAppSecret string
 177                 }
 178                 LoginCluster       string
 179                 RemoteTokenRefresh Duration
 180         }
 181         Mail struct {
 182                 MailchimpAPIKey                string
 183                 MailchimpListID                string
 184                 SendUserSetupNotificationEmail bool
 185                 IssueReporterEmailFrom         string
 186                 IssueReporterEmailTo           string
 187                 SupportEmailAddress            string
 188                 EmailFrom                      string
 189         }
 190         SystemLogs struct {
 191                 LogLevel                string
 192                 Format                  string
 193                 MaxRequestLogParamsSize int
 194         }
 195         TLS struct {
 196                 Certificate string
 197                 Key         string
 198                 Insecure    bool
 199         }
 200         Users struct {
 201                 AnonymousUserToken                    string
 202                 AdminNotifierEmailFrom                string
 203                 AutoAdminFirstUser                    bool
 204                 AutoAdminUserWithEmail                string
 205                 AutoSetupNewUsers                     bool
 206                 AutoSetupNewUsersWithRepository       bool
 207                 AutoSetupNewUsersWithVmUUID           string
 208                 AutoSetupUsernameBlacklist            StringSet
 209                 EmailSubjectPrefix                    string
 210                 NewInactiveUserNotificationRecipients StringSet
 211                 NewUserNotificationRecipients         StringSet
 212                 NewUsersAreActive                     bool
 213                 UserNotifierEmailFrom                 string
 214                 UserProfileNotificationAddress        string
 215                 PreferDomainForUsername               string
 216         }
 217         Volumes   map[string]Volume
 218         Workbench struct {
 219                 ActivationContactLink            string
 220                 APIClientConnectTimeout          Duration
 221                 APIClientReceiveTimeout          Duration
 222                 APIResponseCompression           bool
 223                 ApplicationMimetypesWithViewIcon StringSet
 224                 ArvadosDocsite                   string
 225                 ArvadosPublicDataDocURL          string
 226                 DefaultOpenIdPrefix              string
 227                 EnableGettingStartedPopup        bool
 228                 EnablePublicProjectsPage         bool
 229                 FileViewersConfigURL             string
 230                 LogViewerMaxBytes                ByteSize
 231                 MultiSiteSearch                  string
 232                 ProfilingEnabled                 bool
 233                 Repositories                     bool
 234                 RepositoryCache                  string
 235                 RunningJobLogRecordsToFetch      int
 236                 SecretKeyBase                    string
 237                 ShowRecentCollectionsOnDashboard bool
 238                 ShowUserAgreementInline          bool
 239                 ShowUserNotifications            bool
 240                 SiteName                         string
 241                 Theme                            string
 242                 UserProfileFormFields            map[string]struct {
 243                         Type                 string
 244                         FormFieldTitle       string
 245                         FormFieldDescription string
 246                         Required             bool
 247                         Position             int
 248                         Options              map[string]struct{}
 249                 }
 250                 UserProfileFormMessage string
 251                 VocabularyURL          string
 252                 WelcomePageHTML        string
 253                 InactivePageHTML       string
 254                 SSHHelpPageHTML        string
 255                 SSHHelpHostSuffix      string
 256         }
 257
 258         ForceLegacyAPI14 bool
 259 }
 260
 261 type Volume struct {
 262         AccessViaHosts   map[URL]VolumeAccess
 263         ReadOnly         bool
 264         Replication      int
 265         StorageClasses   map[string]bool
 266         Driver           string
 267         DriverParameters json.RawMessage
 268 }
 269
 270 type S3VolumeDriverParameters struct {
 271         IAMRole            string
 272         AccessKey          string
 273         SecretKey          string
 274         Endpoint           string
 275         Region             string
 276         Bucket             string
 277         LocationConstraint bool
 278         V2Signature        bool
 279         IndexPageSize      int
 280         ConnectTimeout     Duration
 281         ReadTimeout        Duration
 282         RaceWindow         Duration
 283         UnsafeDelete       bool
 284 }
 285
 286 type AzureVolumeDriverParameters struct {
 287         StorageAccountName   string
 288         StorageAccountKey    string
 289         StorageBaseURL       string
 290         ContainerName        string
 291         RequestTimeout       Duration
 292         ListBlobsRetryDelay  Duration
 293         ListBlobsMaxAttempts int
 294 }
 295
 296 type DirectoryVolumeDriverParameters struct {
 297         Root      string
 298         Serialize bool
 299 }
 300
 301 type VolumeAccess struct {
 302         ReadOnly bool
 303 }
 304
 305 type Services struct {
 306         Composer       Service
 307         Controller     Service
 308         DispatchCloud  Service
 309         GitHTTP        Service
 310         GitSSH         Service
 311         Health         Service
 312         Keepbalance    Service
 313         Keepproxy      Service
 314         Keepstore      Service
 315         Nodemanager    Service
 316         RailsAPI       Service
 317         SSO            Service
 318         WebDAVDownload Service
 319         WebDAV         Service
 320         WebShell       Service
 321         Websocket      Service
 322         Workbench1     Service
 323         Workbench2     Service
 324 }
 325
 326 type Service struct {
 327         InternalURLs map[URL]ServiceInstance
 328         ExternalURL  URL
 329 }
 330
 331 // URL is a url.URL that is also usable as a JSON key/value.
 332 type URL url.URL
 333
 334 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 335 // used as a JSON key/value.
 336 func (su *URL) UnmarshalText(text []byte) error {
 337         u, err := url.Parse(string(text))
 338         if err == nil {
 339                 *su = URL(*u)
 340                 if su.Path == "" && su.Host != "" {
 341                         // http://example really means http://example/
 342                         su.Path = "/"
 343                 }
 344         }
 345         return err
 346 }
 347
 348 func (su URL) MarshalText() ([]byte, error) {
 349         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 350 }
 351
 352 func (su URL) String() string {
 353         return (*url.URL)(&su).String()
 354 }
 355
 356 type ServiceInstance struct {
 357         Rendezvous string `json:",omitempty"`
 358 }
 359
 360 type PostgreSQL struct {
 361         Connection     PostgreSQLConnection
 362         ConnectionPool int
 363 }
 364
 365 type PostgreSQLConnection map[string]string
 366
 367 type RemoteCluster struct {
 368         Host          string
 369         Proxy         bool
 370         Scheme        string
 371         Insecure      bool
 372         ActivateUsers bool
 373 }
 374
 375 type InstanceType struct {
 376         Name            string
 377         ProviderType    string
 378         VCPUs           int
 379         RAM             ByteSize
 380         Scratch         ByteSize
 381         IncludedScratch ByteSize
 382         AddedScratch    ByteSize
 383         Price           float64
 384         Preemptible     bool
 385 }
 386
 387 type ContainersConfig struct {
 388         CloudVMs                    CloudVMsConfig
 389         CrunchRunCommand            string
 390         CrunchRunArgumentsList      []string
 391         DefaultKeepCacheRAM         ByteSize
 392         DispatchPrivateKey          string
 393         LogReuseDecisions           bool
 394         MaxComputeVMs               int
 395         MaxDispatchAttempts         int
 396         MaxRetryAttempts            int
 397         MinRetryPeriod              Duration
 398         ReserveExtraRAM             ByteSize
 399         StaleLockTimeout            Duration
 400         SupportedDockerImageFormats StringSet
 401         UsePreemptibleInstances     bool
 402
 403         JobsAPI struct {
 404                 Enable         string
 405                 GitInternalDir string
 406         }
 407         Logging struct {
 408                 MaxAge                       Duration
 409                 LogBytesPerEvent             int
 410                 LogSecondsBetweenEvents      Duration
 411                 LogThrottlePeriod            Duration
 412                 LogThrottleBytes             int
 413                 LogThrottleLines             int
 414                 LimitLogBytesPerJob          int
 415                 LogPartialLineThrottlePeriod Duration
 416                 LogUpdatePeriod              Duration
 417                 LogUpdateSize                ByteSize
 418         }
 419         SLURM struct {
 420                 PrioritySpread             int64
 421                 SbatchArgumentsList        []string
 422                 SbatchEnvironmentVariables map[string]string
 423                 Managed                    struct {
 424                         DNSServerConfDir       string
 425                         DNSServerConfTemplate  string
 426                         DNSServerReloadCommand string
 427                         DNSServerUpdateCommand string
 428                         ComputeNodeDomain      string
 429                         ComputeNodeNameservers StringSet
 430                         AssignNodeHostname     string
 431                 }
 432         }
 433 }
 434
 435 type CloudVMsConfig struct {
 436         Enable bool
 437
 438         BootProbeCommand     string
 439         DeployRunnerBinary   string
 440         ImageID              string
 441         MaxCloudOpsPerSecond int
 442         MaxProbesPerSecond   int
 443         PollInterval         Duration
 444         ProbeInterval        Duration
 445         SSHPort              string
 446         SyncInterval         Duration
 447         TimeoutBooting       Duration
 448         TimeoutIdle          Duration
 449         TimeoutProbe         Duration
 450         TimeoutShutdown      Duration
 451         TimeoutSignal        Duration
 452         TimeoutTERM          Duration
 453         ResourceTags         map[string]string
 454         TagKeyPrefix         string
 455
 456         Driver           string
 457         DriverParameters json.RawMessage
 458 }
 459
 460 type InstanceTypeMap map[string]InstanceType
 461
 462 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 463
 464 // UnmarshalJSON handles old config files that provide an array of
 465 // instance types instead of a hash.
 466 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 467         fixup := func(t InstanceType) (InstanceType, error) {
 468                 if t.ProviderType == "" {
 469                         t.ProviderType = t.Name
 470                 }
 471                 if t.Scratch == 0 {
 472                         t.Scratch = t.IncludedScratch + t.AddedScratch
 473                 } else if t.AddedScratch == 0 {
 474                         t.AddedScratch = t.Scratch - t.IncludedScratch
 475                 } else if t.IncludedScratch == 0 {
 476                         t.IncludedScratch = t.Scratch - t.AddedScratch
 477                 }
 478
 479                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 480                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 481                 }
 482                 return t, nil
 483         }
 484
 485         if len(data) > 0 && data[0] == '[' {
 486                 var arr []InstanceType
 487                 err := json.Unmarshal(data, &arr)
 488                 if err != nil {
 489                         return err
 490                 }
 491                 if len(arr) == 0 {
 492                         *it = nil
 493                         return nil
 494                 }
 495                 *it = make(map[string]InstanceType, len(arr))
 496                 for _, t := range arr {
 497                         if _, ok := (*it)[t.Name]; ok {
 498                                 return errDuplicateInstanceTypeName
 499                         }
 500                         t, err := fixup(t)
 501                         if err != nil {
 502                                 return err
 503                         }
 504                         (*it)[t.Name] = t
 505                 }
 506                 return nil
 507         }
 508         var hash map[string]InstanceType
 509         err := json.Unmarshal(data, &hash)
 510         if err != nil {
 511                 return err
 512         }
 513         // Fill in Name field (and ProviderType field, if not
 514         // specified) using hash key.
 515         *it = InstanceTypeMap(hash)
 516         for name, t := range *it {
 517                 t.Name = name
 518                 t, err := fixup(t)
 519                 if err != nil {
 520                         return err
 521                 }
 522                 (*it)[name] = t
 523         }
 524         return nil
 525 }
 526
 527 type StringSet map[string]struct{}
 528
 529 // UnmarshalJSON handles old config files that provide an array of
 530 // instance types instead of a hash.
 531 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 532         if len(data) > 0 && data[0] == '[' {
 533                 var arr []string
 534                 err := json.Unmarshal(data, &arr)
 535                 if err != nil {
 536                         return err
 537                 }
 538                 if len(arr) == 0 {
 539                         *ss = nil
 540                         return nil
 541                 }
 542                 *ss = make(map[string]struct{}, len(arr))
 543                 for _, t := range arr {
 544                         (*ss)[t] = struct{}{}
 545                 }
 546                 return nil
 547         }
 548         var hash map[string]struct{}
 549         err := json.Unmarshal(data, &hash)
 550         if err != nil {
 551                 return err
 552         }
 553         *ss = make(map[string]struct{}, len(hash))
 554         for t, _ := range hash {
 555                 (*ss)[t] = struct{}{}
 556         }
 557
 558         return nil
 559 }
 560
 561 type ServiceName string
 562
 563 const (
 564         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 565         ServiceNameController    ServiceName = "arvados-controller"
 566         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 567         ServiceNameHealth        ServiceName = "arvados-health"
 568         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 569         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 570         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 571         ServiceNameWebsocket     ServiceName = "arvados-ws"
 572         ServiceNameKeepbalance   ServiceName = "keep-balance"
 573         ServiceNameKeepweb       ServiceName = "keep-web"
 574         ServiceNameKeepproxy     ServiceName = "keepproxy"
 575         ServiceNameKeepstore     ServiceName = "keepstore"
 576 )
 577
 578 // Map returns all services as a map, suitable for iterating over all
 579 // services or looking up a service by name.
 580 func (svcs Services) Map() map[ServiceName]Service {
 581         return map[ServiceName]Service{
 582                 ServiceNameRailsAPI:      svcs.RailsAPI,
 583                 ServiceNameController:    svcs.Controller,
 584                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 585                 ServiceNameHealth:        svcs.Health,
 586                 ServiceNameNodemanager:   svcs.Nodemanager,
 587                 ServiceNameWorkbench1:    svcs.Workbench1,
 588                 ServiceNameWorkbench2:    svcs.Workbench2,
 589                 ServiceNameWebsocket:     svcs.Websocket,
 590                 ServiceNameKeepbalance:   svcs.Keepbalance,
 591                 ServiceNameKeepweb:       svcs.WebDAV,
 592                 ServiceNameKeepproxy:     svcs.Keepproxy,
 593                 ServiceNameKeepstore:     svcs.Keepstore,
 594         }
 595 }