sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.arvados.org/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters         map[string]Cluster
  27         AutoReloadConfig bool
  28 }
  29
  30 // GetConfig returns the current system config, loading it from
  31 // configFile if needed.
  32 func GetConfig(configFile string) (*Config, error) {
  33         var cfg Config
  34         err := config.LoadFile(&cfg, configFile)
  35         return &cfg, err
  36 }
  37
  38 // GetCluster returns the cluster ID and config for the given
  39 // cluster, or the default/only configured cluster if clusterID is "".
  40 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  41         if clusterID == "" {
  42                 if len(sc.Clusters) == 0 {
  43                         return nil, fmt.Errorf("no clusters configured")
  44                 } else if len(sc.Clusters) > 1 {
  45                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  46                 } else {
  47                         for id, cc := range sc.Clusters {
  48                                 cc.ClusterID = id
  49                                 return &cc, nil
  50                         }
  51                 }
  52         }
  53         if cc, ok := sc.Clusters[clusterID]; !ok {
  54                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  55         } else {
  56                 cc.ClusterID = clusterID
  57                 return &cc, nil
  58         }
  59 }
  60
  61 type WebDAVCacheConfig struct {
  62         TTL                  Duration
  63         UUIDTTL              Duration
  64         MaxBlockEntries      int
  65         MaxCollectionEntries int
  66         MaxCollectionBytes   int64
  67         MaxPermissionEntries int
  68         MaxUUIDEntries       int
  69 }
  70
  71 type Cluster struct {
  72         ClusterID       string `json:"-"`
  73         ManagementToken string
  74         SystemRootToken string
  75         Services        Services
  76         InstanceTypes   InstanceTypeMap
  77         Containers      ContainersConfig
  78         RemoteClusters  map[string]RemoteCluster
  79         PostgreSQL      PostgreSQL
  80
  81         API struct {
  82                 AsyncPermissionsUpdateInterval Duration
  83                 DisabledAPIs                   StringSet
  84                 MaxIndexDatabaseRead           int
  85                 MaxItemsPerResponse            int
  86                 MaxConcurrentRequests          int
  87                 MaxKeepBlobBuffers             int
  88                 MaxRequestAmplification        int
  89                 MaxRequestSize                 int
  90                 RailsSessionSecretToken        string
  91                 RequestTimeout                 Duration
  92                 SendTimeout                    Duration
  93                 WebsocketClientEventQueue      int
  94                 WebsocketServerEventQueue      int
  95                 KeepServiceRequestTimeout      Duration
  96         }
  97         AuditLogs struct {
  98                 MaxAge             Duration
  99                 MaxDeleteBatch     int
 100                 UnloggedAttributes StringSet
 101         }
 102         Collections struct {
 103                 BlobSigning              bool
 104                 BlobSigningKey           string
 105                 BlobSigningTTL           Duration
 106                 BlobTrash                bool
 107                 BlobTrashLifetime        Duration
 108                 BlobTrashCheckInterval   Duration
 109                 BlobTrashConcurrency     int
 110                 BlobDeleteConcurrency    int
 111                 BlobReplicateConcurrency int
 112                 CollectionVersioning     bool
 113                 DefaultTrashLifetime     Duration
 114                 DefaultReplication       int
 115                 ManagedProperties        map[string]struct {
 116                         Value     interface{}
 117                         Function  string
 118                         Protected bool
 119                 }
 120                 PreserveVersionIfIdle        Duration
 121                 TrashSweepInterval           Duration
 122                 TrustAllContent              bool
 123                 ForwardSlashNameSubstitution string
 124
 125                 BlobMissingReport        string
 126                 BalancePeriod            Duration
 127                 BalanceCollectionBatch   int
 128                 BalanceCollectionBuffers int
 129
 130                 WebDAVCache WebDAVCacheConfig
 131         }
 132         Git struct {
 133                 GitCommand   string
 134                 GitoliteHome string
 135                 Repositories string
 136         }
 137         Login struct {
 138                 LDAP struct {
 139                         Enable             bool
 140                         URL                URL
 141                         StartTLS           bool
 142                         InsecureTLS        bool
 143                         StripDomain        string
 144                         AppendDomain       string
 145                         SearchAttribute    string
 146                         SearchBindUser     string
 147                         SearchBindPassword string
 148                         SearchBase         string
 149                         SearchFilters      string
 150                         EmailAttribute     string
 151                         UsernameAttribute  string
 152                 }
 153                 Google struct {
 154                         Enable                  bool
 155                         ClientID                string
 156                         ClientSecret            string
 157                         AlternateEmailAddresses bool
 158                 }
 159                 PAM struct {
 160                         Enable             bool
 161                         Service            string
 162                         DefaultEmailDomain string
 163                 }
 164                 SSO struct {
 165                         Enable            bool
 166                         ProviderAppID     string
 167                         ProviderAppSecret string
 168                 }
 169                 LoginCluster       string
 170                 RemoteTokenRefresh Duration
 171         }
 172         Mail struct {
 173                 MailchimpAPIKey                string
 174                 MailchimpListID                string
 175                 SendUserSetupNotificationEmail bool
 176                 IssueReporterEmailFrom         string
 177                 IssueReporterEmailTo           string
 178                 SupportEmailAddress            string
 179                 EmailFrom                      string
 180         }
 181         SystemLogs struct {
 182                 LogLevel                string
 183                 Format                  string
 184                 MaxRequestLogParamsSize int
 185         }
 186         TLS struct {
 187                 Certificate string
 188                 Key         string
 189                 Insecure    bool
 190         }
 191         Users struct {
 192                 AnonymousUserToken                    string
 193                 AdminNotifierEmailFrom                string
 194                 AutoAdminFirstUser                    bool
 195                 AutoAdminUserWithEmail                string
 196                 AutoSetupNewUsers                     bool
 197                 AutoSetupNewUsersWithRepository       bool
 198                 AutoSetupNewUsersWithVmUUID           string
 199                 AutoSetupUsernameBlacklist            StringSet
 200                 EmailSubjectPrefix                    string
 201                 NewInactiveUserNotificationRecipients StringSet
 202                 NewUserNotificationRecipients         StringSet
 203                 NewUsersAreActive                     bool
 204                 UserNotifierEmailFrom                 string
 205                 UserProfileNotificationAddress        string
 206                 PreferDomainForUsername               string
 207         }
 208         Volumes   map[string]Volume
 209         Workbench struct {
 210                 ActivationContactLink            string
 211                 APIClientConnectTimeout          Duration
 212                 APIClientReceiveTimeout          Duration
 213                 APIResponseCompression           bool
 214                 ApplicationMimetypesWithViewIcon StringSet
 215                 ArvadosDocsite                   string
 216                 ArvadosPublicDataDocURL          string
 217                 DefaultOpenIdPrefix              string
 218                 EnableGettingStartedPopup        bool
 219                 EnablePublicProjectsPage         bool
 220                 FileViewersConfigURL             string
 221                 LogViewerMaxBytes                ByteSize
 222                 MultiSiteSearch                  string
 223                 ProfilingEnabled                 bool
 224                 Repositories                     bool
 225                 RepositoryCache                  string
 226                 RunningJobLogRecordsToFetch      int
 227                 SecretKeyBase                    string
 228                 ShowRecentCollectionsOnDashboard bool
 229                 ShowUserAgreementInline          bool
 230                 ShowUserNotifications            bool
 231                 SiteName                         string
 232                 Theme                            string
 233                 UserProfileFormFields            map[string]struct {
 234                         Type                 string
 235                         FormFieldTitle       string
 236                         FormFieldDescription string
 237                         Required             bool
 238                         Position             int
 239                         Options              map[string]struct{}
 240                 }
 241                 UserProfileFormMessage string
 242                 VocabularyURL          string
 243                 WelcomePageHTML        string
 244                 InactivePageHTML       string
 245                 SSHHelpPageHTML        string
 246                 SSHHelpHostSuffix      string
 247         }
 248
 249         ForceLegacyAPI14 bool
 250 }
 251
 252 type Volume struct {
 253         AccessViaHosts   map[URL]VolumeAccess
 254         ReadOnly         bool
 255         Replication      int
 256         StorageClasses   map[string]bool
 257         Driver           string
 258         DriverParameters json.RawMessage
 259 }
 260
 261 type S3VolumeDriverParameters struct {
 262         AccessKey          string
 263         SecretKey          string
 264         Endpoint           string
 265         Region             string
 266         Bucket             string
 267         LocationConstraint bool
 268         IndexPageSize      int
 269         ConnectTimeout     Duration
 270         ReadTimeout        Duration
 271         RaceWindow         Duration
 272         UnsafeDelete       bool
 273 }
 274
 275 type AzureVolumeDriverParameters struct {
 276         StorageAccountName   string
 277         StorageAccountKey    string
 278         StorageBaseURL       string
 279         ContainerName        string
 280         RequestTimeout       Duration
 281         ListBlobsRetryDelay  Duration
 282         ListBlobsMaxAttempts int
 283 }
 284
 285 type DirectoryVolumeDriverParameters struct {
 286         Root      string
 287         Serialize bool
 288 }
 289
 290 type VolumeAccess struct {
 291         ReadOnly bool
 292 }
 293
 294 type Services struct {
 295         Composer       Service
 296         Controller     Service
 297         DispatchCloud  Service
 298         GitHTTP        Service
 299         GitSSH         Service
 300         Health         Service
 301         Keepbalance    Service
 302         Keepproxy      Service
 303         Keepstore      Service
 304         Nodemanager    Service
 305         RailsAPI       Service
 306         SSO            Service
 307         WebDAVDownload Service
 308         WebDAV         Service
 309         WebShell       Service
 310         Websocket      Service
 311         Workbench1     Service
 312         Workbench2     Service
 313 }
 314
 315 type Service struct {
 316         InternalURLs map[URL]ServiceInstance
 317         ExternalURL  URL
 318 }
 319
 320 // URL is a url.URL that is also usable as a JSON key/value.
 321 type URL url.URL
 322
 323 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 324 // used as a JSON key/value.
 325 func (su *URL) UnmarshalText(text []byte) error {
 326         u, err := url.Parse(string(text))
 327         if err == nil {
 328                 *su = URL(*u)
 329                 if su.Path == "" && su.Host != "" {
 330                         // http://example really means http://example/
 331                         su.Path = "/"
 332                 }
 333         }
 334         return err
 335 }
 336
 337 func (su URL) MarshalText() ([]byte, error) {
 338         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 339 }
 340
 341 func (su URL) String() string {
 342         return (*url.URL)(&su).String()
 343 }
 344
 345 type ServiceInstance struct {
 346         Rendezvous string `json:",omitempty"`
 347 }
 348
 349 type PostgreSQL struct {
 350         Connection     PostgreSQLConnection
 351         ConnectionPool int
 352 }
 353
 354 type PostgreSQLConnection map[string]string
 355
 356 type RemoteCluster struct {
 357         Host          string
 358         Proxy         bool
 359         Scheme        string
 360         Insecure      bool
 361         ActivateUsers bool
 362 }
 363
 364 type InstanceType struct {
 365         Name            string
 366         ProviderType    string
 367         VCPUs           int
 368         RAM             ByteSize
 369         Scratch         ByteSize
 370         IncludedScratch ByteSize
 371         AddedScratch    ByteSize
 372         Price           float64
 373         Preemptible     bool
 374 }
 375
 376 type ContainersConfig struct {
 377         CloudVMs                    CloudVMsConfig
 378         CrunchRunCommand            string
 379         CrunchRunArgumentsList      []string
 380         DefaultKeepCacheRAM         ByteSize
 381         DispatchPrivateKey          string
 382         LogReuseDecisions           bool
 383         MaxComputeVMs               int
 384         MaxDispatchAttempts         int
 385         MaxRetryAttempts            int
 386         MinRetryPeriod              Duration
 387         ReserveExtraRAM             ByteSize
 388         StaleLockTimeout            Duration
 389         SupportedDockerImageFormats StringSet
 390         UsePreemptibleInstances     bool
 391
 392         JobsAPI struct {
 393                 Enable         string
 394                 GitInternalDir string
 395         }
 396         Logging struct {
 397                 MaxAge                       Duration
 398                 LogBytesPerEvent             int
 399                 LogSecondsBetweenEvents      Duration
 400                 LogThrottlePeriod            Duration
 401                 LogThrottleBytes             int
 402                 LogThrottleLines             int
 403                 LimitLogBytesPerJob          int
 404                 LogPartialLineThrottlePeriod Duration
 405                 LogUpdatePeriod              Duration
 406                 LogUpdateSize                ByteSize
 407         }
 408         SLURM struct {
 409                 PrioritySpread             int64
 410                 SbatchArgumentsList        []string
 411                 SbatchEnvironmentVariables map[string]string
 412                 Managed                    struct {
 413                         DNSServerConfDir       string
 414                         DNSServerConfTemplate  string
 415                         DNSServerReloadCommand string
 416                         DNSServerUpdateCommand string
 417                         ComputeNodeDomain      string
 418                         ComputeNodeNameservers StringSet
 419                         AssignNodeHostname     string
 420                 }
 421         }
 422 }
 423
 424 type CloudVMsConfig struct {
 425         Enable bool
 426
 427         BootProbeCommand     string
 428         DeployRunnerBinary   string
 429         ImageID              string
 430         MaxCloudOpsPerSecond int
 431         MaxProbesPerSecond   int
 432         PollInterval         Duration
 433         ProbeInterval        Duration
 434         SSHPort              string
 435         SyncInterval         Duration
 436         TimeoutBooting       Duration
 437         TimeoutIdle          Duration
 438         TimeoutProbe         Duration
 439         TimeoutShutdown      Duration
 440         TimeoutSignal        Duration
 441         TimeoutTERM          Duration
 442         ResourceTags         map[string]string
 443         TagKeyPrefix         string
 444
 445         Driver           string
 446         DriverParameters json.RawMessage
 447 }
 448
 449 type InstanceTypeMap map[string]InstanceType
 450
 451 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 452
 453 // UnmarshalJSON handles old config files that provide an array of
 454 // instance types instead of a hash.
 455 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 456         fixup := func(t InstanceType) (InstanceType, error) {
 457                 if t.ProviderType == "" {
 458                         t.ProviderType = t.Name
 459                 }
 460                 if t.Scratch == 0 {
 461                         t.Scratch = t.IncludedScratch + t.AddedScratch
 462                 } else if t.AddedScratch == 0 {
 463                         t.AddedScratch = t.Scratch - t.IncludedScratch
 464                 } else if t.IncludedScratch == 0 {
 465                         t.IncludedScratch = t.Scratch - t.AddedScratch
 466                 }
 467
 468                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 469                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 470                 }
 471                 return t, nil
 472         }
 473
 474         if len(data) > 0 && data[0] == '[' {
 475                 var arr []InstanceType
 476                 err := json.Unmarshal(data, &arr)
 477                 if err != nil {
 478                         return err
 479                 }
 480                 if len(arr) == 0 {
 481                         *it = nil
 482                         return nil
 483                 }
 484                 *it = make(map[string]InstanceType, len(arr))
 485                 for _, t := range arr {
 486                         if _, ok := (*it)[t.Name]; ok {
 487                                 return errDuplicateInstanceTypeName
 488                         }
 489                         t, err := fixup(t)
 490                         if err != nil {
 491                                 return err
 492                         }
 493                         (*it)[t.Name] = t
 494                 }
 495                 return nil
 496         }
 497         var hash map[string]InstanceType
 498         err := json.Unmarshal(data, &hash)
 499         if err != nil {
 500                 return err
 501         }
 502         // Fill in Name field (and ProviderType field, if not
 503         // specified) using hash key.
 504         *it = InstanceTypeMap(hash)
 505         for name, t := range *it {
 506                 t.Name = name
 507                 t, err := fixup(t)
 508                 if err != nil {
 509                         return err
 510                 }
 511                 (*it)[name] = t
 512         }
 513         return nil
 514 }
 515
 516 type StringSet map[string]struct{}
 517
 518 // UnmarshalJSON handles old config files that provide an array of
 519 // instance types instead of a hash.
 520 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 521         if len(data) > 0 && data[0] == '[' {
 522                 var arr []string
 523                 err := json.Unmarshal(data, &arr)
 524                 if err != nil {
 525                         return err
 526                 }
 527                 if len(arr) == 0 {
 528                         *ss = nil
 529                         return nil
 530                 }
 531                 *ss = make(map[string]struct{}, len(arr))
 532                 for _, t := range arr {
 533                         (*ss)[t] = struct{}{}
 534                 }
 535                 return nil
 536         }
 537         var hash map[string]struct{}
 538         err := json.Unmarshal(data, &hash)
 539         if err != nil {
 540                 return err
 541         }
 542         *ss = make(map[string]struct{}, len(hash))
 543         for t, _ := range hash {
 544                 (*ss)[t] = struct{}{}
 545         }
 546
 547         return nil
 548 }
 549
 550 type ServiceName string
 551
 552 const (
 553         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 554         ServiceNameController    ServiceName = "arvados-controller"
 555         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 556         ServiceNameHealth        ServiceName = "arvados-health"
 557         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 558         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 559         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 560         ServiceNameWebsocket     ServiceName = "arvados-ws"
 561         ServiceNameKeepbalance   ServiceName = "keep-balance"
 562         ServiceNameKeepweb       ServiceName = "keep-web"
 563         ServiceNameKeepproxy     ServiceName = "keepproxy"
 564         ServiceNameKeepstore     ServiceName = "keepstore"
 565 )
 566
 567 // Map returns all services as a map, suitable for iterating over all
 568 // services or looking up a service by name.
 569 func (svcs Services) Map() map[ServiceName]Service {
 570         return map[ServiceName]Service{
 571                 ServiceNameRailsAPI:      svcs.RailsAPI,
 572                 ServiceNameController:    svcs.Controller,
 573                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 574                 ServiceNameHealth:        svcs.Health,
 575                 ServiceNameNodemanager:   svcs.Nodemanager,
 576                 ServiceNameWorkbench1:    svcs.Workbench1,
 577                 ServiceNameWorkbench2:    svcs.Workbench2,
 578                 ServiceNameWebsocket:     svcs.Websocket,
 579                 ServiceNameKeepbalance:   svcs.Keepbalance,
 580                 ServiceNameKeepweb:       svcs.WebDAV,
 581                 ServiceNameKeepproxy:     svcs.Keepproxy,
 582                 ServiceNameKeepstore:     svcs.Keepstore,
 583         }
 584 }