sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.arvados.org/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters map[string]Cluster
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         if cc, ok := sc.Clusters[clusterID]; !ok {
  53                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  54         } else {
  55                 cc.ClusterID = clusterID
  56                 return &cc, nil
  57         }
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxBlockEntries      int
  64         MaxCollectionEntries int
  65         MaxCollectionBytes   int64
  66         MaxPermissionEntries int
  67         MaxUUIDEntries       int
  68 }
  69 type Cluster struct {
  70         ClusterID       string `json:"-"`
  71         ManagementToken string
  72         SystemRootToken string
  73         Services        Services
  74         InstanceTypes   InstanceTypeMap
  75         Containers      ContainersConfig
  76         RemoteClusters  map[string]RemoteCluster
  77         PostgreSQL      PostgreSQL
  78
  79         API struct {
  80                 AsyncPermissionsUpdateInterval Duration
  81                 DisabledAPIs                   StringSet
  82                 MaxIndexDatabaseRead           int
  83                 MaxItemsPerResponse            int
  84                 MaxConcurrentRequests          int
  85                 MaxKeepBlobBuffers             int
  86                 MaxRequestAmplification        int
  87                 MaxRequestSize                 int
  88                 RailsSessionSecretToken        string
  89                 RequestTimeout                 Duration
  90                 SendTimeout                    Duration
  91                 WebsocketClientEventQueue      int
  92                 WebsocketServerEventQueue      int
  93                 KeepServiceRequestTimeout      Duration
  94         }
  95         AuditLogs struct {
  96                 MaxAge             Duration
  97                 MaxDeleteBatch     int
  98                 UnloggedAttributes StringSet
  99         }
 100         Collections struct {
 101                 BlobSigning              bool
 102                 BlobSigningKey           string
 103                 BlobSigningTTL           Duration
 104                 BlobTrash                bool
 105                 BlobTrashLifetime        Duration
 106                 BlobTrashCheckInterval   Duration
 107                 BlobTrashConcurrency     int
 108                 BlobDeleteConcurrency    int
 109                 BlobReplicateConcurrency int
 110                 CollectionVersioning     bool
 111                 DefaultTrashLifetime     Duration
 112                 DefaultReplication       int
 113                 ManagedProperties        map[string]struct {
 114                         Value     interface{}
 115                         Function  string
 116                         Protected bool
 117                 }
 118                 PreserveVersionIfIdle        Duration
 119                 TrashSweepInterval           Duration
 120                 TrustAllContent              bool
 121                 ForwardSlashNameSubstitution string
 122
 123                 BlobMissingReport        string
 124                 BalancePeriod            Duration
 125                 BalanceCollectionBatch   int
 126                 BalanceCollectionBuffers int
 127
 128                 WebDAVCache WebDAVCacheConfig
 129         }
 130         Git struct {
 131                 GitCommand   string
 132                 GitoliteHome string
 133                 Repositories string
 134         }
 135         Login struct {
 136                 GoogleClientID                string
 137                 GoogleClientSecret            string
 138                 GoogleAlternateEmailAddresses bool
 139                 ProviderAppID                 string
 140                 ProviderAppSecret             string
 141                 LoginCluster                  string
 142                 RemoteTokenRefresh            Duration
 143         }
 144         Mail struct {
 145                 MailchimpAPIKey                string
 146                 MailchimpListID                string
 147                 SendUserSetupNotificationEmail bool
 148                 IssueReporterEmailFrom         string
 149                 IssueReporterEmailTo           string
 150                 SupportEmailAddress            string
 151                 EmailFrom                      string
 152         }
 153         SystemLogs struct {
 154                 LogLevel                string
 155                 Format                  string
 156                 MaxRequestLogParamsSize int
 157         }
 158         TLS struct {
 159                 Certificate string
 160                 Key         string
 161                 Insecure    bool
 162         }
 163         Users struct {
 164                 AnonymousUserToken                    string
 165                 AdminNotifierEmailFrom                string
 166                 AutoAdminFirstUser                    bool
 167                 AutoAdminUserWithEmail                string
 168                 AutoSetupNewUsers                     bool
 169                 AutoSetupNewUsersWithRepository       bool
 170                 AutoSetupNewUsersWithVmUUID           string
 171                 AutoSetupUsernameBlacklist            StringSet
 172                 EmailSubjectPrefix                    string
 173                 NewInactiveUserNotificationRecipients StringSet
 174                 NewUserNotificationRecipients         StringSet
 175                 NewUsersAreActive                     bool
 176                 UserNotifierEmailFrom                 string
 177                 UserProfileNotificationAddress        string
 178                 PreferDomainForUsername               string
 179         }
 180         Volumes   map[string]Volume
 181         Workbench struct {
 182                 ActivationContactLink            string
 183                 APIClientConnectTimeout          Duration
 184                 APIClientReceiveTimeout          Duration
 185                 APIResponseCompression           bool
 186                 ApplicationMimetypesWithViewIcon StringSet
 187                 ArvadosDocsite                   string
 188                 ArvadosPublicDataDocURL          string
 189                 DefaultOpenIdPrefix              string
 190                 EnableGettingStartedPopup        bool
 191                 EnablePublicProjectsPage         bool
 192                 FileViewersConfigURL             string
 193                 LogViewerMaxBytes                ByteSize
 194                 MultiSiteSearch                  string
 195                 ProfilingEnabled                 bool
 196                 Repositories                     bool
 197                 RepositoryCache                  string
 198                 RunningJobLogRecordsToFetch      int
 199                 SecretKeyBase                    string
 200                 ShowRecentCollectionsOnDashboard bool
 201                 ShowUserAgreementInline          bool
 202                 ShowUserNotifications            bool
 203                 SiteName                         string
 204                 Theme                            string
 205                 UserProfileFormFields            map[string]struct {
 206                         Type                 string
 207                         FormFieldTitle       string
 208                         FormFieldDescription string
 209                         Required             bool
 210                         Position             int
 211                         Options              map[string]struct{}
 212                 }
 213                 UserProfileFormMessage string
 214                 VocabularyURL          string
 215                 WelcomePageHTML        string
 216                 InactivePageHTML       string
 217                 SSHHelpPageHTML        string
 218                 SSHHelpHostSuffix      string
 219         }
 220
 221         ForceLegacyAPI14 bool
 222 }
 223
 224 type Volume struct {
 225         AccessViaHosts   map[URL]VolumeAccess
 226         ReadOnly         bool
 227         Replication      int
 228         StorageClasses   map[string]bool
 229         Driver           string
 230         DriverParameters json.RawMessage
 231 }
 232
 233 type S3VolumeDriverParameters struct {
 234         AccessKey          string
 235         SecretKey          string
 236         Endpoint           string
 237         Region             string
 238         Bucket             string
 239         LocationConstraint bool
 240         IndexPageSize      int
 241         ConnectTimeout     Duration
 242         ReadTimeout        Duration
 243         RaceWindow         Duration
 244         UnsafeDelete       bool
 245 }
 246
 247 type AzureVolumeDriverParameters struct {
 248         StorageAccountName   string
 249         StorageAccountKey    string
 250         StorageBaseURL       string
 251         ContainerName        string
 252         RequestTimeout       Duration
 253         ListBlobsRetryDelay  Duration
 254         ListBlobsMaxAttempts int
 255 }
 256
 257 type DirectoryVolumeDriverParameters struct {
 258         Root      string
 259         Serialize bool
 260 }
 261
 262 type VolumeAccess struct {
 263         ReadOnly bool
 264 }
 265
 266 type Services struct {
 267         Composer       Service
 268         Controller     Service
 269         DispatchCloud  Service
 270         GitHTTP        Service
 271         GitSSH         Service
 272         Health         Service
 273         Keepbalance    Service
 274         Keepproxy      Service
 275         Keepstore      Service
 276         Nodemanager    Service
 277         RailsAPI       Service
 278         SSO            Service
 279         WebDAVDownload Service
 280         WebDAV         Service
 281         WebShell       Service
 282         Websocket      Service
 283         Workbench1     Service
 284         Workbench2     Service
 285 }
 286
 287 type Service struct {
 288         InternalURLs map[URL]ServiceInstance
 289         ExternalURL  URL
 290 }
 291
 292 // URL is a url.URL that is also usable as a JSON key/value.
 293 type URL url.URL
 294
 295 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 296 // used as a JSON key/value.
 297 func (su *URL) UnmarshalText(text []byte) error {
 298         u, err := url.Parse(string(text))
 299         if err == nil {
 300                 *su = URL(*u)
 301         }
 302         return err
 303 }
 304
 305 func (su URL) MarshalText() ([]byte, error) {
 306         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 307 }
 308
 309 func (su URL) String() string {
 310         return (*url.URL)(&su).String()
 311 }
 312
 313 type ServiceInstance struct {
 314         Rendezvous string `json:",omitempty"`
 315 }
 316
 317 type PostgreSQL struct {
 318         Connection     PostgreSQLConnection
 319         ConnectionPool int
 320 }
 321
 322 type PostgreSQLConnection map[string]string
 323
 324 type RemoteCluster struct {
 325         Host          string
 326         Proxy         bool
 327         Scheme        string
 328         Insecure      bool
 329         ActivateUsers bool
 330 }
 331
 332 type InstanceType struct {
 333         Name            string
 334         ProviderType    string
 335         VCPUs           int
 336         RAM             ByteSize
 337         Scratch         ByteSize
 338         IncludedScratch ByteSize
 339         AddedScratch    ByteSize
 340         Price           float64
 341         Preemptible     bool
 342 }
 343
 344 type ContainersConfig struct {
 345         CloudVMs                    CloudVMsConfig
 346         CrunchRunCommand            string
 347         CrunchRunArgumentsList      []string
 348         DefaultKeepCacheRAM         ByteSize
 349         DispatchPrivateKey          string
 350         LogReuseDecisions           bool
 351         MaxComputeVMs               int
 352         MaxDispatchAttempts         int
 353         MaxRetryAttempts            int
 354         MinRetryPeriod              Duration
 355         ReserveExtraRAM             ByteSize
 356         StaleLockTimeout            Duration
 357         SupportedDockerImageFormats StringSet
 358         UsePreemptibleInstances     bool
 359
 360         JobsAPI struct {
 361                 Enable         string
 362                 GitInternalDir string
 363         }
 364         Logging struct {
 365                 MaxAge                       Duration
 366                 LogBytesPerEvent             int
 367                 LogSecondsBetweenEvents      Duration
 368                 LogThrottlePeriod            Duration
 369                 LogThrottleBytes             int
 370                 LogThrottleLines             int
 371                 LimitLogBytesPerJob          int
 372                 LogPartialLineThrottlePeriod Duration
 373                 LogUpdatePeriod              Duration
 374                 LogUpdateSize                ByteSize
 375         }
 376         SLURM struct {
 377                 PrioritySpread             int64
 378                 SbatchArgumentsList        []string
 379                 SbatchEnvironmentVariables map[string]string
 380                 Managed                    struct {
 381                         DNSServerConfDir       string
 382                         DNSServerConfTemplate  string
 383                         DNSServerReloadCommand string
 384                         DNSServerUpdateCommand string
 385                         ComputeNodeDomain      string
 386                         ComputeNodeNameservers StringSet
 387                         AssignNodeHostname     string
 388                 }
 389         }
 390 }
 391
 392 type CloudVMsConfig struct {
 393         Enable bool
 394
 395         BootProbeCommand     string
 396         DeployRunnerBinary   string
 397         ImageID              string
 398         MaxCloudOpsPerSecond int
 399         MaxProbesPerSecond   int
 400         PollInterval         Duration
 401         ProbeInterval        Duration
 402         SSHPort              string
 403         SyncInterval         Duration
 404         TimeoutBooting       Duration
 405         TimeoutIdle          Duration
 406         TimeoutProbe         Duration
 407         TimeoutShutdown      Duration
 408         TimeoutSignal        Duration
 409         TimeoutTERM          Duration
 410         ResourceTags         map[string]string
 411         TagKeyPrefix         string
 412
 413         Driver           string
 414         DriverParameters json.RawMessage
 415 }
 416
 417 type InstanceTypeMap map[string]InstanceType
 418
 419 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 420
 421 // UnmarshalJSON handles old config files that provide an array of
 422 // instance types instead of a hash.
 423 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 424         fixup := func(t InstanceType) (InstanceType, error) {
 425                 if t.ProviderType == "" {
 426                         t.ProviderType = t.Name
 427                 }
 428                 if t.Scratch == 0 {
 429                         t.Scratch = t.IncludedScratch + t.AddedScratch
 430                 } else if t.AddedScratch == 0 {
 431                         t.AddedScratch = t.Scratch - t.IncludedScratch
 432                 } else if t.IncludedScratch == 0 {
 433                         t.IncludedScratch = t.Scratch - t.AddedScratch
 434                 }
 435
 436                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 437                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 438                 }
 439                 return t, nil
 440         }
 441
 442         if len(data) > 0 && data[0] == '[' {
 443                 var arr []InstanceType
 444                 err := json.Unmarshal(data, &arr)
 445                 if err != nil {
 446                         return err
 447                 }
 448                 if len(arr) == 0 {
 449                         *it = nil
 450                         return nil
 451                 }
 452                 *it = make(map[string]InstanceType, len(arr))
 453                 for _, t := range arr {
 454                         if _, ok := (*it)[t.Name]; ok {
 455                                 return errDuplicateInstanceTypeName
 456                         }
 457                         t, err := fixup(t)
 458                         if err != nil {
 459                                 return err
 460                         }
 461                         (*it)[t.Name] = t
 462                 }
 463                 return nil
 464         }
 465         var hash map[string]InstanceType
 466         err := json.Unmarshal(data, &hash)
 467         if err != nil {
 468                 return err
 469         }
 470         // Fill in Name field (and ProviderType field, if not
 471         // specified) using hash key.
 472         *it = InstanceTypeMap(hash)
 473         for name, t := range *it {
 474                 t.Name = name
 475                 t, err := fixup(t)
 476                 if err != nil {
 477                         return err
 478                 }
 479                 (*it)[name] = t
 480         }
 481         return nil
 482 }
 483
 484 type StringSet map[string]struct{}
 485
 486 // UnmarshalJSON handles old config files that provide an array of
 487 // instance types instead of a hash.
 488 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 489         if len(data) > 0 && data[0] == '[' {
 490                 var arr []string
 491                 err := json.Unmarshal(data, &arr)
 492                 if err != nil {
 493                         return err
 494                 }
 495                 if len(arr) == 0 {
 496                         *ss = nil
 497                         return nil
 498                 }
 499                 *ss = make(map[string]struct{}, len(arr))
 500                 for _, t := range arr {
 501                         (*ss)[t] = struct{}{}
 502                 }
 503                 return nil
 504         }
 505         var hash map[string]struct{}
 506         err := json.Unmarshal(data, &hash)
 507         if err != nil {
 508                 return err
 509         }
 510         *ss = make(map[string]struct{}, len(hash))
 511         for t, _ := range hash {
 512                 (*ss)[t] = struct{}{}
 513         }
 514
 515         return nil
 516 }
 517
 518 type ServiceName string
 519
 520 const (
 521         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 522         ServiceNameController    ServiceName = "arvados-controller"
 523         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 524         ServiceNameHealth        ServiceName = "arvados-health"
 525         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 526         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 527         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 528         ServiceNameWebsocket     ServiceName = "arvados-ws"
 529         ServiceNameKeepbalance   ServiceName = "keep-balance"
 530         ServiceNameKeepweb       ServiceName = "keep-web"
 531         ServiceNameKeepproxy     ServiceName = "keepproxy"
 532         ServiceNameKeepstore     ServiceName = "keepstore"
 533 )
 534
 535 // Map returns all services as a map, suitable for iterating over all
 536 // services or looking up a service by name.
 537 func (svcs Services) Map() map[ServiceName]Service {
 538         return map[ServiceName]Service{
 539                 ServiceNameRailsAPI:      svcs.RailsAPI,
 540                 ServiceNameController:    svcs.Controller,
 541                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 542                 ServiceNameHealth:        svcs.Health,
 543                 ServiceNameNodemanager:   svcs.Nodemanager,
 544                 ServiceNameWorkbench1:    svcs.Workbench1,
 545                 ServiceNameWorkbench2:    svcs.Workbench2,
 546                 ServiceNameWebsocket:     svcs.Websocket,
 547                 ServiceNameKeepbalance:   svcs.Keepbalance,
 548                 ServiceNameKeepweb:       svcs.WebDAV,
 549                 ServiceNameKeepproxy:     svcs.Keepproxy,
 550                 ServiceNameKeepstore:     svcs.Keepstore,
 551         }
 552 }