sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.arvados.org/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters map[string]Cluster
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         if cc, ok := sc.Clusters[clusterID]; !ok {
  53                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  54         } else {
  55                 cc.ClusterID = clusterID
  56                 return &cc, nil
  57         }
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxBlockEntries      int
  64         MaxCollectionEntries int
  65         MaxCollectionBytes   int64
  66         MaxPermissionEntries int
  67         MaxUUIDEntries       int
  68 }
  69 type Cluster struct {
  70         ClusterID       string `json:"-"`
  71         ManagementToken string
  72         SystemRootToken string
  73         Services        Services
  74         InstanceTypes   InstanceTypeMap
  75         Containers      ContainersConfig
  76         RemoteClusters  map[string]RemoteCluster
  77         PostgreSQL      PostgreSQL
  78
  79         API struct {
  80                 AsyncPermissionsUpdateInterval Duration
  81                 DisabledAPIs                   StringSet
  82                 MaxIndexDatabaseRead           int
  83                 MaxItemsPerResponse            int
  84                 MaxConcurrentRequests          int
  85                 MaxKeepBlobBuffers             int
  86                 MaxRequestAmplification        int
  87                 MaxRequestSize                 int
  88                 RailsSessionSecretToken        string
  89                 RequestTimeout                 Duration
  90                 SendTimeout                    Duration
  91                 WebsocketClientEventQueue      int
  92                 WebsocketServerEventQueue      int
  93                 KeepServiceRequestTimeout      Duration
  94         }
  95         AuditLogs struct {
  96                 MaxAge             Duration
  97                 MaxDeleteBatch     int
  98                 UnloggedAttributes StringSet
  99         }
 100         Collections struct {
 101                 BlobSigning              bool
 102                 BlobSigningKey           string
 103                 BlobSigningTTL           Duration
 104                 BlobTrash                bool
 105                 BlobTrashLifetime        Duration
 106                 BlobTrashCheckInterval   Duration
 107                 BlobTrashConcurrency     int
 108                 BlobDeleteConcurrency    int
 109                 BlobReplicateConcurrency int
 110                 CollectionVersioning     bool
 111                 DefaultTrashLifetime     Duration
 112                 DefaultReplication       int
 113                 ManagedProperties        map[string]struct {
 114                         Value     interface{}
 115                         Function  string
 116                         Protected bool
 117                 }
 118                 PreserveVersionIfIdle        Duration
 119                 TrashSweepInterval           Duration
 120                 TrustAllContent              bool
 121                 ForwardSlashNameSubstitution string
 122
 123                 BlobMissingReport        string
 124                 BalancePeriod            Duration
 125                 BalanceCollectionBatch   int
 126                 BalanceCollectionBuffers int
 127
 128                 WebDAVCache WebDAVCacheConfig
 129         }
 130         Git struct {
 131                 GitCommand   string
 132                 GitoliteHome string
 133                 Repositories string
 134         }
 135         Login struct {
 136                 GoogleClientID                string
 137                 GoogleClientSecret            string
 138                 GoogleAlternateEmailAddresses bool
 139                 PAM                           bool
 140                 PAMService                    string
 141                 PAMDefaultEmailDomain         string
 142                 ProviderAppID                 string
 143                 ProviderAppSecret             string
 144                 LoginCluster                  string
 145                 RemoteTokenRefresh            Duration
 146         }
 147         Mail struct {
 148                 MailchimpAPIKey                string
 149                 MailchimpListID                string
 150                 SendUserSetupNotificationEmail bool
 151                 IssueReporterEmailFrom         string
 152                 IssueReporterEmailTo           string
 153                 SupportEmailAddress            string
 154                 EmailFrom                      string
 155         }
 156         SystemLogs struct {
 157                 LogLevel                string
 158                 Format                  string
 159                 MaxRequestLogParamsSize int
 160         }
 161         TLS struct {
 162                 Certificate string
 163                 Key         string
 164                 Insecure    bool
 165         }
 166         Users struct {
 167                 AnonymousUserToken                    string
 168                 AdminNotifierEmailFrom                string
 169                 AutoAdminFirstUser                    bool
 170                 AutoAdminUserWithEmail                string
 171                 AutoSetupNewUsers                     bool
 172                 AutoSetupNewUsersWithRepository       bool
 173                 AutoSetupNewUsersWithVmUUID           string
 174                 AutoSetupUsernameBlacklist            StringSet
 175                 EmailSubjectPrefix                    string
 176                 NewInactiveUserNotificationRecipients StringSet
 177                 NewUserNotificationRecipients         StringSet
 178                 NewUsersAreActive                     bool
 179                 UserNotifierEmailFrom                 string
 180                 UserProfileNotificationAddress        string
 181                 PreferDomainForUsername               string
 182         }
 183         Volumes   map[string]Volume
 184         Workbench struct {
 185                 ActivationContactLink            string
 186                 APIClientConnectTimeout          Duration
 187                 APIClientReceiveTimeout          Duration
 188                 APIResponseCompression           bool
 189                 ApplicationMimetypesWithViewIcon StringSet
 190                 ArvadosDocsite                   string
 191                 ArvadosPublicDataDocURL          string
 192                 DefaultOpenIdPrefix              string
 193                 EnableGettingStartedPopup        bool
 194                 EnablePublicProjectsPage         bool
 195                 FileViewersConfigURL             string
 196                 LogViewerMaxBytes                ByteSize
 197                 MultiSiteSearch                  string
 198                 ProfilingEnabled                 bool
 199                 Repositories                     bool
 200                 RepositoryCache                  string
 201                 RunningJobLogRecordsToFetch      int
 202                 SecretKeyBase                    string
 203                 ShowRecentCollectionsOnDashboard bool
 204                 ShowUserAgreementInline          bool
 205                 ShowUserNotifications            bool
 206                 SiteName                         string
 207                 Theme                            string
 208                 UserProfileFormFields            map[string]struct {
 209                         Type                 string
 210                         FormFieldTitle       string
 211                         FormFieldDescription string
 212                         Required             bool
 213                         Position             int
 214                         Options              map[string]struct{}
 215                 }
 216                 UserProfileFormMessage string
 217                 VocabularyURL          string
 218                 WelcomePageHTML        string
 219                 InactivePageHTML       string
 220                 SSHHelpPageHTML        string
 221                 SSHHelpHostSuffix      string
 222         }
 223
 224         ForceLegacyAPI14 bool
 225 }
 226
 227 type Volume struct {
 228         AccessViaHosts   map[URL]VolumeAccess
 229         ReadOnly         bool
 230         Replication      int
 231         StorageClasses   map[string]bool
 232         Driver           string
 233         DriverParameters json.RawMessage
 234 }
 235
 236 type S3VolumeDriverParameters struct {
 237         AccessKey          string
 238         SecretKey          string
 239         Endpoint           string
 240         Region             string
 241         Bucket             string
 242         LocationConstraint bool
 243         IndexPageSize      int
 244         ConnectTimeout     Duration
 245         ReadTimeout        Duration
 246         RaceWindow         Duration
 247         UnsafeDelete       bool
 248 }
 249
 250 type AzureVolumeDriverParameters struct {
 251         StorageAccountName   string
 252         StorageAccountKey    string
 253         StorageBaseURL       string
 254         ContainerName        string
 255         RequestTimeout       Duration
 256         ListBlobsRetryDelay  Duration
 257         ListBlobsMaxAttempts int
 258 }
 259
 260 type DirectoryVolumeDriverParameters struct {
 261         Root      string
 262         Serialize bool
 263 }
 264
 265 type VolumeAccess struct {
 266         ReadOnly bool
 267 }
 268
 269 type Services struct {
 270         Composer       Service
 271         Controller     Service
 272         DispatchCloud  Service
 273         GitHTTP        Service
 274         GitSSH         Service
 275         Health         Service
 276         Keepbalance    Service
 277         Keepproxy      Service
 278         Keepstore      Service
 279         Nodemanager    Service
 280         RailsAPI       Service
 281         SSO            Service
 282         WebDAVDownload Service
 283         WebDAV         Service
 284         WebShell       Service
 285         Websocket      Service
 286         Workbench1     Service
 287         Workbench2     Service
 288 }
 289
 290 type Service struct {
 291         InternalURLs map[URL]ServiceInstance
 292         ExternalURL  URL
 293 }
 294
 295 // URL is a url.URL that is also usable as a JSON key/value.
 296 type URL url.URL
 297
 298 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 299 // used as a JSON key/value.
 300 func (su *URL) UnmarshalText(text []byte) error {
 301         u, err := url.Parse(string(text))
 302         if err == nil {
 303                 *su = URL(*u)
 304         }
 305         return err
 306 }
 307
 308 func (su URL) MarshalText() ([]byte, error) {
 309         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 310 }
 311
 312 func (su URL) String() string {
 313         return (*url.URL)(&su).String()
 314 }
 315
 316 type ServiceInstance struct {
 317         Rendezvous string `json:",omitempty"`
 318 }
 319
 320 type PostgreSQL struct {
 321         Connection     PostgreSQLConnection
 322         ConnectionPool int
 323 }
 324
 325 type PostgreSQLConnection map[string]string
 326
 327 type RemoteCluster struct {
 328         Host          string
 329         Proxy         bool
 330         Scheme        string
 331         Insecure      bool
 332         ActivateUsers bool
 333 }
 334
 335 type InstanceType struct {
 336         Name            string
 337         ProviderType    string
 338         VCPUs           int
 339         RAM             ByteSize
 340         Scratch         ByteSize
 341         IncludedScratch ByteSize
 342         AddedScratch    ByteSize
 343         Price           float64
 344         Preemptible     bool
 345 }
 346
 347 type ContainersConfig struct {
 348         CloudVMs                    CloudVMsConfig
 349         CrunchRunCommand            string
 350         CrunchRunArgumentsList      []string
 351         DefaultKeepCacheRAM         ByteSize
 352         DispatchPrivateKey          string
 353         LogReuseDecisions           bool
 354         MaxComputeVMs               int
 355         MaxDispatchAttempts         int
 356         MaxRetryAttempts            int
 357         MinRetryPeriod              Duration
 358         ReserveExtraRAM             ByteSize
 359         StaleLockTimeout            Duration
 360         SupportedDockerImageFormats StringSet
 361         UsePreemptibleInstances     bool
 362
 363         JobsAPI struct {
 364                 Enable         string
 365                 GitInternalDir string
 366         }
 367         Logging struct {
 368                 MaxAge                       Duration
 369                 LogBytesPerEvent             int
 370                 LogSecondsBetweenEvents      Duration
 371                 LogThrottlePeriod            Duration
 372                 LogThrottleBytes             int
 373                 LogThrottleLines             int
 374                 LimitLogBytesPerJob          int
 375                 LogPartialLineThrottlePeriod Duration
 376                 LogUpdatePeriod              Duration
 377                 LogUpdateSize                ByteSize
 378         }
 379         SLURM struct {
 380                 PrioritySpread             int64
 381                 SbatchArgumentsList        []string
 382                 SbatchEnvironmentVariables map[string]string
 383                 Managed                    struct {
 384                         DNSServerConfDir       string
 385                         DNSServerConfTemplate  string
 386                         DNSServerReloadCommand string
 387                         DNSServerUpdateCommand string
 388                         ComputeNodeDomain      string
 389                         ComputeNodeNameservers StringSet
 390                         AssignNodeHostname     string
 391                 }
 392         }
 393 }
 394
 395 type CloudVMsConfig struct {
 396         Enable bool
 397
 398         BootProbeCommand     string
 399         DeployRunnerBinary   string
 400         ImageID              string
 401         MaxCloudOpsPerSecond int
 402         MaxProbesPerSecond   int
 403         PollInterval         Duration
 404         ProbeInterval        Duration
 405         SSHPort              string
 406         SyncInterval         Duration
 407         TimeoutBooting       Duration
 408         TimeoutIdle          Duration
 409         TimeoutProbe         Duration
 410         TimeoutShutdown      Duration
 411         TimeoutSignal        Duration
 412         TimeoutTERM          Duration
 413         ResourceTags         map[string]string
 414         TagKeyPrefix         string
 415
 416         Driver           string
 417         DriverParameters json.RawMessage
 418 }
 419
 420 type InstanceTypeMap map[string]InstanceType
 421
 422 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 423
 424 // UnmarshalJSON handles old config files that provide an array of
 425 // instance types instead of a hash.
 426 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 427         fixup := func(t InstanceType) (InstanceType, error) {
 428                 if t.ProviderType == "" {
 429                         t.ProviderType = t.Name
 430                 }
 431                 if t.Scratch == 0 {
 432                         t.Scratch = t.IncludedScratch + t.AddedScratch
 433                 } else if t.AddedScratch == 0 {
 434                         t.AddedScratch = t.Scratch - t.IncludedScratch
 435                 } else if t.IncludedScratch == 0 {
 436                         t.IncludedScratch = t.Scratch - t.AddedScratch
 437                 }
 438
 439                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 440                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 441                 }
 442                 return t, nil
 443         }
 444
 445         if len(data) > 0 && data[0] == '[' {
 446                 var arr []InstanceType
 447                 err := json.Unmarshal(data, &arr)
 448                 if err != nil {
 449                         return err
 450                 }
 451                 if len(arr) == 0 {
 452                         *it = nil
 453                         return nil
 454                 }
 455                 *it = make(map[string]InstanceType, len(arr))
 456                 for _, t := range arr {
 457                         if _, ok := (*it)[t.Name]; ok {
 458                                 return errDuplicateInstanceTypeName
 459                         }
 460                         t, err := fixup(t)
 461                         if err != nil {
 462                                 return err
 463                         }
 464                         (*it)[t.Name] = t
 465                 }
 466                 return nil
 467         }
 468         var hash map[string]InstanceType
 469         err := json.Unmarshal(data, &hash)
 470         if err != nil {
 471                 return err
 472         }
 473         // Fill in Name field (and ProviderType field, if not
 474         // specified) using hash key.
 475         *it = InstanceTypeMap(hash)
 476         for name, t := range *it {
 477                 t.Name = name
 478                 t, err := fixup(t)
 479                 if err != nil {
 480                         return err
 481                 }
 482                 (*it)[name] = t
 483         }
 484         return nil
 485 }
 486
 487 type StringSet map[string]struct{}
 488
 489 // UnmarshalJSON handles old config files that provide an array of
 490 // instance types instead of a hash.
 491 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 492         if len(data) > 0 && data[0] == '[' {
 493                 var arr []string
 494                 err := json.Unmarshal(data, &arr)
 495                 if err != nil {
 496                         return err
 497                 }
 498                 if len(arr) == 0 {
 499                         *ss = nil
 500                         return nil
 501                 }
 502                 *ss = make(map[string]struct{}, len(arr))
 503                 for _, t := range arr {
 504                         (*ss)[t] = struct{}{}
 505                 }
 506                 return nil
 507         }
 508         var hash map[string]struct{}
 509         err := json.Unmarshal(data, &hash)
 510         if err != nil {
 511                 return err
 512         }
 513         *ss = make(map[string]struct{}, len(hash))
 514         for t, _ := range hash {
 515                 (*ss)[t] = struct{}{}
 516         }
 517
 518         return nil
 519 }
 520
 521 type ServiceName string
 522
 523 const (
 524         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 525         ServiceNameController    ServiceName = "arvados-controller"
 526         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 527         ServiceNameHealth        ServiceName = "arvados-health"
 528         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 529         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 530         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 531         ServiceNameWebsocket     ServiceName = "arvados-ws"
 532         ServiceNameKeepbalance   ServiceName = "keep-balance"
 533         ServiceNameKeepweb       ServiceName = "keep-web"
 534         ServiceNameKeepproxy     ServiceName = "keepproxy"
 535         ServiceNameKeepstore     ServiceName = "keepstore"
 536 )
 537
 538 // Map returns all services as a map, suitable for iterating over all
 539 // services or looking up a service by name.
 540 func (svcs Services) Map() map[ServiceName]Service {
 541         return map[ServiceName]Service{
 542                 ServiceNameRailsAPI:      svcs.RailsAPI,
 543                 ServiceNameController:    svcs.Controller,
 544                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 545                 ServiceNameHealth:        svcs.Health,
 546                 ServiceNameNodemanager:   svcs.Nodemanager,
 547                 ServiceNameWorkbench1:    svcs.Workbench1,
 548                 ServiceNameWorkbench2:    svcs.Workbench2,
 549                 ServiceNameWebsocket:     svcs.Websocket,
 550                 ServiceNameKeepbalance:   svcs.Keepbalance,
 551                 ServiceNameKeepweb:       svcs.WebDAV,
 552                 ServiceNameKeepproxy:     svcs.Keepproxy,
 553                 ServiceNameKeepstore:     svcs.Keepstore,
 554         }
 555 }