sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.arvados.org/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters         map[string]Cluster
  27         AutoReloadConfig bool
  28 }
  29
  30 // GetConfig returns the current system config, loading it from
  31 // configFile if needed.
  32 func GetConfig(configFile string) (*Config, error) {
  33         var cfg Config
  34         err := config.LoadFile(&cfg, configFile)
  35         return &cfg, err
  36 }
  37
  38 // GetCluster returns the cluster ID and config for the given
  39 // cluster, or the default/only configured cluster if clusterID is "".
  40 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  41         if clusterID == "" {
  42                 if len(sc.Clusters) == 0 {
  43                         return nil, fmt.Errorf("no clusters configured")
  44                 } else if len(sc.Clusters) > 1 {
  45                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  46                 } else {
  47                         for id, cc := range sc.Clusters {
  48                                 cc.ClusterID = id
  49                                 return &cc, nil
  50                         }
  51                 }
  52         }
  53         if cc, ok := sc.Clusters[clusterID]; !ok {
  54                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  55         } else {
  56                 cc.ClusterID = clusterID
  57                 return &cc, nil
  58         }
  59 }
  60
  61 type WebDAVCacheConfig struct {
  62         TTL                  Duration
  63         UUIDTTL              Duration
  64         MaxBlockEntries      int
  65         MaxCollectionEntries int
  66         MaxCollectionBytes   int64
  67         MaxPermissionEntries int
  68         MaxUUIDEntries       int
  69 }
  70
  71 type Cluster struct {
  72         ClusterID       string `json:"-"`
  73         ManagementToken string
  74         SystemRootToken string
  75         Services        Services
  76         InstanceTypes   InstanceTypeMap
  77         Containers      ContainersConfig
  78         RemoteClusters  map[string]RemoteCluster
  79         PostgreSQL      PostgreSQL
  80
  81         API struct {
  82                 AsyncPermissionsUpdateInterval Duration
  83                 DisabledAPIs                   StringSet
  84                 MaxIndexDatabaseRead           int
  85                 MaxItemsPerResponse            int
  86                 MaxConcurrentRequests          int
  87                 MaxKeepBlobBuffers             int
  88                 MaxRequestAmplification        int
  89                 MaxRequestSize                 int
  90                 RailsSessionSecretToken        string
  91                 RequestTimeout                 Duration
  92                 SendTimeout                    Duration
  93                 WebsocketClientEventQueue      int
  94                 WebsocketServerEventQueue      int
  95                 KeepServiceRequestTimeout      Duration
  96         }
  97         AuditLogs struct {
  98                 MaxAge             Duration
  99                 MaxDeleteBatch     int
 100                 UnloggedAttributes StringSet
 101         }
 102         Collections struct {
 103                 BlobSigning              bool
 104                 BlobSigningKey           string
 105                 BlobSigningTTL           Duration
 106                 BlobTrash                bool
 107                 BlobTrashLifetime        Duration
 108                 BlobTrashCheckInterval   Duration
 109                 BlobTrashConcurrency     int
 110                 BlobDeleteConcurrency    int
 111                 BlobReplicateConcurrency int
 112                 CollectionVersioning     bool
 113                 DefaultTrashLifetime     Duration
 114                 DefaultReplication       int
 115                 ManagedProperties        map[string]struct {
 116                         Value     interface{}
 117                         Function  string
 118                         Protected bool
 119                 }
 120                 PreserveVersionIfIdle        Duration
 121                 TrashSweepInterval           Duration
 122                 TrustAllContent              bool
 123                 ForwardSlashNameSubstitution string
 124
 125                 BlobMissingReport        string
 126                 BalancePeriod            Duration
 127                 BalanceCollectionBatch   int
 128                 BalanceCollectionBuffers int
 129
 130                 WebDAVCache WebDAVCacheConfig
 131         }
 132         Git struct {
 133                 GitCommand   string
 134                 GitoliteHome string
 135                 Repositories string
 136         }
 137         Login struct {
 138                 LDAP struct {
 139                         Enable             bool
 140                         URL                URL
 141                         StartTLS           bool
 142                         InsecureTLS        bool
 143                         StripDomain        string
 144                         AppendDomain       string
 145                         SearchAttribute    string
 146                         SearchBindUser     string
 147                         SearchBindPassword string
 148                         SearchBase         string
 149                         SearchFilters      string
 150                         EmailAttribute     string
 151                         UsernameAttribute  string
 152                 }
 153                 Google struct {
 154                         Enable                  bool
 155                         ClientID                string
 156                         ClientSecret            string
 157                         AlternateEmailAddresses bool
 158                 }
 159                 PAM struct {
 160                         Enable             bool
 161                         Service            string
 162                         DefaultEmailDomain string
 163                 }
 164                 SSO struct {
 165                         Enable            bool
 166                         ProviderAppID     string
 167                         ProviderAppSecret string
 168                 }
 169                 LoginCluster       string
 170                 RemoteTokenRefresh Duration
 171         }
 172         Mail struct {
 173                 MailchimpAPIKey                string
 174                 MailchimpListID                string
 175                 SendUserSetupNotificationEmail bool
 176                 IssueReporterEmailFrom         string
 177                 IssueReporterEmailTo           string
 178                 SupportEmailAddress            string
 179                 EmailFrom                      string
 180         }
 181         SystemLogs struct {
 182                 LogLevel                string
 183                 Format                  string
 184                 MaxRequestLogParamsSize int
 185         }
 186         TLS struct {
 187                 Certificate string
 188                 Key         string
 189                 Insecure    bool
 190         }
 191         Users struct {
 192                 AnonymousUserToken                    string
 193                 AdminNotifierEmailFrom                string
 194                 AutoAdminFirstUser                    bool
 195                 AutoAdminUserWithEmail                string
 196                 AutoSetupNewUsers                     bool
 197                 AutoSetupNewUsersWithRepository       bool
 198                 AutoSetupNewUsersWithVmUUID           string
 199                 AutoSetupUsernameBlacklist            StringSet
 200                 EmailSubjectPrefix                    string
 201                 NewInactiveUserNotificationRecipients StringSet
 202                 NewUserNotificationRecipients         StringSet
 203                 NewUsersAreActive                     bool
 204                 UserNotifierEmailFrom                 string
 205                 UserProfileNotificationAddress        string
 206                 PreferDomainForUsername               string
 207         }
 208         Volumes   map[string]Volume
 209         Workbench struct {
 210                 ActivationContactLink            string
 211                 APIClientConnectTimeout          Duration
 212                 APIClientReceiveTimeout          Duration
 213                 APIResponseCompression           bool
 214                 ApplicationMimetypesWithViewIcon StringSet
 215                 ArvadosDocsite                   string
 216                 ArvadosPublicDataDocURL          string
 217                 DefaultOpenIdPrefix              string
 218                 EnableGettingStartedPopup        bool
 219                 EnablePublicProjectsPage         bool
 220                 FileViewersConfigURL             string
 221                 LogViewerMaxBytes                ByteSize
 222                 MultiSiteSearch                  string
 223                 ProfilingEnabled                 bool
 224                 Repositories                     bool
 225                 RepositoryCache                  string
 226                 RunningJobLogRecordsToFetch      int
 227                 SecretKeyBase                    string
 228                 ShowRecentCollectionsOnDashboard bool
 229                 ShowUserAgreementInline          bool
 230                 ShowUserNotifications            bool
 231                 SiteName                         string
 232                 Theme                            string
 233                 UserProfileFormFields            map[string]struct {
 234                         Type                 string
 235                         FormFieldTitle       string
 236                         FormFieldDescription string
 237                         Required             bool
 238                         Position             int
 239                         Options              map[string]struct{}
 240                 }
 241                 UserProfileFormMessage string
 242                 VocabularyURL          string
 243                 WelcomePageHTML        string
 244                 InactivePageHTML       string
 245                 SSHHelpPageHTML        string
 246                 SSHHelpHostSuffix      string
 247         }
 248
 249         ForceLegacyAPI14 bool
 250 }
 251
 252 type Volume struct {
 253         AccessViaHosts   map[URL]VolumeAccess
 254         ReadOnly         bool
 255         Replication      int
 256         StorageClasses   map[string]bool
 257         Driver           string
 258         DriverParameters json.RawMessage
 259 }
 260
 261 type S3VolumeDriverParameters struct {
 262         IAMRole            string
 263         AccessKey          string
 264         SecretKey          string
 265         Endpoint           string
 266         Region             string
 267         Bucket             string
 268         LocationConstraint bool
 269         V2Signature        bool
 270         IndexPageSize      int
 271         ConnectTimeout     Duration
 272         ReadTimeout        Duration
 273         RaceWindow         Duration
 274         UnsafeDelete       bool
 275 }
 276
 277 type AzureVolumeDriverParameters struct {
 278         StorageAccountName   string
 279         StorageAccountKey    string
 280         StorageBaseURL       string
 281         ContainerName        string
 282         RequestTimeout       Duration
 283         ListBlobsRetryDelay  Duration
 284         ListBlobsMaxAttempts int
 285 }
 286
 287 type DirectoryVolumeDriverParameters struct {
 288         Root      string
 289         Serialize bool
 290 }
 291
 292 type VolumeAccess struct {
 293         ReadOnly bool
 294 }
 295
 296 type Services struct {
 297         Composer       Service
 298         Controller     Service
 299         DispatchCloud  Service
 300         GitHTTP        Service
 301         GitSSH         Service
 302         Health         Service
 303         Keepbalance    Service
 304         Keepproxy      Service
 305         Keepstore      Service
 306         Nodemanager    Service
 307         RailsAPI       Service
 308         SSO            Service
 309         WebDAVDownload Service
 310         WebDAV         Service
 311         WebShell       Service
 312         Websocket      Service
 313         Workbench1     Service
 314         Workbench2     Service
 315 }
 316
 317 type Service struct {
 318         InternalURLs map[URL]ServiceInstance
 319         ExternalURL  URL
 320 }
 321
 322 // URL is a url.URL that is also usable as a JSON key/value.
 323 type URL url.URL
 324
 325 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 326 // used as a JSON key/value.
 327 func (su *URL) UnmarshalText(text []byte) error {
 328         u, err := url.Parse(string(text))
 329         if err == nil {
 330                 *su = URL(*u)
 331                 if su.Path == "" && su.Host != "" {
 332                         // http://example really means http://example/
 333                         su.Path = "/"
 334                 }
 335         }
 336         return err
 337 }
 338
 339 func (su URL) MarshalText() ([]byte, error) {
 340         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 341 }
 342
 343 func (su URL) String() string {
 344         return (*url.URL)(&su).String()
 345 }
 346
 347 type ServiceInstance struct {
 348         Rendezvous string `json:",omitempty"`
 349 }
 350
 351 type PostgreSQL struct {
 352         Connection     PostgreSQLConnection
 353         ConnectionPool int
 354 }
 355
 356 type PostgreSQLConnection map[string]string
 357
 358 type RemoteCluster struct {
 359         Host          string
 360         Proxy         bool
 361         Scheme        string
 362         Insecure      bool
 363         ActivateUsers bool
 364 }
 365
 366 type InstanceType struct {
 367         Name            string
 368         ProviderType    string
 369         VCPUs           int
 370         RAM             ByteSize
 371         Scratch         ByteSize
 372         IncludedScratch ByteSize
 373         AddedScratch    ByteSize
 374         Price           float64
 375         Preemptible     bool
 376 }
 377
 378 type ContainersConfig struct {
 379         CloudVMs                    CloudVMsConfig
 380         CrunchRunCommand            string
 381         CrunchRunArgumentsList      []string
 382         DefaultKeepCacheRAM         ByteSize
 383         DispatchPrivateKey          string
 384         LogReuseDecisions           bool
 385         MaxComputeVMs               int
 386         MaxDispatchAttempts         int
 387         MaxRetryAttempts            int
 388         MinRetryPeriod              Duration
 389         ReserveExtraRAM             ByteSize
 390         StaleLockTimeout            Duration
 391         SupportedDockerImageFormats StringSet
 392         UsePreemptibleInstances     bool
 393
 394         JobsAPI struct {
 395                 Enable         string
 396                 GitInternalDir string
 397         }
 398         Logging struct {
 399                 MaxAge                       Duration
 400                 LogBytesPerEvent             int
 401                 LogSecondsBetweenEvents      Duration
 402                 LogThrottlePeriod            Duration
 403                 LogThrottleBytes             int
 404                 LogThrottleLines             int
 405                 LimitLogBytesPerJob          int
 406                 LogPartialLineThrottlePeriod Duration
 407                 LogUpdatePeriod              Duration
 408                 LogUpdateSize                ByteSize
 409         }
 410         SLURM struct {
 411                 PrioritySpread             int64
 412                 SbatchArgumentsList        []string
 413                 SbatchEnvironmentVariables map[string]string
 414                 Managed                    struct {
 415                         DNSServerConfDir       string
 416                         DNSServerConfTemplate  string
 417                         DNSServerReloadCommand string
 418                         DNSServerUpdateCommand string
 419                         ComputeNodeDomain      string
 420                         ComputeNodeNameservers StringSet
 421                         AssignNodeHostname     string
 422                 }
 423         }
 424 }
 425
 426 type CloudVMsConfig struct {
 427         Enable bool
 428
 429         BootProbeCommand     string
 430         DeployRunnerBinary   string
 431         ImageID              string
 432         MaxCloudOpsPerSecond int
 433         MaxProbesPerSecond   int
 434         PollInterval         Duration
 435         ProbeInterval        Duration
 436         SSHPort              string
 437         SyncInterval         Duration
 438         TimeoutBooting       Duration
 439         TimeoutIdle          Duration
 440         TimeoutProbe         Duration
 441         TimeoutShutdown      Duration
 442         TimeoutSignal        Duration
 443         TimeoutTERM          Duration
 444         ResourceTags         map[string]string
 445         TagKeyPrefix         string
 446
 447         Driver           string
 448         DriverParameters json.RawMessage
 449 }
 450
 451 type InstanceTypeMap map[string]InstanceType
 452
 453 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 454
 455 // UnmarshalJSON handles old config files that provide an array of
 456 // instance types instead of a hash.
 457 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 458         fixup := func(t InstanceType) (InstanceType, error) {
 459                 if t.ProviderType == "" {
 460                         t.ProviderType = t.Name
 461                 }
 462                 if t.Scratch == 0 {
 463                         t.Scratch = t.IncludedScratch + t.AddedScratch
 464                 } else if t.AddedScratch == 0 {
 465                         t.AddedScratch = t.Scratch - t.IncludedScratch
 466                 } else if t.IncludedScratch == 0 {
 467                         t.IncludedScratch = t.Scratch - t.AddedScratch
 468                 }
 469
 470                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 471                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 472                 }
 473                 return t, nil
 474         }
 475
 476         if len(data) > 0 && data[0] == '[' {
 477                 var arr []InstanceType
 478                 err := json.Unmarshal(data, &arr)
 479                 if err != nil {
 480                         return err
 481                 }
 482                 if len(arr) == 0 {
 483                         *it = nil
 484                         return nil
 485                 }
 486                 *it = make(map[string]InstanceType, len(arr))
 487                 for _, t := range arr {
 488                         if _, ok := (*it)[t.Name]; ok {
 489                                 return errDuplicateInstanceTypeName
 490                         }
 491                         t, err := fixup(t)
 492                         if err != nil {
 493                                 return err
 494                         }
 495                         (*it)[t.Name] = t
 496                 }
 497                 return nil
 498         }
 499         var hash map[string]InstanceType
 500         err := json.Unmarshal(data, &hash)
 501         if err != nil {
 502                 return err
 503         }
 504         // Fill in Name field (and ProviderType field, if not
 505         // specified) using hash key.
 506         *it = InstanceTypeMap(hash)
 507         for name, t := range *it {
 508                 t.Name = name
 509                 t, err := fixup(t)
 510                 if err != nil {
 511                         return err
 512                 }
 513                 (*it)[name] = t
 514         }
 515         return nil
 516 }
 517
 518 type StringSet map[string]struct{}
 519
 520 // UnmarshalJSON handles old config files that provide an array of
 521 // instance types instead of a hash.
 522 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 523         if len(data) > 0 && data[0] == '[' {
 524                 var arr []string
 525                 err := json.Unmarshal(data, &arr)
 526                 if err != nil {
 527                         return err
 528                 }
 529                 if len(arr) == 0 {
 530                         *ss = nil
 531                         return nil
 532                 }
 533                 *ss = make(map[string]struct{}, len(arr))
 534                 for _, t := range arr {
 535                         (*ss)[t] = struct{}{}
 536                 }
 537                 return nil
 538         }
 539         var hash map[string]struct{}
 540         err := json.Unmarshal(data, &hash)
 541         if err != nil {
 542                 return err
 543         }
 544         *ss = make(map[string]struct{}, len(hash))
 545         for t, _ := range hash {
 546                 (*ss)[t] = struct{}{}
 547         }
 548
 549         return nil
 550 }
 551
 552 type ServiceName string
 553
 554 const (
 555         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 556         ServiceNameController    ServiceName = "arvados-controller"
 557         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 558         ServiceNameHealth        ServiceName = "arvados-health"
 559         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 560         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 561         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 562         ServiceNameWebsocket     ServiceName = "arvados-ws"
 563         ServiceNameKeepbalance   ServiceName = "keep-balance"
 564         ServiceNameKeepweb       ServiceName = "keep-web"
 565         ServiceNameKeepproxy     ServiceName = "keepproxy"
 566         ServiceNameKeepstore     ServiceName = "keepstore"
 567 )
 568
 569 // Map returns all services as a map, suitable for iterating over all
 570 // services or looking up a service by name.
 571 func (svcs Services) Map() map[ServiceName]Service {
 572         return map[ServiceName]Service{
 573                 ServiceNameRailsAPI:      svcs.RailsAPI,
 574                 ServiceNameController:    svcs.Controller,
 575                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 576                 ServiceNameHealth:        svcs.Health,
 577                 ServiceNameNodemanager:   svcs.Nodemanager,
 578                 ServiceNameWorkbench1:    svcs.Workbench1,
 579                 ServiceNameWorkbench2:    svcs.Workbench2,
 580                 ServiceNameWebsocket:     svcs.Websocket,
 581                 ServiceNameKeepbalance:   svcs.Keepbalance,
 582                 ServiceNameKeepweb:       svcs.WebDAV,
 583                 ServiceNameKeepproxy:     svcs.Keepproxy,
 584                 ServiceNameKeepstore:     svcs.Keepstore,
 585         }
 586 }