sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.arvados.org/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters         map[string]Cluster
  27         AutoReloadConfig bool
  28 }
  29
  30 // GetConfig returns the current system config, loading it from
  31 // configFile if needed.
  32 func GetConfig(configFile string) (*Config, error) {
  33         var cfg Config
  34         err := config.LoadFile(&cfg, configFile)
  35         return &cfg, err
  36 }
  37
  38 // GetCluster returns the cluster ID and config for the given
  39 // cluster, or the default/only configured cluster if clusterID is "".
  40 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  41         if clusterID == "" {
  42                 if len(sc.Clusters) == 0 {
  43                         return nil, fmt.Errorf("no clusters configured")
  44                 } else if len(sc.Clusters) > 1 {
  45                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  46                 } else {
  47                         for id, cc := range sc.Clusters {
  48                                 cc.ClusterID = id
  49                                 return &cc, nil
  50                         }
  51                 }
  52         }
  53         if cc, ok := sc.Clusters[clusterID]; !ok {
  54                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  55         } else {
  56                 cc.ClusterID = clusterID
  57                 return &cc, nil
  58         }
  59 }
  60
  61 type WebDAVCacheConfig struct {
  62         TTL                  Duration
  63         UUIDTTL              Duration
  64         MaxBlockEntries      int
  65         MaxCollectionEntries int
  66         MaxCollectionBytes   int64
  67         MaxPermissionEntries int
  68         MaxUUIDEntries       int
  69 }
  70
  71 type Cluster struct {
  72         ClusterID       string `json:"-"`
  73         ManagementToken string
  74         SystemRootToken string
  75         Services        Services
  76         InstanceTypes   InstanceTypeMap
  77         Containers      ContainersConfig
  78         RemoteClusters  map[string]RemoteCluster
  79         PostgreSQL      PostgreSQL
  80
  81         API struct {
  82                 AsyncPermissionsUpdateInterval Duration
  83                 DisabledAPIs                   StringSet
  84                 MaxIndexDatabaseRead           int
  85                 MaxItemsPerResponse            int
  86                 MaxConcurrentRequests          int
  87                 MaxKeepBlobBuffers             int
  88                 MaxRequestAmplification        int
  89                 MaxRequestSize                 int
  90                 RailsSessionSecretToken        string
  91                 RequestTimeout                 Duration
  92                 SendTimeout                    Duration
  93                 WebsocketClientEventQueue      int
  94                 WebsocketServerEventQueue      int
  95                 KeepServiceRequestTimeout      Duration
  96         }
  97         AuditLogs struct {
  98                 MaxAge             Duration
  99                 MaxDeleteBatch     int
 100                 UnloggedAttributes StringSet
 101         }
 102         Collections struct {
 103                 BlobSigning              bool
 104                 BlobSigningKey           string
 105                 BlobSigningTTL           Duration
 106                 BlobTrash                bool
 107                 BlobTrashLifetime        Duration
 108                 BlobTrashCheckInterval   Duration
 109                 BlobTrashConcurrency     int
 110                 BlobDeleteConcurrency    int
 111                 BlobReplicateConcurrency int
 112                 CollectionVersioning     bool
 113                 DefaultTrashLifetime     Duration
 114                 DefaultReplication       int
 115                 ManagedProperties        map[string]struct {
 116                         Value     interface{}
 117                         Function  string
 118                         Protected bool
 119                 }
 120                 PreserveVersionIfIdle        Duration
 121                 TrashSweepInterval           Duration
 122                 TrustAllContent              bool
 123                 ForwardSlashNameSubstitution string
 124
 125                 BlobMissingReport        string
 126                 BalancePeriod            Duration
 127                 BalanceCollectionBatch   int
 128                 BalanceCollectionBuffers int
 129
 130                 WebDAVCache WebDAVCacheConfig
 131         }
 132         Git struct {
 133                 GitCommand   string
 134                 GitoliteHome string
 135                 Repositories string
 136         }
 137         Login struct {
 138                 LDAP struct {
 139                         Enable             bool
 140                         URL                URL
 141                         StartTLS           bool
 142                         InsecureTLS        bool
 143                         StripDomain        string
 144                         AppendDomain       string
 145                         SearchAttribute    string
 146                         SearchBindUser     string
 147                         SearchBindPassword string
 148                         SearchBase         string
 149                         SearchFilters      string
 150                         EmailAttribute     string
 151                         UsernameAttribute  string
 152                 }
 153                 GoogleClientID                string
 154                 GoogleClientSecret            string
 155                 GoogleAlternateEmailAddresses bool
 156                 PAM                           bool
 157                 PAMService                    string
 158                 PAMDefaultEmailDomain         string
 159                 ProviderAppID                 string
 160                 ProviderAppSecret             string
 161                 LoginCluster                  string
 162                 RemoteTokenRefresh            Duration
 163         }
 164         Mail struct {
 165                 MailchimpAPIKey                string
 166                 MailchimpListID                string
 167                 SendUserSetupNotificationEmail bool
 168                 IssueReporterEmailFrom         string
 169                 IssueReporterEmailTo           string
 170                 SupportEmailAddress            string
 171                 EmailFrom                      string
 172         }
 173         SystemLogs struct {
 174                 LogLevel                string
 175                 Format                  string
 176                 MaxRequestLogParamsSize int
 177         }
 178         TLS struct {
 179                 Certificate string
 180                 Key         string
 181                 Insecure    bool
 182         }
 183         Users struct {
 184                 AnonymousUserToken                    string
 185                 AdminNotifierEmailFrom                string
 186                 AutoAdminFirstUser                    bool
 187                 AutoAdminUserWithEmail                string
 188                 AutoSetupNewUsers                     bool
 189                 AutoSetupNewUsersWithRepository       bool
 190                 AutoSetupNewUsersWithVmUUID           string
 191                 AutoSetupUsernameBlacklist            StringSet
 192                 EmailSubjectPrefix                    string
 193                 NewInactiveUserNotificationRecipients StringSet
 194                 NewUserNotificationRecipients         StringSet
 195                 NewUsersAreActive                     bool
 196                 UserNotifierEmailFrom                 string
 197                 UserProfileNotificationAddress        string
 198                 PreferDomainForUsername               string
 199         }
 200         Volumes   map[string]Volume
 201         Workbench struct {
 202                 ActivationContactLink            string
 203                 APIClientConnectTimeout          Duration
 204                 APIClientReceiveTimeout          Duration
 205                 APIResponseCompression           bool
 206                 ApplicationMimetypesWithViewIcon StringSet
 207                 ArvadosDocsite                   string
 208                 ArvadosPublicDataDocURL          string
 209                 DefaultOpenIdPrefix              string
 210                 EnableGettingStartedPopup        bool
 211                 EnablePublicProjectsPage         bool
 212                 FileViewersConfigURL             string
 213                 LogViewerMaxBytes                ByteSize
 214                 MultiSiteSearch                  string
 215                 ProfilingEnabled                 bool
 216                 Repositories                     bool
 217                 RepositoryCache                  string
 218                 RunningJobLogRecordsToFetch      int
 219                 SecretKeyBase                    string
 220                 ShowRecentCollectionsOnDashboard bool
 221                 ShowUserAgreementInline          bool
 222                 ShowUserNotifications            bool
 223                 SiteName                         string
 224                 Theme                            string
 225                 UserProfileFormFields            map[string]struct {
 226                         Type                 string
 227                         FormFieldTitle       string
 228                         FormFieldDescription string
 229                         Required             bool
 230                         Position             int
 231                         Options              map[string]struct{}
 232                 }
 233                 UserProfileFormMessage string
 234                 VocabularyURL          string
 235                 WelcomePageHTML        string
 236                 InactivePageHTML       string
 237                 SSHHelpPageHTML        string
 238                 SSHHelpHostSuffix      string
 239         }
 240
 241         ForceLegacyAPI14 bool
 242 }
 243
 244 type Volume struct {
 245         AccessViaHosts   map[URL]VolumeAccess
 246         ReadOnly         bool
 247         Replication      int
 248         StorageClasses   map[string]bool
 249         Driver           string
 250         DriverParameters json.RawMessage
 251 }
 252
 253 type S3VolumeDriverParameters struct {
 254         AccessKey          string
 255         SecretKey          string
 256         Endpoint           string
 257         Region             string
 258         Bucket             string
 259         LocationConstraint bool
 260         IndexPageSize      int
 261         ConnectTimeout     Duration
 262         ReadTimeout        Duration
 263         RaceWindow         Duration
 264         UnsafeDelete       bool
 265 }
 266
 267 type AzureVolumeDriverParameters struct {
 268         StorageAccountName   string
 269         StorageAccountKey    string
 270         StorageBaseURL       string
 271         ContainerName        string
 272         RequestTimeout       Duration
 273         ListBlobsRetryDelay  Duration
 274         ListBlobsMaxAttempts int
 275 }
 276
 277 type DirectoryVolumeDriverParameters struct {
 278         Root      string
 279         Serialize bool
 280 }
 281
 282 type VolumeAccess struct {
 283         ReadOnly bool
 284 }
 285
 286 type Services struct {
 287         Composer       Service
 288         Controller     Service
 289         DispatchCloud  Service
 290         GitHTTP        Service
 291         GitSSH         Service
 292         Health         Service
 293         Keepbalance    Service
 294         Keepproxy      Service
 295         Keepstore      Service
 296         Nodemanager    Service
 297         RailsAPI       Service
 298         SSO            Service
 299         WebDAVDownload Service
 300         WebDAV         Service
 301         WebShell       Service
 302         Websocket      Service
 303         Workbench1     Service
 304         Workbench2     Service
 305 }
 306
 307 type Service struct {
 308         InternalURLs map[URL]ServiceInstance
 309         ExternalURL  URL
 310 }
 311
 312 // URL is a url.URL that is also usable as a JSON key/value.
 313 type URL url.URL
 314
 315 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 316 // used as a JSON key/value.
 317 func (su *URL) UnmarshalText(text []byte) error {
 318         u, err := url.Parse(string(text))
 319         if err == nil {
 320                 *su = URL(*u)
 321         }
 322         return err
 323 }
 324
 325 func (su URL) MarshalText() ([]byte, error) {
 326         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 327 }
 328
 329 func (su URL) String() string {
 330         return (*url.URL)(&su).String()
 331 }
 332
 333 type ServiceInstance struct {
 334         Rendezvous string `json:",omitempty"`
 335 }
 336
 337 type PostgreSQL struct {
 338         Connection     PostgreSQLConnection
 339         ConnectionPool int
 340 }
 341
 342 type PostgreSQLConnection map[string]string
 343
 344 type RemoteCluster struct {
 345         Host          string
 346         Proxy         bool
 347         Scheme        string
 348         Insecure      bool
 349         ActivateUsers bool
 350 }
 351
 352 type InstanceType struct {
 353         Name            string
 354         ProviderType    string
 355         VCPUs           int
 356         RAM             ByteSize
 357         Scratch         ByteSize
 358         IncludedScratch ByteSize
 359         AddedScratch    ByteSize
 360         Price           float64
 361         Preemptible     bool
 362 }
 363
 364 type ContainersConfig struct {
 365         CloudVMs                    CloudVMsConfig
 366         CrunchRunCommand            string
 367         CrunchRunArgumentsList      []string
 368         DefaultKeepCacheRAM         ByteSize
 369         DispatchPrivateKey          string
 370         LogReuseDecisions           bool
 371         MaxComputeVMs               int
 372         MaxDispatchAttempts         int
 373         MaxRetryAttempts            int
 374         MinRetryPeriod              Duration
 375         ReserveExtraRAM             ByteSize
 376         StaleLockTimeout            Duration
 377         SupportedDockerImageFormats StringSet
 378         UsePreemptibleInstances     bool
 379
 380         JobsAPI struct {
 381                 Enable         string
 382                 GitInternalDir string
 383         }
 384         Logging struct {
 385                 MaxAge                       Duration
 386                 LogBytesPerEvent             int
 387                 LogSecondsBetweenEvents      Duration
 388                 LogThrottlePeriod            Duration
 389                 LogThrottleBytes             int
 390                 LogThrottleLines             int
 391                 LimitLogBytesPerJob          int
 392                 LogPartialLineThrottlePeriod Duration
 393                 LogUpdatePeriod              Duration
 394                 LogUpdateSize                ByteSize
 395         }
 396         SLURM struct {
 397                 PrioritySpread             int64
 398                 SbatchArgumentsList        []string
 399                 SbatchEnvironmentVariables map[string]string
 400                 Managed                    struct {
 401                         DNSServerConfDir       string
 402                         DNSServerConfTemplate  string
 403                         DNSServerReloadCommand string
 404                         DNSServerUpdateCommand string
 405                         ComputeNodeDomain      string
 406                         ComputeNodeNameservers StringSet
 407                         AssignNodeHostname     string
 408                 }
 409         }
 410 }
 411
 412 type CloudVMsConfig struct {
 413         Enable bool
 414
 415         BootProbeCommand     string
 416         DeployRunnerBinary   string
 417         ImageID              string
 418         MaxCloudOpsPerSecond int
 419         MaxProbesPerSecond   int
 420         PollInterval         Duration
 421         ProbeInterval        Duration
 422         SSHPort              string
 423         SyncInterval         Duration
 424         TimeoutBooting       Duration
 425         TimeoutIdle          Duration
 426         TimeoutProbe         Duration
 427         TimeoutShutdown      Duration
 428         TimeoutSignal        Duration
 429         TimeoutTERM          Duration
 430         ResourceTags         map[string]string
 431         TagKeyPrefix         string
 432
 433         Driver           string
 434         DriverParameters json.RawMessage
 435 }
 436
 437 type InstanceTypeMap map[string]InstanceType
 438
 439 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 440
 441 // UnmarshalJSON handles old config files that provide an array of
 442 // instance types instead of a hash.
 443 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 444         fixup := func(t InstanceType) (InstanceType, error) {
 445                 if t.ProviderType == "" {
 446                         t.ProviderType = t.Name
 447                 }
 448                 if t.Scratch == 0 {
 449                         t.Scratch = t.IncludedScratch + t.AddedScratch
 450                 } else if t.AddedScratch == 0 {
 451                         t.AddedScratch = t.Scratch - t.IncludedScratch
 452                 } else if t.IncludedScratch == 0 {
 453                         t.IncludedScratch = t.Scratch - t.AddedScratch
 454                 }
 455
 456                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 457                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 458                 }
 459                 return t, nil
 460         }
 461
 462         if len(data) > 0 && data[0] == '[' {
 463                 var arr []InstanceType
 464                 err := json.Unmarshal(data, &arr)
 465                 if err != nil {
 466                         return err
 467                 }
 468                 if len(arr) == 0 {
 469                         *it = nil
 470                         return nil
 471                 }
 472                 *it = make(map[string]InstanceType, len(arr))
 473                 for _, t := range arr {
 474                         if _, ok := (*it)[t.Name]; ok {
 475                                 return errDuplicateInstanceTypeName
 476                         }
 477                         t, err := fixup(t)
 478                         if err != nil {
 479                                 return err
 480                         }
 481                         (*it)[t.Name] = t
 482                 }
 483                 return nil
 484         }
 485         var hash map[string]InstanceType
 486         err := json.Unmarshal(data, &hash)
 487         if err != nil {
 488                 return err
 489         }
 490         // Fill in Name field (and ProviderType field, if not
 491         // specified) using hash key.
 492         *it = InstanceTypeMap(hash)
 493         for name, t := range *it {
 494                 t.Name = name
 495                 t, err := fixup(t)
 496                 if err != nil {
 497                         return err
 498                 }
 499                 (*it)[name] = t
 500         }
 501         return nil
 502 }
 503
 504 type StringSet map[string]struct{}
 505
 506 // UnmarshalJSON handles old config files that provide an array of
 507 // instance types instead of a hash.
 508 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 509         if len(data) > 0 && data[0] == '[' {
 510                 var arr []string
 511                 err := json.Unmarshal(data, &arr)
 512                 if err != nil {
 513                         return err
 514                 }
 515                 if len(arr) == 0 {
 516                         *ss = nil
 517                         return nil
 518                 }
 519                 *ss = make(map[string]struct{}, len(arr))
 520                 for _, t := range arr {
 521                         (*ss)[t] = struct{}{}
 522                 }
 523                 return nil
 524         }
 525         var hash map[string]struct{}
 526         err := json.Unmarshal(data, &hash)
 527         if err != nil {
 528                 return err
 529         }
 530         *ss = make(map[string]struct{}, len(hash))
 531         for t, _ := range hash {
 532                 (*ss)[t] = struct{}{}
 533         }
 534
 535         return nil
 536 }
 537
 538 type ServiceName string
 539
 540 const (
 541         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 542         ServiceNameController    ServiceName = "arvados-controller"
 543         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 544         ServiceNameHealth        ServiceName = "arvados-health"
 545         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 546         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 547         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 548         ServiceNameWebsocket     ServiceName = "arvados-ws"
 549         ServiceNameKeepbalance   ServiceName = "keep-balance"
 550         ServiceNameKeepweb       ServiceName = "keep-web"
 551         ServiceNameKeepproxy     ServiceName = "keepproxy"
 552         ServiceNameKeepstore     ServiceName = "keepstore"
 553 )
 554
 555 // Map returns all services as a map, suitable for iterating over all
 556 // services or looking up a service by name.
 557 func (svcs Services) Map() map[ServiceName]Service {
 558         return map[ServiceName]Service{
 559                 ServiceNameRailsAPI:      svcs.RailsAPI,
 560                 ServiceNameController:    svcs.Controller,
 561                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 562                 ServiceNameHealth:        svcs.Health,
 563                 ServiceNameNodemanager:   svcs.Nodemanager,
 564                 ServiceNameWorkbench1:    svcs.Workbench1,
 565                 ServiceNameWorkbench2:    svcs.Workbench2,
 566                 ServiceNameWebsocket:     svcs.Websocket,
 567                 ServiceNameKeepbalance:   svcs.Keepbalance,
 568                 ServiceNameKeepweb:       svcs.WebDAV,
 569                 ServiceNameKeepproxy:     svcs.Keepproxy,
 570                 ServiceNameKeepstore:     svcs.Keepstore,
 571         }
 572 }