sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.arvados.org/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters         map[string]Cluster
  27         AutoReloadConfig bool
  28 }
  29
  30 // GetConfig returns the current system config, loading it from
  31 // configFile if needed.
  32 func GetConfig(configFile string) (*Config, error) {
  33         var cfg Config
  34         err := config.LoadFile(&cfg, configFile)
  35         return &cfg, err
  36 }
  37
  38 // GetCluster returns the cluster ID and config for the given
  39 // cluster, or the default/only configured cluster if clusterID is "".
  40 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  41         if clusterID == "" {
  42                 if len(sc.Clusters) == 0 {
  43                         return nil, fmt.Errorf("no clusters configured")
  44                 } else if len(sc.Clusters) > 1 {
  45                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  46                 } else {
  47                         for id, cc := range sc.Clusters {
  48                                 cc.ClusterID = id
  49                                 return &cc, nil
  50                         }
  51                 }
  52         }
  53         if cc, ok := sc.Clusters[clusterID]; !ok {
  54                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  55         } else {
  56                 cc.ClusterID = clusterID
  57                 return &cc, nil
  58         }
  59 }
  60
  61 type WebDAVCacheConfig struct {
  62         TTL                  Duration
  63         UUIDTTL              Duration
  64         MaxBlockEntries      int
  65         MaxCollectionEntries int
  66         MaxCollectionBytes   int64
  67         MaxPermissionEntries int
  68         MaxUUIDEntries       int
  69 }
  70
  71 type Cluster struct {
  72         ClusterID       string `json:"-"`
  73         ManagementToken string
  74         SystemRootToken string
  75         Services        Services
  76         InstanceTypes   InstanceTypeMap
  77         Containers      ContainersConfig
  78         RemoteClusters  map[string]RemoteCluster
  79         PostgreSQL      PostgreSQL
  80
  81         API struct {
  82                 AsyncPermissionsUpdateInterval Duration
  83                 DisabledAPIs                   StringSet
  84                 MaxIndexDatabaseRead           int
  85                 MaxItemsPerResponse            int
  86                 MaxConcurrentRequests          int
  87                 MaxKeepBlobBuffers             int
  88                 MaxRequestAmplification        int
  89                 MaxRequestSize                 int
  90                 RailsSessionSecretToken        string
  91                 RequestTimeout                 Duration
  92                 SendTimeout                    Duration
  93                 WebsocketClientEventQueue      int
  94                 WebsocketServerEventQueue      int
  95                 KeepServiceRequestTimeout      Duration
  96         }
  97         AuditLogs struct {
  98                 MaxAge             Duration
  99                 MaxDeleteBatch     int
 100                 UnloggedAttributes StringSet
 101         }
 102         Collections struct {
 103                 BlobSigning              bool
 104                 BlobSigningKey           string
 105                 BlobSigningTTL           Duration
 106                 BlobTrash                bool
 107                 BlobTrashLifetime        Duration
 108                 BlobTrashCheckInterval   Duration
 109                 BlobTrashConcurrency     int
 110                 BlobDeleteConcurrency    int
 111                 BlobReplicateConcurrency int
 112                 CollectionVersioning     bool
 113                 DefaultTrashLifetime     Duration
 114                 DefaultReplication       int
 115                 ManagedProperties        map[string]struct {
 116                         Value     interface{}
 117                         Function  string
 118                         Protected bool
 119                 }
 120                 PreserveVersionIfIdle        Duration
 121                 TrashSweepInterval           Duration
 122                 TrustAllContent              bool
 123                 ForwardSlashNameSubstitution string
 124
 125                 BlobMissingReport        string
 126                 BalancePeriod            Duration
 127                 BalanceCollectionBatch   int
 128                 BalanceCollectionBuffers int
 129
 130                 WebDAVCache WebDAVCacheConfig
 131         }
 132         Git struct {
 133                 GitCommand   string
 134                 GitoliteHome string
 135                 Repositories string
 136         }
 137         Login struct {
 138                 GoogleClientID                string
 139                 GoogleClientSecret            string
 140                 GoogleAlternateEmailAddresses bool
 141                 PAM                           bool
 142                 PAMService                    string
 143                 PAMDefaultEmailDomain         string
 144                 ProviderAppID                 string
 145                 ProviderAppSecret             string
 146                 LoginCluster                  string
 147                 RemoteTokenRefresh            Duration
 148         }
 149         Mail struct {
 150                 MailchimpAPIKey                string
 151                 MailchimpListID                string
 152                 SendUserSetupNotificationEmail bool
 153                 IssueReporterEmailFrom         string
 154                 IssueReporterEmailTo           string
 155                 SupportEmailAddress            string
 156                 EmailFrom                      string
 157         }
 158         SystemLogs struct {
 159                 LogLevel                string
 160                 Format                  string
 161                 MaxRequestLogParamsSize int
 162         }
 163         TLS struct {
 164                 Certificate string
 165                 Key         string
 166                 Insecure    bool
 167         }
 168         Users struct {
 169                 AnonymousUserToken                    string
 170                 AdminNotifierEmailFrom                string
 171                 AutoAdminFirstUser                    bool
 172                 AutoAdminUserWithEmail                string
 173                 AutoSetupNewUsers                     bool
 174                 AutoSetupNewUsersWithRepository       bool
 175                 AutoSetupNewUsersWithVmUUID           string
 176                 AutoSetupUsernameBlacklist            StringSet
 177                 EmailSubjectPrefix                    string
 178                 NewInactiveUserNotificationRecipients StringSet
 179                 NewUserNotificationRecipients         StringSet
 180                 NewUsersAreActive                     bool
 181                 UserNotifierEmailFrom                 string
 182                 UserProfileNotificationAddress        string
 183                 PreferDomainForUsername               string
 184         }
 185         Volumes   map[string]Volume
 186         Workbench struct {
 187                 ActivationContactLink            string
 188                 APIClientConnectTimeout          Duration
 189                 APIClientReceiveTimeout          Duration
 190                 APIResponseCompression           bool
 191                 ApplicationMimetypesWithViewIcon StringSet
 192                 ArvadosDocsite                   string
 193                 ArvadosPublicDataDocURL          string
 194                 DefaultOpenIdPrefix              string
 195                 EnableGettingStartedPopup        bool
 196                 EnablePublicProjectsPage         bool
 197                 FileViewersConfigURL             string
 198                 LogViewerMaxBytes                ByteSize
 199                 MultiSiteSearch                  string
 200                 ProfilingEnabled                 bool
 201                 Repositories                     bool
 202                 RepositoryCache                  string
 203                 RunningJobLogRecordsToFetch      int
 204                 SecretKeyBase                    string
 205                 ShowRecentCollectionsOnDashboard bool
 206                 ShowUserAgreementInline          bool
 207                 ShowUserNotifications            bool
 208                 SiteName                         string
 209                 Theme                            string
 210                 UserProfileFormFields            map[string]struct {
 211                         Type                 string
 212                         FormFieldTitle       string
 213                         FormFieldDescription string
 214                         Required             bool
 215                         Position             int
 216                         Options              map[string]struct{}
 217                 }
 218                 UserProfileFormMessage string
 219                 VocabularyURL          string
 220                 WelcomePageHTML        string
 221                 InactivePageHTML       string
 222                 SSHHelpPageHTML        string
 223                 SSHHelpHostSuffix      string
 224         }
 225
 226         ForceLegacyAPI14 bool
 227 }
 228
 229 type Volume struct {
 230         AccessViaHosts   map[URL]VolumeAccess
 231         ReadOnly         bool
 232         Replication      int
 233         StorageClasses   map[string]bool
 234         Driver           string
 235         DriverParameters json.RawMessage
 236 }
 237
 238 type S3VolumeDriverParameters struct {
 239         AccessKey          string
 240         SecretKey          string
 241         Endpoint           string
 242         Region             string
 243         Bucket             string
 244         LocationConstraint bool
 245         IndexPageSize      int
 246         ConnectTimeout     Duration
 247         ReadTimeout        Duration
 248         RaceWindow         Duration
 249         UnsafeDelete       bool
 250 }
 251
 252 type AzureVolumeDriverParameters struct {
 253         StorageAccountName   string
 254         StorageAccountKey    string
 255         StorageBaseURL       string
 256         ContainerName        string
 257         RequestTimeout       Duration
 258         ListBlobsRetryDelay  Duration
 259         ListBlobsMaxAttempts int
 260 }
 261
 262 type DirectoryVolumeDriverParameters struct {
 263         Root      string
 264         Serialize bool
 265 }
 266
 267 type VolumeAccess struct {
 268         ReadOnly bool
 269 }
 270
 271 type Services struct {
 272         Composer       Service
 273         Controller     Service
 274         DispatchCloud  Service
 275         GitHTTP        Service
 276         GitSSH         Service
 277         Health         Service
 278         Keepbalance    Service
 279         Keepproxy      Service
 280         Keepstore      Service
 281         Nodemanager    Service
 282         RailsAPI       Service
 283         SSO            Service
 284         WebDAVDownload Service
 285         WebDAV         Service
 286         WebShell       Service
 287         Websocket      Service
 288         Workbench1     Service
 289         Workbench2     Service
 290 }
 291
 292 type Service struct {
 293         InternalURLs map[URL]ServiceInstance
 294         ExternalURL  URL
 295 }
 296
 297 // URL is a url.URL that is also usable as a JSON key/value.
 298 type URL url.URL
 299
 300 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 301 // used as a JSON key/value.
 302 func (su *URL) UnmarshalText(text []byte) error {
 303         u, err := url.Parse(string(text))
 304         if err == nil {
 305                 *su = URL(*u)
 306                 if su.Path == "" && su.Host != "" {
 307                         // http://example really means http://example/
 308                         su.Path = "/"
 309                 }
 310         }
 311         return err
 312 }
 313
 314 func (su URL) MarshalText() ([]byte, error) {
 315         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 316 }
 317
 318 func (su URL) String() string {
 319         return (*url.URL)(&su).String()
 320 }
 321
 322 type ServiceInstance struct {
 323         Rendezvous string `json:",omitempty"`
 324 }
 325
 326 type PostgreSQL struct {
 327         Connection     PostgreSQLConnection
 328         ConnectionPool int
 329 }
 330
 331 type PostgreSQLConnection map[string]string
 332
 333 type RemoteCluster struct {
 334         Host          string
 335         Proxy         bool
 336         Scheme        string
 337         Insecure      bool
 338         ActivateUsers bool
 339 }
 340
 341 type InstanceType struct {
 342         Name            string
 343         ProviderType    string
 344         VCPUs           int
 345         RAM             ByteSize
 346         Scratch         ByteSize
 347         IncludedScratch ByteSize
 348         AddedScratch    ByteSize
 349         Price           float64
 350         Preemptible     bool
 351 }
 352
 353 type ContainersConfig struct {
 354         CloudVMs                    CloudVMsConfig
 355         CrunchRunCommand            string
 356         CrunchRunArgumentsList      []string
 357         DefaultKeepCacheRAM         ByteSize
 358         DispatchPrivateKey          string
 359         LogReuseDecisions           bool
 360         MaxComputeVMs               int
 361         MaxDispatchAttempts         int
 362         MaxRetryAttempts            int
 363         MinRetryPeriod              Duration
 364         ReserveExtraRAM             ByteSize
 365         StaleLockTimeout            Duration
 366         SupportedDockerImageFormats StringSet
 367         UsePreemptibleInstances     bool
 368
 369         JobsAPI struct {
 370                 Enable         string
 371                 GitInternalDir string
 372         }
 373         Logging struct {
 374                 MaxAge                       Duration
 375                 LogBytesPerEvent             int
 376                 LogSecondsBetweenEvents      Duration
 377                 LogThrottlePeriod            Duration
 378                 LogThrottleBytes             int
 379                 LogThrottleLines             int
 380                 LimitLogBytesPerJob          int
 381                 LogPartialLineThrottlePeriod Duration
 382                 LogUpdatePeriod              Duration
 383                 LogUpdateSize                ByteSize
 384         }
 385         SLURM struct {
 386                 PrioritySpread             int64
 387                 SbatchArgumentsList        []string
 388                 SbatchEnvironmentVariables map[string]string
 389                 Managed                    struct {
 390                         DNSServerConfDir       string
 391                         DNSServerConfTemplate  string
 392                         DNSServerReloadCommand string
 393                         DNSServerUpdateCommand string
 394                         ComputeNodeDomain      string
 395                         ComputeNodeNameservers StringSet
 396                         AssignNodeHostname     string
 397                 }
 398         }
 399 }
 400
 401 type CloudVMsConfig struct {
 402         Enable bool
 403
 404         BootProbeCommand     string
 405         DeployRunnerBinary   string
 406         ImageID              string
 407         MaxCloudOpsPerSecond int
 408         MaxProbesPerSecond   int
 409         PollInterval         Duration
 410         ProbeInterval        Duration
 411         SSHPort              string
 412         SyncInterval         Duration
 413         TimeoutBooting       Duration
 414         TimeoutIdle          Duration
 415         TimeoutProbe         Duration
 416         TimeoutShutdown      Duration
 417         TimeoutSignal        Duration
 418         TimeoutTERM          Duration
 419         ResourceTags         map[string]string
 420         TagKeyPrefix         string
 421
 422         Driver           string
 423         DriverParameters json.RawMessage
 424 }
 425
 426 type InstanceTypeMap map[string]InstanceType
 427
 428 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 429
 430 // UnmarshalJSON handles old config files that provide an array of
 431 // instance types instead of a hash.
 432 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 433         fixup := func(t InstanceType) (InstanceType, error) {
 434                 if t.ProviderType == "" {
 435                         t.ProviderType = t.Name
 436                 }
 437                 if t.Scratch == 0 {
 438                         t.Scratch = t.IncludedScratch + t.AddedScratch
 439                 } else if t.AddedScratch == 0 {
 440                         t.AddedScratch = t.Scratch - t.IncludedScratch
 441                 } else if t.IncludedScratch == 0 {
 442                         t.IncludedScratch = t.Scratch - t.AddedScratch
 443                 }
 444
 445                 if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 446                         return t, fmt.Errorf("InstanceType %q: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 447                 }
 448                 return t, nil
 449         }
 450
 451         if len(data) > 0 && data[0] == '[' {
 452                 var arr []InstanceType
 453                 err := json.Unmarshal(data, &arr)
 454                 if err != nil {
 455                         return err
 456                 }
 457                 if len(arr) == 0 {
 458                         *it = nil
 459                         return nil
 460                 }
 461                 *it = make(map[string]InstanceType, len(arr))
 462                 for _, t := range arr {
 463                         if _, ok := (*it)[t.Name]; ok {
 464                                 return errDuplicateInstanceTypeName
 465                         }
 466                         t, err := fixup(t)
 467                         if err != nil {
 468                                 return err
 469                         }
 470                         (*it)[t.Name] = t
 471                 }
 472                 return nil
 473         }
 474         var hash map[string]InstanceType
 475         err := json.Unmarshal(data, &hash)
 476         if err != nil {
 477                 return err
 478         }
 479         // Fill in Name field (and ProviderType field, if not
 480         // specified) using hash key.
 481         *it = InstanceTypeMap(hash)
 482         for name, t := range *it {
 483                 t.Name = name
 484                 t, err := fixup(t)
 485                 if err != nil {
 486                         return err
 487                 }
 488                 (*it)[name] = t
 489         }
 490         return nil
 491 }
 492
 493 type StringSet map[string]struct{}
 494
 495 // UnmarshalJSON handles old config files that provide an array of
 496 // instance types instead of a hash.
 497 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 498         if len(data) > 0 && data[0] == '[' {
 499                 var arr []string
 500                 err := json.Unmarshal(data, &arr)
 501                 if err != nil {
 502                         return err
 503                 }
 504                 if len(arr) == 0 {
 505                         *ss = nil
 506                         return nil
 507                 }
 508                 *ss = make(map[string]struct{}, len(arr))
 509                 for _, t := range arr {
 510                         (*ss)[t] = struct{}{}
 511                 }
 512                 return nil
 513         }
 514         var hash map[string]struct{}
 515         err := json.Unmarshal(data, &hash)
 516         if err != nil {
 517                 return err
 518         }
 519         *ss = make(map[string]struct{}, len(hash))
 520         for t, _ := range hash {
 521                 (*ss)[t] = struct{}{}
 522         }
 523
 524         return nil
 525 }
 526
 527 type ServiceName string
 528
 529 const (
 530         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 531         ServiceNameController    ServiceName = "arvados-controller"
 532         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 533         ServiceNameHealth        ServiceName = "arvados-health"
 534         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 535         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 536         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 537         ServiceNameWebsocket     ServiceName = "arvados-ws"
 538         ServiceNameKeepbalance   ServiceName = "keep-balance"
 539         ServiceNameKeepweb       ServiceName = "keep-web"
 540         ServiceNameKeepproxy     ServiceName = "keepproxy"
 541         ServiceNameKeepstore     ServiceName = "keepstore"
 542 )
 543
 544 // Map returns all services as a map, suitable for iterating over all
 545 // services or looking up a service by name.
 546 func (svcs Services) Map() map[ServiceName]Service {
 547         return map[ServiceName]Service{
 548                 ServiceNameRailsAPI:      svcs.RailsAPI,
 549                 ServiceNameController:    svcs.Controller,
 550                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 551                 ServiceNameHealth:        svcs.Health,
 552                 ServiceNameNodemanager:   svcs.Nodemanager,
 553                 ServiceNameWorkbench1:    svcs.Workbench1,
 554                 ServiceNameWorkbench2:    svcs.Workbench2,
 555                 ServiceNameWebsocket:     svcs.Websocket,
 556                 ServiceNameKeepbalance:   svcs.Keepbalance,
 557                 ServiceNameKeepweb:       svcs.WebDAV,
 558                 ServiceNameKeepproxy:     svcs.Keepproxy,
 559                 ServiceNameKeepstore:     svcs.Keepstore,
 560         }
 561 }