sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.curoverse.com/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters map[string]Cluster
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         if cc, ok := sc.Clusters[clusterID]; !ok {
  53                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  54         } else {
  55                 cc.ClusterID = clusterID
  56                 return &cc, nil
  57         }
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxBlockEntries      int
  64         MaxCollectionEntries int
  65         MaxCollectionBytes   int64
  66         MaxPermissionEntries int
  67         MaxUUIDEntries       int
  68 }
  69 type Cluster struct {
  70         ClusterID       string `json:"-"`
  71         ManagementToken string
  72         SystemRootToken string
  73         Services        Services
  74         InstanceTypes   InstanceTypeMap
  75         Containers      ContainersConfig
  76         RemoteClusters  map[string]RemoteCluster
  77         PostgreSQL      PostgreSQL
  78
  79         API struct {
  80                 AsyncPermissionsUpdateInterval Duration
  81                 DisabledAPIs                   StringSet
  82                 MaxIndexDatabaseRead           int
  83                 MaxItemsPerResponse            int
  84                 MaxConcurrentRequests          int
  85                 MaxKeepBlobBuffers             int
  86                 MaxRequestAmplification        int
  87                 MaxRequestSize                 int
  88                 RailsSessionSecretToken        string
  89                 RequestTimeout                 Duration
  90                 SendTimeout                    Duration
  91                 WebsocketClientEventQueue      int
  92                 WebsocketServerEventQueue      int
  93                 KeepServiceRequestTimeout      Duration
  94         }
  95         AuditLogs struct {
  96                 MaxAge             Duration
  97                 MaxDeleteBatch     int
  98                 UnloggedAttributes StringSet
  99         }
 100         Collections struct {
 101                 BlobSigning              bool
 102                 BlobSigningKey           string
 103                 BlobSigningTTL           Duration
 104                 BlobTrash                bool
 105                 BlobTrashLifetime        Duration
 106                 BlobTrashCheckInterval   Duration
 107                 BlobTrashConcurrency     int
 108                 BlobDeleteConcurrency    int
 109                 BlobReplicateConcurrency int
 110                 CollectionVersioning     bool
 111                 DefaultTrashLifetime     Duration
 112                 DefaultReplication       int
 113                 ManagedProperties        map[string]struct {
 114                         Value     interface{}
 115                         Function  string
 116                         Protected bool
 117                 }
 118                 PreserveVersionIfIdle Duration
 119                 TrashSweepInterval    Duration
 120                 TrustAllContent       bool
 121
 122                 WebDAVCache WebDAVCacheConfig
 123         }
 124         Git struct {
 125                 GitCommand   string
 126                 GitoliteHome string
 127                 Repositories string
 128         }
 129         Login struct {
 130                 ProviderAppSecret  string
 131                 ProviderAppID      string
 132                 LoginCluster       string
 133                 RemoteTokenRefresh Duration
 134         }
 135         Mail struct {
 136                 MailchimpAPIKey                string
 137                 MailchimpListID                string
 138                 SendUserSetupNotificationEmail bool
 139                 IssueReporterEmailFrom         string
 140                 IssueReporterEmailTo           string
 141                 SupportEmailAddress            string
 142                 EmailFrom                      string
 143         }
 144         SystemLogs struct {
 145                 LogLevel                string
 146                 Format                  string
 147                 MaxRequestLogParamsSize int
 148         }
 149         TLS struct {
 150                 Certificate string
 151                 Key         string
 152                 Insecure    bool
 153         }
 154         Users struct {
 155                 AnonymousUserToken                    string
 156                 AdminNotifierEmailFrom                string
 157                 AutoAdminFirstUser                    bool
 158                 AutoAdminUserWithEmail                string
 159                 AutoSetupNewUsers                     bool
 160                 AutoSetupNewUsersWithRepository       bool
 161                 AutoSetupNewUsersWithVmUUID           string
 162                 AutoSetupUsernameBlacklist            StringSet
 163                 EmailSubjectPrefix                    string
 164                 NewInactiveUserNotificationRecipients StringSet
 165                 NewUserNotificationRecipients         StringSet
 166                 NewUsersAreActive                     bool
 167                 UserNotifierEmailFrom                 string
 168                 UserProfileNotificationAddress        string
 169         }
 170         Volumes   map[string]Volume
 171         Workbench struct {
 172                 ActivationContactLink            string
 173                 APIClientConnectTimeout          Duration
 174                 APIClientReceiveTimeout          Duration
 175                 APIResponseCompression           bool
 176                 ApplicationMimetypesWithViewIcon StringSet
 177                 ArvadosDocsite                   string
 178                 ArvadosPublicDataDocURL          string
 179                 DefaultOpenIdPrefix              string
 180                 EnableGettingStartedPopup        bool
 181                 EnablePublicProjectsPage         bool
 182                 FileViewersConfigURL             string
 183                 LogViewerMaxBytes                ByteSize
 184                 MultiSiteSearch                  string
 185                 ProfilingEnabled                 bool
 186                 Repositories                     bool
 187                 RepositoryCache                  string
 188                 RunningJobLogRecordsToFetch      int
 189                 SecretKeyBase                    string
 190                 ShowRecentCollectionsOnDashboard bool
 191                 ShowUserAgreementInline          bool
 192                 ShowUserNotifications            bool
 193                 SiteName                         string
 194                 Theme                            string
 195                 UserProfileFormFields            map[string]struct {
 196                         Type                 string
 197                         FormFieldTitle       string
 198                         FormFieldDescription string
 199                         Required             bool
 200                         Position             int
 201                         Options              map[string]struct{}
 202                 }
 203                 UserProfileFormMessage string
 204                 VocabularyURL          string
 205         }
 206
 207         EnableBetaController14287 bool
 208 }
 209
 210 type Volume struct {
 211         AccessViaHosts   map[URL]VolumeAccess
 212         ReadOnly         bool
 213         Replication      int
 214         StorageClasses   map[string]bool
 215         Driver           string
 216         DriverParameters json.RawMessage
 217 }
 218
 219 type S3VolumeDriverParameters struct {
 220         AccessKey          string
 221         SecretKey          string
 222         Endpoint           string
 223         Region             string
 224         Bucket             string
 225         LocationConstraint bool
 226         IndexPageSize      int
 227         ConnectTimeout     Duration
 228         ReadTimeout        Duration
 229         RaceWindow         Duration
 230         UnsafeDelete       bool
 231 }
 232
 233 type AzureVolumeDriverParameters struct {
 234         StorageAccountName   string
 235         StorageAccountKey    string
 236         StorageBaseURL       string
 237         ContainerName        string
 238         RequestTimeout       Duration
 239         ListBlobsRetryDelay  Duration
 240         ListBlobsMaxAttempts int
 241 }
 242
 243 type DirectoryVolumeDriverParameters struct {
 244         Root      string
 245         Serialize bool
 246 }
 247
 248 type VolumeAccess struct {
 249         ReadOnly bool
 250 }
 251
 252 type Services struct {
 253         Composer       Service
 254         Controller     Service
 255         DispatchCloud  Service
 256         GitHTTP        Service
 257         GitSSH         Service
 258         Health         Service
 259         Keepbalance    Service
 260         Keepproxy      Service
 261         Keepstore      Service
 262         Nodemanager    Service
 263         RailsAPI       Service
 264         SSO            Service
 265         WebDAVDownload Service
 266         WebDAV         Service
 267         WebShell       Service
 268         Websocket      Service
 269         Workbench1     Service
 270         Workbench2     Service
 271 }
 272
 273 type Service struct {
 274         InternalURLs map[URL]ServiceInstance
 275         ExternalURL  URL
 276 }
 277
 278 // URL is a url.URL that is also usable as a JSON key/value.
 279 type URL url.URL
 280
 281 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 282 // used as a JSON key/value.
 283 func (su *URL) UnmarshalText(text []byte) error {
 284         u, err := url.Parse(string(text))
 285         if err == nil {
 286                 *su = URL(*u)
 287         }
 288         return err
 289 }
 290
 291 func (su URL) MarshalText() ([]byte, error) {
 292         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 293 }
 294
 295 func (su URL) String() string {
 296         return (*url.URL)(&su).String()
 297 }
 298
 299 type ServiceInstance struct {
 300         // Rendezvous is normally empty; when changing the URL of a
 301         // Keepstore service, Rendezvous can be set to the old URL to
 302         // preserve rendezvous ordering.
 303         Rendezvous string `json:",omitempty"`
 304 }
 305
 306 type PostgreSQL struct {
 307         Connection     PostgreSQLConnection
 308         ConnectionPool int
 309 }
 310
 311 type PostgreSQLConnection map[string]string
 312
 313 type RemoteCluster struct {
 314         Host          string
 315         Proxy         bool
 316         Scheme        string
 317         Insecure      bool
 318         ActivateUsers bool
 319 }
 320
 321 type InstanceType struct {
 322         Name            string
 323         ProviderType    string
 324         VCPUs           int
 325         RAM             ByteSize
 326         Scratch         ByteSize
 327         IncludedScratch ByteSize
 328         AddedScratch    ByteSize
 329         Price           float64
 330         Preemptible     bool
 331 }
 332
 333 type ContainersConfig struct {
 334         CloudVMs                    CloudVMsConfig
 335         CrunchRunCommand            string
 336         CrunchRunArgumentsList      []string
 337         DefaultKeepCacheRAM         ByteSize
 338         DispatchPrivateKey          string
 339         LogReuseDecisions           bool
 340         MaxComputeVMs               int
 341         MaxDispatchAttempts         int
 342         MaxRetryAttempts            int
 343         MinRetryPeriod              Duration
 344         ReserveExtraRAM             ByteSize
 345         StaleLockTimeout            Duration
 346         SupportedDockerImageFormats StringSet
 347         UsePreemptibleInstances     bool
 348
 349         JobsAPI struct {
 350                 Enable         string
 351                 GitInternalDir string
 352         }
 353         Logging struct {
 354                 MaxAge                       Duration
 355                 LogBytesPerEvent             int
 356                 LogSecondsBetweenEvents      int
 357                 LogThrottlePeriod            Duration
 358                 LogThrottleBytes             int
 359                 LogThrottleLines             int
 360                 LimitLogBytesPerJob          int
 361                 LogPartialLineThrottlePeriod Duration
 362                 LogUpdatePeriod              Duration
 363                 LogUpdateSize                ByteSize
 364         }
 365         SLURM struct {
 366                 PrioritySpread             int64
 367                 SbatchArgumentsList        []string
 368                 SbatchEnvironmentVariables map[string]string
 369                 Managed                    struct {
 370                         DNSServerConfDir       string
 371                         DNSServerConfTemplate  string
 372                         DNSServerReloadCommand string
 373                         DNSServerUpdateCommand string
 374                         ComputeNodeDomain      string
 375                         ComputeNodeNameservers StringSet
 376                         AssignNodeHostname     string
 377                 }
 378         }
 379 }
 380
 381 type CloudVMsConfig struct {
 382         Enable bool
 383
 384         BootProbeCommand     string
 385         ImageID              string
 386         MaxCloudOpsPerSecond int
 387         MaxProbesPerSecond   int
 388         PollInterval         Duration
 389         ProbeInterval        Duration
 390         SSHPort              string
 391         SyncInterval         Duration
 392         TimeoutBooting       Duration
 393         TimeoutIdle          Duration
 394         TimeoutProbe         Duration
 395         TimeoutShutdown      Duration
 396         TimeoutSignal        Duration
 397         TimeoutTERM          Duration
 398         ResourceTags         map[string]string
 399         TagKeyPrefix         string
 400
 401         Driver           string
 402         DriverParameters json.RawMessage
 403 }
 404
 405 type InstanceTypeMap map[string]InstanceType
 406
 407 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 408
 409 // UnmarshalJSON handles old config files that provide an array of
 410 // instance types instead of a hash.
 411 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 412         if len(data) > 0 && data[0] == '[' {
 413                 var arr []InstanceType
 414                 err := json.Unmarshal(data, &arr)
 415                 if err != nil {
 416                         return err
 417                 }
 418                 if len(arr) == 0 {
 419                         *it = nil
 420                         return nil
 421                 }
 422                 *it = make(map[string]InstanceType, len(arr))
 423                 for _, t := range arr {
 424                         if _, ok := (*it)[t.Name]; ok {
 425                                 return errDuplicateInstanceTypeName
 426                         }
 427                         if t.ProviderType == "" {
 428                                 t.ProviderType = t.Name
 429                         }
 430                         if t.Scratch == 0 {
 431                                 t.Scratch = t.IncludedScratch + t.AddedScratch
 432                         } else if t.AddedScratch == 0 {
 433                                 t.AddedScratch = t.Scratch - t.IncludedScratch
 434                         } else if t.IncludedScratch == 0 {
 435                                 t.IncludedScratch = t.Scratch - t.AddedScratch
 436                         }
 437
 438                         if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 439                                 return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 440                         }
 441                         (*it)[t.Name] = t
 442                 }
 443                 return nil
 444         }
 445         var hash map[string]InstanceType
 446         err := json.Unmarshal(data, &hash)
 447         if err != nil {
 448                 return err
 449         }
 450         // Fill in Name field (and ProviderType field, if not
 451         // specified) using hash key.
 452         *it = InstanceTypeMap(hash)
 453         for name, t := range *it {
 454                 t.Name = name
 455                 if t.ProviderType == "" {
 456                         t.ProviderType = name
 457                 }
 458                 (*it)[name] = t
 459         }
 460         return nil
 461 }
 462
 463 type StringSet map[string]struct{}
 464
 465 // UnmarshalJSON handles old config files that provide an array of
 466 // instance types instead of a hash.
 467 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 468         if len(data) > 0 && data[0] == '[' {
 469                 var arr []string
 470                 err := json.Unmarshal(data, &arr)
 471                 if err != nil {
 472                         return err
 473                 }
 474                 if len(arr) == 0 {
 475                         *ss = nil
 476                         return nil
 477                 }
 478                 *ss = make(map[string]struct{}, len(arr))
 479                 for _, t := range arr {
 480                         (*ss)[t] = struct{}{}
 481                 }
 482                 return nil
 483         }
 484         var hash map[string]struct{}
 485         err := json.Unmarshal(data, &hash)
 486         if err != nil {
 487                 return err
 488         }
 489         *ss = make(map[string]struct{}, len(hash))
 490         for t, _ := range hash {
 491                 (*ss)[t] = struct{}{}
 492         }
 493
 494         return nil
 495 }
 496
 497 type ServiceName string
 498
 499 const (
 500         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 501         ServiceNameController    ServiceName = "arvados-controller"
 502         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 503         ServiceNameHealth        ServiceName = "arvados-health"
 504         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 505         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 506         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 507         ServiceNameWebsocket     ServiceName = "arvados-ws"
 508         ServiceNameKeepbalance   ServiceName = "keep-balance"
 509         ServiceNameKeepweb       ServiceName = "keep-web"
 510         ServiceNameKeepproxy     ServiceName = "keepproxy"
 511         ServiceNameKeepstore     ServiceName = "keepstore"
 512 )
 513
 514 // Map returns all services as a map, suitable for iterating over all
 515 // services or looking up a service by name.
 516 func (svcs Services) Map() map[ServiceName]Service {
 517         return map[ServiceName]Service{
 518                 ServiceNameRailsAPI:      svcs.RailsAPI,
 519                 ServiceNameController:    svcs.Controller,
 520                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 521                 ServiceNameHealth:        svcs.Health,
 522                 ServiceNameNodemanager:   svcs.Nodemanager,
 523                 ServiceNameWorkbench1:    svcs.Workbench1,
 524                 ServiceNameWorkbench2:    svcs.Workbench2,
 525                 ServiceNameWebsocket:     svcs.Websocket,
 526                 ServiceNameKeepbalance:   svcs.Keepbalance,
 527                 ServiceNameKeepweb:       svcs.WebDAV,
 528                 ServiceNameKeepproxy:     svcs.Keepproxy,
 529                 ServiceNameKeepstore:     svcs.Keepstore,
 530         }
 531 }