sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.curoverse.com/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters map[string]Cluster
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         if cc, ok := sc.Clusters[clusterID]; !ok {
  53                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  54         } else {
  55                 cc.ClusterID = clusterID
  56                 return &cc, nil
  57         }
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxBlockEntries      int
  64         MaxCollectionEntries int
  65         MaxCollectionBytes   int64
  66         MaxPermissionEntries int
  67         MaxUUIDEntries       int
  68 }
  69 type Cluster struct {
  70         ClusterID       string `json:"-"`
  71         ManagementToken string
  72         SystemRootToken string
  73         Services        Services
  74         InstanceTypes   InstanceTypeMap
  75         Containers      ContainersConfig
  76         RemoteClusters  map[string]RemoteCluster
  77         PostgreSQL      PostgreSQL
  78
  79         API struct {
  80                 AsyncPermissionsUpdateInterval Duration
  81                 DisabledAPIs                   StringSet
  82                 MaxIndexDatabaseRead           int
  83                 MaxItemsPerResponse            int
  84                 MaxConcurrentRequests          int
  85                 MaxKeepBlobBuffers             int
  86                 MaxRequestAmplification        int
  87                 MaxRequestSize                 int
  88                 RailsSessionSecretToken        string
  89                 RequestTimeout                 Duration
  90                 SendTimeout                    Duration
  91                 WebsocketClientEventQueue      int
  92                 WebsocketServerEventQueue      int
  93                 KeepServiceRequestTimeout      Duration
  94         }
  95         AuditLogs struct {
  96                 MaxAge             Duration
  97                 MaxDeleteBatch     int
  98                 UnloggedAttributes StringSet
  99         }
 100         Collections struct {
 101                 BlobSigning              bool
 102                 BlobSigningKey           string
 103                 BlobSigningTTL           Duration
 104                 BlobTrash                bool
 105                 BlobTrashLifetime        Duration
 106                 BlobTrashCheckInterval   Duration
 107                 BlobTrashConcurrency     int
 108                 BlobDeleteConcurrency    int
 109                 BlobReplicateConcurrency int
 110                 CollectionVersioning     bool
 111                 DefaultTrashLifetime     Duration
 112                 DefaultReplication       int
 113                 ManagedProperties        map[string]struct {
 114                         Value     interface{}
 115                         Function  string
 116                         Protected bool
 117                 }
 118                 PreserveVersionIfIdle Duration
 119                 TrashSweepInterval    Duration
 120                 TrustAllContent       bool
 121
 122                 BlobMissingReport        string
 123                 BalancePeriod            Duration
 124                 BalanceCollectionBatch   int
 125                 BalanceCollectionBuffers int
 126
 127                 WebDAVCache WebDAVCacheConfig
 128         }
 129         Git struct {
 130                 GitCommand   string
 131                 GitoliteHome string
 132                 Repositories string
 133         }
 134         Login struct {
 135                 ProviderAppSecret  string
 136                 ProviderAppID      string
 137                 LoginCluster       string
 138                 RemoteTokenRefresh Duration
 139         }
 140         Mail struct {
 141                 MailchimpAPIKey                string
 142                 MailchimpListID                string
 143                 SendUserSetupNotificationEmail bool
 144                 IssueReporterEmailFrom         string
 145                 IssueReporterEmailTo           string
 146                 SupportEmailAddress            string
 147                 EmailFrom                      string
 148         }
 149         SystemLogs struct {
 150                 LogLevel                string
 151                 Format                  string
 152                 MaxRequestLogParamsSize int
 153         }
 154         TLS struct {
 155                 Certificate string
 156                 Key         string
 157                 Insecure    bool
 158         }
 159         Users struct {
 160                 AnonymousUserToken                    string
 161                 AdminNotifierEmailFrom                string
 162                 AutoAdminFirstUser                    bool
 163                 AutoAdminUserWithEmail                string
 164                 AutoSetupNewUsers                     bool
 165                 AutoSetupNewUsersWithRepository       bool
 166                 AutoSetupNewUsersWithVmUUID           string
 167                 AutoSetupUsernameBlacklist            StringSet
 168                 EmailSubjectPrefix                    string
 169                 NewInactiveUserNotificationRecipients StringSet
 170                 NewUserNotificationRecipients         StringSet
 171                 NewUsersAreActive                     bool
 172                 UserNotifierEmailFrom                 string
 173                 UserProfileNotificationAddress        string
 174         }
 175         Volumes   map[string]Volume
 176         Workbench struct {
 177                 ActivationContactLink            string
 178                 APIClientConnectTimeout          Duration
 179                 APIClientReceiveTimeout          Duration
 180                 APIResponseCompression           bool
 181                 ApplicationMimetypesWithViewIcon StringSet
 182                 ArvadosDocsite                   string
 183                 ArvadosPublicDataDocURL          string
 184                 DefaultOpenIdPrefix              string
 185                 EnableGettingStartedPopup        bool
 186                 EnablePublicProjectsPage         bool
 187                 FileViewersConfigURL             string
 188                 LogViewerMaxBytes                ByteSize
 189                 MultiSiteSearch                  string
 190                 ProfilingEnabled                 bool
 191                 Repositories                     bool
 192                 RepositoryCache                  string
 193                 RunningJobLogRecordsToFetch      int
 194                 SecretKeyBase                    string
 195                 ShowRecentCollectionsOnDashboard bool
 196                 ShowUserAgreementInline          bool
 197                 ShowUserNotifications            bool
 198                 SiteName                         string
 199                 Theme                            string
 200                 UserProfileFormFields            map[string]struct {
 201                         Type                 string
 202                         FormFieldTitle       string
 203                         FormFieldDescription string
 204                         Required             bool
 205                         Position             int
 206                         Options              map[string]struct{}
 207                 }
 208                 UserProfileFormMessage string
 209                 VocabularyURL          string
 210                 WelcomePageHTML        string
 211                 InactivePageHTML       string
 212         }
 213
 214         EnableBetaController14287 bool
 215 }
 216
 217 type Volume struct {
 218         AccessViaHosts   map[URL]VolumeAccess
 219         ReadOnly         bool
 220         Replication      int
 221         StorageClasses   map[string]bool
 222         Driver           string
 223         DriverParameters json.RawMessage
 224 }
 225
 226 type S3VolumeDriverParameters struct {
 227         AccessKey          string
 228         SecretKey          string
 229         Endpoint           string
 230         Region             string
 231         Bucket             string
 232         LocationConstraint bool
 233         IndexPageSize      int
 234         ConnectTimeout     Duration
 235         ReadTimeout        Duration
 236         RaceWindow         Duration
 237         UnsafeDelete       bool
 238 }
 239
 240 type AzureVolumeDriverParameters struct {
 241         StorageAccountName   string
 242         StorageAccountKey    string
 243         StorageBaseURL       string
 244         ContainerName        string
 245         RequestTimeout       Duration
 246         ListBlobsRetryDelay  Duration
 247         ListBlobsMaxAttempts int
 248 }
 249
 250 type DirectoryVolumeDriverParameters struct {
 251         Root      string
 252         Serialize bool
 253 }
 254
 255 type VolumeAccess struct {
 256         ReadOnly bool
 257 }
 258
 259 type Services struct {
 260         Composer       Service
 261         Controller     Service
 262         DispatchCloud  Service
 263         GitHTTP        Service
 264         GitSSH         Service
 265         Health         Service
 266         Keepbalance    Service
 267         Keepproxy      Service
 268         Keepstore      Service
 269         Nodemanager    Service
 270         RailsAPI       Service
 271         SSO            Service
 272         WebDAVDownload Service
 273         WebDAV         Service
 274         WebShell       Service
 275         Websocket      Service
 276         Workbench1     Service
 277         Workbench2     Service
 278 }
 279
 280 type Service struct {
 281         InternalURLs map[URL]ServiceInstance
 282         ExternalURL  URL
 283 }
 284
 285 // URL is a url.URL that is also usable as a JSON key/value.
 286 type URL url.URL
 287
 288 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 289 // used as a JSON key/value.
 290 func (su *URL) UnmarshalText(text []byte) error {
 291         u, err := url.Parse(string(text))
 292         if err == nil {
 293                 *su = URL(*u)
 294         }
 295         return err
 296 }
 297
 298 func (su URL) MarshalText() ([]byte, error) {
 299         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 300 }
 301
 302 func (su URL) String() string {
 303         return (*url.URL)(&su).String()
 304 }
 305
 306 type ServiceInstance struct {
 307         Rendezvous string `json:",omitempty"`
 308 }
 309
 310 type PostgreSQL struct {
 311         Connection     PostgreSQLConnection
 312         ConnectionPool int
 313 }
 314
 315 type PostgreSQLConnection map[string]string
 316
 317 type RemoteCluster struct {
 318         Host          string
 319         Proxy         bool
 320         Scheme        string
 321         Insecure      bool
 322         ActivateUsers bool
 323 }
 324
 325 type InstanceType struct {
 326         Name            string
 327         ProviderType    string
 328         VCPUs           int
 329         RAM             ByteSize
 330         Scratch         ByteSize
 331         IncludedScratch ByteSize
 332         AddedScratch    ByteSize
 333         Price           float64
 334         Preemptible     bool
 335 }
 336
 337 type ContainersConfig struct {
 338         CloudVMs                    CloudVMsConfig
 339         CrunchRunCommand            string
 340         CrunchRunArgumentsList      []string
 341         DefaultKeepCacheRAM         ByteSize
 342         DispatchPrivateKey          string
 343         LogReuseDecisions           bool
 344         MaxComputeVMs               int
 345         MaxDispatchAttempts         int
 346         MaxRetryAttempts            int
 347         MinRetryPeriod              Duration
 348         ReserveExtraRAM             ByteSize
 349         StaleLockTimeout            Duration
 350         SupportedDockerImageFormats StringSet
 351         UsePreemptibleInstances     bool
 352
 353         JobsAPI struct {
 354                 Enable         string
 355                 GitInternalDir string
 356         }
 357         Logging struct {
 358                 MaxAge                       Duration
 359                 LogBytesPerEvent             int
 360                 LogSecondsBetweenEvents      int
 361                 LogThrottlePeriod            Duration
 362                 LogThrottleBytes             int
 363                 LogThrottleLines             int
 364                 LimitLogBytesPerJob          int
 365                 LogPartialLineThrottlePeriod Duration
 366                 LogUpdatePeriod              Duration
 367                 LogUpdateSize                ByteSize
 368         }
 369         SLURM struct {
 370                 PrioritySpread             int64
 371                 SbatchArgumentsList        []string
 372                 SbatchEnvironmentVariables map[string]string
 373                 Managed                    struct {
 374                         DNSServerConfDir       string
 375                         DNSServerConfTemplate  string
 376                         DNSServerReloadCommand string
 377                         DNSServerUpdateCommand string
 378                         ComputeNodeDomain      string
 379                         ComputeNodeNameservers StringSet
 380                         AssignNodeHostname     string
 381                 }
 382         }
 383 }
 384
 385 type CloudVMsConfig struct {
 386         Enable bool
 387
 388         BootProbeCommand     string
 389         ImageID              string
 390         MaxCloudOpsPerSecond int
 391         MaxProbesPerSecond   int
 392         PollInterval         Duration
 393         ProbeInterval        Duration
 394         SSHPort              string
 395         SyncInterval         Duration
 396         TimeoutBooting       Duration
 397         TimeoutIdle          Duration
 398         TimeoutProbe         Duration
 399         TimeoutShutdown      Duration
 400         TimeoutSignal        Duration
 401         TimeoutTERM          Duration
 402         ResourceTags         map[string]string
 403         TagKeyPrefix         string
 404
 405         Driver           string
 406         DriverParameters json.RawMessage
 407 }
 408
 409 type InstanceTypeMap map[string]InstanceType
 410
 411 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 412
 413 // UnmarshalJSON handles old config files that provide an array of
 414 // instance types instead of a hash.
 415 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 416         if len(data) > 0 && data[0] == '[' {
 417                 var arr []InstanceType
 418                 err := json.Unmarshal(data, &arr)
 419                 if err != nil {
 420                         return err
 421                 }
 422                 if len(arr) == 0 {
 423                         *it = nil
 424                         return nil
 425                 }
 426                 *it = make(map[string]InstanceType, len(arr))
 427                 for _, t := range arr {
 428                         if _, ok := (*it)[t.Name]; ok {
 429                                 return errDuplicateInstanceTypeName
 430                         }
 431                         if t.ProviderType == "" {
 432                                 t.ProviderType = t.Name
 433                         }
 434                         if t.Scratch == 0 {
 435                                 t.Scratch = t.IncludedScratch + t.AddedScratch
 436                         } else if t.AddedScratch == 0 {
 437                                 t.AddedScratch = t.Scratch - t.IncludedScratch
 438                         } else if t.IncludedScratch == 0 {
 439                                 t.IncludedScratch = t.Scratch - t.AddedScratch
 440                         }
 441
 442                         if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 443                                 return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 444                         }
 445                         (*it)[t.Name] = t
 446                 }
 447                 return nil
 448         }
 449         var hash map[string]InstanceType
 450         err := json.Unmarshal(data, &hash)
 451         if err != nil {
 452                 return err
 453         }
 454         // Fill in Name field (and ProviderType field, if not
 455         // specified) using hash key.
 456         *it = InstanceTypeMap(hash)
 457         for name, t := range *it {
 458                 t.Name = name
 459                 if t.ProviderType == "" {
 460                         t.ProviderType = name
 461                 }
 462                 (*it)[name] = t
 463         }
 464         return nil
 465 }
 466
 467 type StringSet map[string]struct{}
 468
 469 // UnmarshalJSON handles old config files that provide an array of
 470 // instance types instead of a hash.
 471 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 472         if len(data) > 0 && data[0] == '[' {
 473                 var arr []string
 474                 err := json.Unmarshal(data, &arr)
 475                 if err != nil {
 476                         return err
 477                 }
 478                 if len(arr) == 0 {
 479                         *ss = nil
 480                         return nil
 481                 }
 482                 *ss = make(map[string]struct{}, len(arr))
 483                 for _, t := range arr {
 484                         (*ss)[t] = struct{}{}
 485                 }
 486                 return nil
 487         }
 488         var hash map[string]struct{}
 489         err := json.Unmarshal(data, &hash)
 490         if err != nil {
 491                 return err
 492         }
 493         *ss = make(map[string]struct{}, len(hash))
 494         for t, _ := range hash {
 495                 (*ss)[t] = struct{}{}
 496         }
 497
 498         return nil
 499 }
 500
 501 type ServiceName string
 502
 503 const (
 504         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 505         ServiceNameController    ServiceName = "arvados-controller"
 506         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 507         ServiceNameHealth        ServiceName = "arvados-health"
 508         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 509         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 510         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 511         ServiceNameWebsocket     ServiceName = "arvados-ws"
 512         ServiceNameKeepbalance   ServiceName = "keep-balance"
 513         ServiceNameKeepweb       ServiceName = "keep-web"
 514         ServiceNameKeepproxy     ServiceName = "keepproxy"
 515         ServiceNameKeepstore     ServiceName = "keepstore"
 516 )
 517
 518 // Map returns all services as a map, suitable for iterating over all
 519 // services or looking up a service by name.
 520 func (svcs Services) Map() map[ServiceName]Service {
 521         return map[ServiceName]Service{
 522                 ServiceNameRailsAPI:      svcs.RailsAPI,
 523                 ServiceNameController:    svcs.Controller,
 524                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 525                 ServiceNameHealth:        svcs.Health,
 526                 ServiceNameNodemanager:   svcs.Nodemanager,
 527                 ServiceNameWorkbench1:    svcs.Workbench1,
 528                 ServiceNameWorkbench2:    svcs.Workbench2,
 529                 ServiceNameWebsocket:     svcs.Websocket,
 530                 ServiceNameKeepbalance:   svcs.Keepbalance,
 531                 ServiceNameKeepweb:       svcs.WebDAV,
 532                 ServiceNameKeepproxy:     svcs.Keepproxy,
 533                 ServiceNameKeepstore:     svcs.Keepstore,
 534         }
 535 }