sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.curoverse.com/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters map[string]Cluster
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         if cc, ok := sc.Clusters[clusterID]; !ok {
  53                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  54         } else {
  55                 cc.ClusterID = clusterID
  56                 return &cc, nil
  57         }
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxBlockEntries      int
  64         MaxCollectionEntries int
  65         MaxCollectionBytes   int64
  66         MaxPermissionEntries int
  67         MaxUUIDEntries       int
  68 }
  69 type Cluster struct {
  70         ClusterID       string `json:"-"`
  71         ManagementToken string
  72         SystemRootToken string
  73         Services        Services
  74         InstanceTypes   InstanceTypeMap
  75         Containers      ContainersConfig
  76         RemoteClusters  map[string]RemoteCluster
  77         PostgreSQL      PostgreSQL
  78
  79         API struct {
  80                 AsyncPermissionsUpdateInterval Duration
  81                 DisabledAPIs                   StringSet
  82                 MaxIndexDatabaseRead           int
  83                 MaxItemsPerResponse            int
  84                 MaxConcurrentRequests          int
  85                 MaxKeepBlobBuffers             int
  86                 MaxRequestAmplification        int
  87                 MaxRequestSize                 int
  88                 RailsSessionSecretToken        string
  89                 RequestTimeout                 Duration
  90                 SendTimeout                    Duration
  91                 WebsocketClientEventQueue      int
  92                 WebsocketServerEventQueue      int
  93                 KeepServiceRequestTimeout      Duration
  94         }
  95         AuditLogs struct {
  96                 MaxAge             Duration
  97                 MaxDeleteBatch     int
  98                 UnloggedAttributes StringSet
  99         }
 100         Collections struct {
 101                 BlobSigning              bool
 102                 BlobSigningKey           string
 103                 BlobSigningTTL           Duration
 104                 BlobTrash                bool
 105                 BlobTrashLifetime        Duration
 106                 BlobTrashCheckInterval   Duration
 107                 BlobTrashConcurrency     int
 108                 BlobDeleteConcurrency    int
 109                 BlobReplicateConcurrency int
 110                 CollectionVersioning     bool
 111                 DefaultTrashLifetime     Duration
 112                 DefaultReplication       int
 113                 ManagedProperties        map[string]struct {
 114                         Value     interface{}
 115                         Function  string
 116                         Protected bool
 117                 }
 118                 PreserveVersionIfIdle Duration
 119                 TrashSweepInterval    Duration
 120                 TrustAllContent       bool
 121
 122                 BlobMissingReport        string
 123                 BalancePeriod            Duration
 124                 BalanceCollectionBatch   int
 125                 BalanceCollectionBuffers int
 126
 127                 WebDAVCache WebDAVCacheConfig
 128         }
 129         Git struct {
 130                 GitCommand   string
 131                 GitoliteHome string
 132                 Repositories string
 133         }
 134         Login struct {
 135                 GoogleClientID                string
 136                 GoogleClientSecret            string
 137                 GoogleAlternateEmailAddresses bool
 138                 ProviderAppID                 string
 139                 ProviderAppSecret             string
 140                 LoginCluster                  string
 141                 RemoteTokenRefresh            Duration
 142         }
 143         Mail struct {
 144                 MailchimpAPIKey                string
 145                 MailchimpListID                string
 146                 SendUserSetupNotificationEmail bool
 147                 IssueReporterEmailFrom         string
 148                 IssueReporterEmailTo           string
 149                 SupportEmailAddress            string
 150                 EmailFrom                      string
 151         }
 152         SystemLogs struct {
 153                 LogLevel                string
 154                 Format                  string
 155                 MaxRequestLogParamsSize int
 156         }
 157         TLS struct {
 158                 Certificate string
 159                 Key         string
 160                 Insecure    bool
 161         }
 162         Users struct {
 163                 AnonymousUserToken                    string
 164                 AdminNotifierEmailFrom                string
 165                 AutoAdminFirstUser                    bool
 166                 AutoAdminUserWithEmail                string
 167                 AutoSetupNewUsers                     bool
 168                 AutoSetupNewUsersWithRepository       bool
 169                 AutoSetupNewUsersWithVmUUID           string
 170                 AutoSetupUsernameBlacklist            StringSet
 171                 EmailSubjectPrefix                    string
 172                 NewInactiveUserNotificationRecipients StringSet
 173                 NewUserNotificationRecipients         StringSet
 174                 NewUsersAreActive                     bool
 175                 UserNotifierEmailFrom                 string
 176                 UserProfileNotificationAddress        string
 177         }
 178         Volumes   map[string]Volume
 179         Workbench struct {
 180                 ActivationContactLink            string
 181                 APIClientConnectTimeout          Duration
 182                 APIClientReceiveTimeout          Duration
 183                 APIResponseCompression           bool
 184                 ApplicationMimetypesWithViewIcon StringSet
 185                 ArvadosDocsite                   string
 186                 ArvadosPublicDataDocURL          string
 187                 DefaultOpenIdPrefix              string
 188                 EnableGettingStartedPopup        bool
 189                 EnablePublicProjectsPage         bool
 190                 FileViewersConfigURL             string
 191                 LogViewerMaxBytes                ByteSize
 192                 MultiSiteSearch                  string
 193                 ProfilingEnabled                 bool
 194                 Repositories                     bool
 195                 RepositoryCache                  string
 196                 RunningJobLogRecordsToFetch      int
 197                 SecretKeyBase                    string
 198                 ShowRecentCollectionsOnDashboard bool
 199                 ShowUserAgreementInline          bool
 200                 ShowUserNotifications            bool
 201                 SiteName                         string
 202                 Theme                            string
 203                 UserProfileFormFields            map[string]struct {
 204                         Type                 string
 205                         FormFieldTitle       string
 206                         FormFieldDescription string
 207                         Required             bool
 208                         Position             int
 209                         Options              map[string]struct{}
 210                 }
 211                 UserProfileFormMessage string
 212                 VocabularyURL          string
 213                 WelcomePageHTML        string
 214                 InactivePageHTML       string
 215         }
 216
 217         EnableBetaController14287 bool
 218 }
 219
 220 type Volume struct {
 221         AccessViaHosts   map[URL]VolumeAccess
 222         ReadOnly         bool
 223         Replication      int
 224         StorageClasses   map[string]bool
 225         Driver           string
 226         DriverParameters json.RawMessage
 227 }
 228
 229 type S3VolumeDriverParameters struct {
 230         AccessKey          string
 231         SecretKey          string
 232         Endpoint           string
 233         Region             string
 234         Bucket             string
 235         LocationConstraint bool
 236         IndexPageSize      int
 237         ConnectTimeout     Duration
 238         ReadTimeout        Duration
 239         RaceWindow         Duration
 240         UnsafeDelete       bool
 241 }
 242
 243 type AzureVolumeDriverParameters struct {
 244         StorageAccountName   string
 245         StorageAccountKey    string
 246         StorageBaseURL       string
 247         ContainerName        string
 248         RequestTimeout       Duration
 249         ListBlobsRetryDelay  Duration
 250         ListBlobsMaxAttempts int
 251 }
 252
 253 type DirectoryVolumeDriverParameters struct {
 254         Root      string
 255         Serialize bool
 256 }
 257
 258 type VolumeAccess struct {
 259         ReadOnly bool
 260 }
 261
 262 type Services struct {
 263         Composer       Service
 264         Controller     Service
 265         DispatchCloud  Service
 266         GitHTTP        Service
 267         GitSSH         Service
 268         Health         Service
 269         Keepbalance    Service
 270         Keepproxy      Service
 271         Keepstore      Service
 272         Nodemanager    Service
 273         RailsAPI       Service
 274         SSO            Service
 275         WebDAVDownload Service
 276         WebDAV         Service
 277         WebShell       Service
 278         Websocket      Service
 279         Workbench1     Service
 280         Workbench2     Service
 281 }
 282
 283 type Service struct {
 284         InternalURLs map[URL]ServiceInstance
 285         ExternalURL  URL
 286 }
 287
 288 // URL is a url.URL that is also usable as a JSON key/value.
 289 type URL url.URL
 290
 291 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 292 // used as a JSON key/value.
 293 func (su *URL) UnmarshalText(text []byte) error {
 294         u, err := url.Parse(string(text))
 295         if err == nil {
 296                 *su = URL(*u)
 297         }
 298         return err
 299 }
 300
 301 func (su URL) MarshalText() ([]byte, error) {
 302         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 303 }
 304
 305 func (su URL) String() string {
 306         return (*url.URL)(&su).String()
 307 }
 308
 309 type ServiceInstance struct {
 310         Rendezvous string `json:",omitempty"`
 311 }
 312
 313 type PostgreSQL struct {
 314         Connection     PostgreSQLConnection
 315         ConnectionPool int
 316 }
 317
 318 type PostgreSQLConnection map[string]string
 319
 320 type RemoteCluster struct {
 321         Host          string
 322         Proxy         bool
 323         Scheme        string
 324         Insecure      bool
 325         ActivateUsers bool
 326 }
 327
 328 type InstanceType struct {
 329         Name            string
 330         ProviderType    string
 331         VCPUs           int
 332         RAM             ByteSize
 333         Scratch         ByteSize
 334         IncludedScratch ByteSize
 335         AddedScratch    ByteSize
 336         Price           float64
 337         Preemptible     bool
 338 }
 339
 340 type ContainersConfig struct {
 341         CloudVMs                    CloudVMsConfig
 342         CrunchRunCommand            string
 343         CrunchRunArgumentsList      []string
 344         DefaultKeepCacheRAM         ByteSize
 345         DispatchPrivateKey          string
 346         LogReuseDecisions           bool
 347         MaxComputeVMs               int
 348         MaxDispatchAttempts         int
 349         MaxRetryAttempts            int
 350         MinRetryPeriod              Duration
 351         ReserveExtraRAM             ByteSize
 352         StaleLockTimeout            Duration
 353         SupportedDockerImageFormats StringSet
 354         UsePreemptibleInstances     bool
 355
 356         JobsAPI struct {
 357                 Enable         string
 358                 GitInternalDir string
 359         }
 360         Logging struct {
 361                 MaxAge                       Duration
 362                 LogBytesPerEvent             int
 363                 LogSecondsBetweenEvents      int
 364                 LogThrottlePeriod            Duration
 365                 LogThrottleBytes             int
 366                 LogThrottleLines             int
 367                 LimitLogBytesPerJob          int
 368                 LogPartialLineThrottlePeriod Duration
 369                 LogUpdatePeriod              Duration
 370                 LogUpdateSize                ByteSize
 371         }
 372         SLURM struct {
 373                 PrioritySpread             int64
 374                 SbatchArgumentsList        []string
 375                 SbatchEnvironmentVariables map[string]string
 376                 Managed                    struct {
 377                         DNSServerConfDir       string
 378                         DNSServerConfTemplate  string
 379                         DNSServerReloadCommand string
 380                         DNSServerUpdateCommand string
 381                         ComputeNodeDomain      string
 382                         ComputeNodeNameservers StringSet
 383                         AssignNodeHostname     string
 384                 }
 385         }
 386 }
 387
 388 type CloudVMsConfig struct {
 389         Enable bool
 390
 391         BootProbeCommand     string
 392         ImageID              string
 393         MaxCloudOpsPerSecond int
 394         MaxProbesPerSecond   int
 395         PollInterval         Duration
 396         ProbeInterval        Duration
 397         SSHPort              string
 398         SyncInterval         Duration
 399         TimeoutBooting       Duration
 400         TimeoutIdle          Duration
 401         TimeoutProbe         Duration
 402         TimeoutShutdown      Duration
 403         TimeoutSignal        Duration
 404         TimeoutTERM          Duration
 405         ResourceTags         map[string]string
 406         TagKeyPrefix         string
 407
 408         Driver           string
 409         DriverParameters json.RawMessage
 410 }
 411
 412 type InstanceTypeMap map[string]InstanceType
 413
 414 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 415
 416 // UnmarshalJSON handles old config files that provide an array of
 417 // instance types instead of a hash.
 418 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 419         if len(data) > 0 && data[0] == '[' {
 420                 var arr []InstanceType
 421                 err := json.Unmarshal(data, &arr)
 422                 if err != nil {
 423                         return err
 424                 }
 425                 if len(arr) == 0 {
 426                         *it = nil
 427                         return nil
 428                 }
 429                 *it = make(map[string]InstanceType, len(arr))
 430                 for _, t := range arr {
 431                         if _, ok := (*it)[t.Name]; ok {
 432                                 return errDuplicateInstanceTypeName
 433                         }
 434                         if t.ProviderType == "" {
 435                                 t.ProviderType = t.Name
 436                         }
 437                         if t.Scratch == 0 {
 438                                 t.Scratch = t.IncludedScratch + t.AddedScratch
 439                         } else if t.AddedScratch == 0 {
 440                                 t.AddedScratch = t.Scratch - t.IncludedScratch
 441                         } else if t.IncludedScratch == 0 {
 442                                 t.IncludedScratch = t.Scratch - t.AddedScratch
 443                         }
 444
 445                         if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 446                                 return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 447                         }
 448                         (*it)[t.Name] = t
 449                 }
 450                 return nil
 451         }
 452         var hash map[string]InstanceType
 453         err := json.Unmarshal(data, &hash)
 454         if err != nil {
 455                 return err
 456         }
 457         // Fill in Name field (and ProviderType field, if not
 458         // specified) using hash key.
 459         *it = InstanceTypeMap(hash)
 460         for name, t := range *it {
 461                 t.Name = name
 462                 if t.ProviderType == "" {
 463                         t.ProviderType = name
 464                 }
 465                 (*it)[name] = t
 466         }
 467         return nil
 468 }
 469
 470 type StringSet map[string]struct{}
 471
 472 // UnmarshalJSON handles old config files that provide an array of
 473 // instance types instead of a hash.
 474 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 475         if len(data) > 0 && data[0] == '[' {
 476                 var arr []string
 477                 err := json.Unmarshal(data, &arr)
 478                 if err != nil {
 479                         return err
 480                 }
 481                 if len(arr) == 0 {
 482                         *ss = nil
 483                         return nil
 484                 }
 485                 *ss = make(map[string]struct{}, len(arr))
 486                 for _, t := range arr {
 487                         (*ss)[t] = struct{}{}
 488                 }
 489                 return nil
 490         }
 491         var hash map[string]struct{}
 492         err := json.Unmarshal(data, &hash)
 493         if err != nil {
 494                 return err
 495         }
 496         *ss = make(map[string]struct{}, len(hash))
 497         for t, _ := range hash {
 498                 (*ss)[t] = struct{}{}
 499         }
 500
 501         return nil
 502 }
 503
 504 type ServiceName string
 505
 506 const (
 507         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 508         ServiceNameController    ServiceName = "arvados-controller"
 509         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 510         ServiceNameHealth        ServiceName = "arvados-health"
 511         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 512         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 513         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 514         ServiceNameWebsocket     ServiceName = "arvados-ws"
 515         ServiceNameKeepbalance   ServiceName = "keep-balance"
 516         ServiceNameKeepweb       ServiceName = "keep-web"
 517         ServiceNameKeepproxy     ServiceName = "keepproxy"
 518         ServiceNameKeepstore     ServiceName = "keepstore"
 519 )
 520
 521 // Map returns all services as a map, suitable for iterating over all
 522 // services or looking up a service by name.
 523 func (svcs Services) Map() map[ServiceName]Service {
 524         return map[ServiceName]Service{
 525                 ServiceNameRailsAPI:      svcs.RailsAPI,
 526                 ServiceNameController:    svcs.Controller,
 527                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 528                 ServiceNameHealth:        svcs.Health,
 529                 ServiceNameNodemanager:   svcs.Nodemanager,
 530                 ServiceNameWorkbench1:    svcs.Workbench1,
 531                 ServiceNameWorkbench2:    svcs.Workbench2,
 532                 ServiceNameWebsocket:     svcs.Websocket,
 533                 ServiceNameKeepbalance:   svcs.Keepbalance,
 534                 ServiceNameKeepweb:       svcs.WebDAV,
 535                 ServiceNameKeepproxy:     svcs.Keepproxy,
 536                 ServiceNameKeepstore:     svcs.Keepstore,
 537         }
 538 }