sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.curoverse.com/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters map[string]Cluster
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         if cc, ok := sc.Clusters[clusterID]; !ok {
  53                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  54         } else {
  55                 cc.ClusterID = clusterID
  56                 return &cc, nil
  57         }
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxBlockEntries      int
  64         MaxCollectionEntries int
  65         MaxCollectionBytes   int64
  66         MaxPermissionEntries int
  67         MaxUUIDEntries       int
  68 }
  69 type Cluster struct {
  70         ClusterID       string `json:"-"`
  71         ManagementToken string
  72         SystemRootToken string
  73         Services        Services
  74         InstanceTypes   InstanceTypeMap
  75         Containers      ContainersConfig
  76         RemoteClusters  map[string]RemoteCluster
  77         PostgreSQL      PostgreSQL
  78
  79         API struct {
  80                 AsyncPermissionsUpdateInterval Duration
  81                 DisabledAPIs                   StringSet
  82                 MaxIndexDatabaseRead           int
  83                 MaxItemsPerResponse            int
  84                 MaxConcurrentRequests          int
  85                 MaxKeepBlobBuffers             int
  86                 MaxRequestAmplification        int
  87                 MaxRequestSize                 int
  88                 RailsSessionSecretToken        string
  89                 RequestTimeout                 Duration
  90                 SendTimeout                    Duration
  91                 WebsocketClientEventQueue      int
  92                 WebsocketServerEventQueue      int
  93                 KeepServiceRequestTimeout      Duration
  94         }
  95         AuditLogs struct {
  96                 MaxAge             Duration
  97                 MaxDeleteBatch     int
  98                 UnloggedAttributes StringSet
  99         }
 100         Collections struct {
 101                 BlobSigning              bool
 102                 BlobSigningKey           string
 103                 BlobSigningTTL           Duration
 104                 BlobTrash                bool
 105                 BlobTrashLifetime        Duration
 106                 BlobTrashCheckInterval   Duration
 107                 BlobTrashConcurrency     int
 108                 BlobDeleteConcurrency    int
 109                 BlobReplicateConcurrency int
 110                 CollectionVersioning     bool
 111                 DefaultTrashLifetime     Duration
 112                 DefaultReplication       int
 113                 ManagedProperties        map[string]struct {
 114                         Value     interface{}
 115                         Function  string
 116                         Protected bool
 117                 }
 118                 PreserveVersionIfIdle Duration
 119                 TrashSweepInterval    Duration
 120                 TrustAllContent       bool
 121
 122                 BlobMissingReport        string
 123                 BalancePeriod            Duration
 124                 BalanceCollectionBatch   int
 125                 BalanceCollectionBuffers int
 126
 127                 WebDAVCache WebDAVCacheConfig
 128         }
 129         Git struct {
 130                 GitCommand   string
 131                 GitoliteHome string
 132                 Repositories string
 133         }
 134         Login struct {
 135                 GoogleClientID                string
 136                 GoogleClientSecret            string
 137                 GoogleAlternateEmailAddresses bool
 138                 ProviderAppID                 string
 139                 ProviderAppSecret             string
 140                 LoginCluster                  string
 141                 RemoteTokenRefresh            Duration
 142         }
 143         Mail struct {
 144                 MailchimpAPIKey                string
 145                 MailchimpListID                string
 146                 SendUserSetupNotificationEmail bool
 147                 IssueReporterEmailFrom         string
 148                 IssueReporterEmailTo           string
 149                 SupportEmailAddress            string
 150                 EmailFrom                      string
 151         }
 152         SystemLogs struct {
 153                 LogLevel                string
 154                 Format                  string
 155                 MaxRequestLogParamsSize int
 156         }
 157         TLS struct {
 158                 Certificate string
 159                 Key         string
 160                 Insecure    bool
 161         }
 162         Users struct {
 163                 AnonymousUserToken                    string
 164                 AdminNotifierEmailFrom                string
 165                 AutoAdminFirstUser                    bool
 166                 AutoAdminUserWithEmail                string
 167                 AutoSetupNewUsers                     bool
 168                 AutoSetupNewUsersWithRepository       bool
 169                 AutoSetupNewUsersWithVmUUID           string
 170                 AutoSetupUsernameBlacklist            StringSet
 171                 EmailSubjectPrefix                    string
 172                 NewInactiveUserNotificationRecipients StringSet
 173                 NewUserNotificationRecipients         StringSet
 174                 NewUsersAreActive                     bool
 175                 UserNotifierEmailFrom                 string
 176                 UserProfileNotificationAddress        string
 177                 PreferDomainForUsername               string
 178         }
 179         Volumes   map[string]Volume
 180         Workbench struct {
 181                 ActivationContactLink            string
 182                 APIClientConnectTimeout          Duration
 183                 APIClientReceiveTimeout          Duration
 184                 APIResponseCompression           bool
 185                 ApplicationMimetypesWithViewIcon StringSet
 186                 ArvadosDocsite                   string
 187                 ArvadosPublicDataDocURL          string
 188                 DefaultOpenIdPrefix              string
 189                 EnableGettingStartedPopup        bool
 190                 EnablePublicProjectsPage         bool
 191                 FileViewersConfigURL             string
 192                 LogViewerMaxBytes                ByteSize
 193                 MultiSiteSearch                  string
 194                 ProfilingEnabled                 bool
 195                 Repositories                     bool
 196                 RepositoryCache                  string
 197                 RunningJobLogRecordsToFetch      int
 198                 SecretKeyBase                    string
 199                 ShowRecentCollectionsOnDashboard bool
 200                 ShowUserAgreementInline          bool
 201                 ShowUserNotifications            bool
 202                 SiteName                         string
 203                 Theme                            string
 204                 UserProfileFormFields            map[string]struct {
 205                         Type                 string
 206                         FormFieldTitle       string
 207                         FormFieldDescription string
 208                         Required             bool
 209                         Position             int
 210                         Options              map[string]struct{}
 211                 }
 212                 UserProfileFormMessage string
 213                 VocabularyURL          string
 214                 WelcomePageHTML        string
 215                 InactivePageHTML       string
 216         }
 217
 218         EnableBetaController14287 bool
 219 }
 220
 221 type Volume struct {
 222         AccessViaHosts   map[URL]VolumeAccess
 223         ReadOnly         bool
 224         Replication      int
 225         StorageClasses   map[string]bool
 226         Driver           string
 227         DriverParameters json.RawMessage
 228 }
 229
 230 type S3VolumeDriverParameters struct {
 231         AccessKey          string
 232         SecretKey          string
 233         Endpoint           string
 234         Region             string
 235         Bucket             string
 236         LocationConstraint bool
 237         IndexPageSize      int
 238         ConnectTimeout     Duration
 239         ReadTimeout        Duration
 240         RaceWindow         Duration
 241         UnsafeDelete       bool
 242 }
 243
 244 type AzureVolumeDriverParameters struct {
 245         StorageAccountName   string
 246         StorageAccountKey    string
 247         StorageBaseURL       string
 248         ContainerName        string
 249         RequestTimeout       Duration
 250         ListBlobsRetryDelay  Duration
 251         ListBlobsMaxAttempts int
 252 }
 253
 254 type DirectoryVolumeDriverParameters struct {
 255         Root      string
 256         Serialize bool
 257 }
 258
 259 type VolumeAccess struct {
 260         ReadOnly bool
 261 }
 262
 263 type Services struct {
 264         Composer       Service
 265         Controller     Service
 266         DispatchCloud  Service
 267         GitHTTP        Service
 268         GitSSH         Service
 269         Health         Service
 270         Keepbalance    Service
 271         Keepproxy      Service
 272         Keepstore      Service
 273         Nodemanager    Service
 274         RailsAPI       Service
 275         SSO            Service
 276         WebDAVDownload Service
 277         WebDAV         Service
 278         WebShell       Service
 279         Websocket      Service
 280         Workbench1     Service
 281         Workbench2     Service
 282 }
 283
 284 type Service struct {
 285         InternalURLs map[URL]ServiceInstance
 286         ExternalURL  URL
 287 }
 288
 289 // URL is a url.URL that is also usable as a JSON key/value.
 290 type URL url.URL
 291
 292 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 293 // used as a JSON key/value.
 294 func (su *URL) UnmarshalText(text []byte) error {
 295         u, err := url.Parse(string(text))
 296         if err == nil {
 297                 *su = URL(*u)
 298         }
 299         return err
 300 }
 301
 302 func (su URL) MarshalText() ([]byte, error) {
 303         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 304 }
 305
 306 func (su URL) String() string {
 307         return (*url.URL)(&su).String()
 308 }
 309
 310 type ServiceInstance struct {
 311         Rendezvous string `json:",omitempty"`
 312 }
 313
 314 type PostgreSQL struct {
 315         Connection     PostgreSQLConnection
 316         ConnectionPool int
 317 }
 318
 319 type PostgreSQLConnection map[string]string
 320
 321 type RemoteCluster struct {
 322         Host          string
 323         Proxy         bool
 324         Scheme        string
 325         Insecure      bool
 326         ActivateUsers bool
 327 }
 328
 329 type InstanceType struct {
 330         Name            string
 331         ProviderType    string
 332         VCPUs           int
 333         RAM             ByteSize
 334         Scratch         ByteSize
 335         IncludedScratch ByteSize
 336         AddedScratch    ByteSize
 337         Price           float64
 338         Preemptible     bool
 339 }
 340
 341 type ContainersConfig struct {
 342         CloudVMs                    CloudVMsConfig
 343         CrunchRunCommand            string
 344         CrunchRunArgumentsList      []string
 345         DefaultKeepCacheRAM         ByteSize
 346         DispatchPrivateKey          string
 347         LogReuseDecisions           bool
 348         MaxComputeVMs               int
 349         MaxDispatchAttempts         int
 350         MaxRetryAttempts            int
 351         MinRetryPeriod              Duration
 352         ReserveExtraRAM             ByteSize
 353         StaleLockTimeout            Duration
 354         SupportedDockerImageFormats StringSet
 355         UsePreemptibleInstances     bool
 356
 357         JobsAPI struct {
 358                 Enable         string
 359                 GitInternalDir string
 360         }
 361         Logging struct {
 362                 MaxAge                       Duration
 363                 LogBytesPerEvent             int
 364                 LogSecondsBetweenEvents      int
 365                 LogThrottlePeriod            Duration
 366                 LogThrottleBytes             int
 367                 LogThrottleLines             int
 368                 LimitLogBytesPerJob          int
 369                 LogPartialLineThrottlePeriod Duration
 370                 LogUpdatePeriod              Duration
 371                 LogUpdateSize                ByteSize
 372         }
 373         SLURM struct {
 374                 PrioritySpread             int64
 375                 SbatchArgumentsList        []string
 376                 SbatchEnvironmentVariables map[string]string
 377                 Managed                    struct {
 378                         DNSServerConfDir       string
 379                         DNSServerConfTemplate  string
 380                         DNSServerReloadCommand string
 381                         DNSServerUpdateCommand string
 382                         ComputeNodeDomain      string
 383                         ComputeNodeNameservers StringSet
 384                         AssignNodeHostname     string
 385                 }
 386         }
 387 }
 388
 389 type CloudVMsConfig struct {
 390         Enable bool
 391
 392         BootProbeCommand     string
 393         ImageID              string
 394         MaxCloudOpsPerSecond int
 395         MaxProbesPerSecond   int
 396         PollInterval         Duration
 397         ProbeInterval        Duration
 398         SSHPort              string
 399         SyncInterval         Duration
 400         TimeoutBooting       Duration
 401         TimeoutIdle          Duration
 402         TimeoutProbe         Duration
 403         TimeoutShutdown      Duration
 404         TimeoutSignal        Duration
 405         TimeoutTERM          Duration
 406         ResourceTags         map[string]string
 407         TagKeyPrefix         string
 408
 409         Driver           string
 410         DriverParameters json.RawMessage
 411 }
 412
 413 type InstanceTypeMap map[string]InstanceType
 414
 415 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 416
 417 // UnmarshalJSON handles old config files that provide an array of
 418 // instance types instead of a hash.
 419 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 420         if len(data) > 0 && data[0] == '[' {
 421                 var arr []InstanceType
 422                 err := json.Unmarshal(data, &arr)
 423                 if err != nil {
 424                         return err
 425                 }
 426                 if len(arr) == 0 {
 427                         *it = nil
 428                         return nil
 429                 }
 430                 *it = make(map[string]InstanceType, len(arr))
 431                 for _, t := range arr {
 432                         if _, ok := (*it)[t.Name]; ok {
 433                                 return errDuplicateInstanceTypeName
 434                         }
 435                         if t.ProviderType == "" {
 436                                 t.ProviderType = t.Name
 437                         }
 438                         if t.Scratch == 0 {
 439                                 t.Scratch = t.IncludedScratch + t.AddedScratch
 440                         } else if t.AddedScratch == 0 {
 441                                 t.AddedScratch = t.Scratch - t.IncludedScratch
 442                         } else if t.IncludedScratch == 0 {
 443                                 t.IncludedScratch = t.Scratch - t.AddedScratch
 444                         }
 445
 446                         if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 447                                 return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 448                         }
 449                         (*it)[t.Name] = t
 450                 }
 451                 return nil
 452         }
 453         var hash map[string]InstanceType
 454         err := json.Unmarshal(data, &hash)
 455         if err != nil {
 456                 return err
 457         }
 458         // Fill in Name field (and ProviderType field, if not
 459         // specified) using hash key.
 460         *it = InstanceTypeMap(hash)
 461         for name, t := range *it {
 462                 t.Name = name
 463                 if t.ProviderType == "" {
 464                         t.ProviderType = name
 465                 }
 466                 (*it)[name] = t
 467         }
 468         return nil
 469 }
 470
 471 type StringSet map[string]struct{}
 472
 473 // UnmarshalJSON handles old config files that provide an array of
 474 // instance types instead of a hash.
 475 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 476         if len(data) > 0 && data[0] == '[' {
 477                 var arr []string
 478                 err := json.Unmarshal(data, &arr)
 479                 if err != nil {
 480                         return err
 481                 }
 482                 if len(arr) == 0 {
 483                         *ss = nil
 484                         return nil
 485                 }
 486                 *ss = make(map[string]struct{}, len(arr))
 487                 for _, t := range arr {
 488                         (*ss)[t] = struct{}{}
 489                 }
 490                 return nil
 491         }
 492         var hash map[string]struct{}
 493         err := json.Unmarshal(data, &hash)
 494         if err != nil {
 495                 return err
 496         }
 497         *ss = make(map[string]struct{}, len(hash))
 498         for t, _ := range hash {
 499                 (*ss)[t] = struct{}{}
 500         }
 501
 502         return nil
 503 }
 504
 505 type ServiceName string
 506
 507 const (
 508         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 509         ServiceNameController    ServiceName = "arvados-controller"
 510         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 511         ServiceNameHealth        ServiceName = "arvados-health"
 512         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 513         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 514         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 515         ServiceNameWebsocket     ServiceName = "arvados-ws"
 516         ServiceNameKeepbalance   ServiceName = "keep-balance"
 517         ServiceNameKeepweb       ServiceName = "keep-web"
 518         ServiceNameKeepproxy     ServiceName = "keepproxy"
 519         ServiceNameKeepstore     ServiceName = "keepstore"
 520 )
 521
 522 // Map returns all services as a map, suitable for iterating over all
 523 // services or looking up a service by name.
 524 func (svcs Services) Map() map[ServiceName]Service {
 525         return map[ServiceName]Service{
 526                 ServiceNameRailsAPI:      svcs.RailsAPI,
 527                 ServiceNameController:    svcs.Controller,
 528                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 529                 ServiceNameHealth:        svcs.Health,
 530                 ServiceNameNodemanager:   svcs.Nodemanager,
 531                 ServiceNameWorkbench1:    svcs.Workbench1,
 532                 ServiceNameWorkbench2:    svcs.Workbench2,
 533                 ServiceNameWebsocket:     svcs.Websocket,
 534                 ServiceNameKeepbalance:   svcs.Keepbalance,
 535                 ServiceNameKeepweb:       svcs.WebDAV,
 536                 ServiceNameKeepproxy:     svcs.Keepproxy,
 537                 ServiceNameKeepstore:     svcs.Keepstore,
 538         }
 539 }