sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.curoverse.com/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters map[string]Cluster
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         if cc, ok := sc.Clusters[clusterID]; !ok {
  53                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  54         } else {
  55                 cc.ClusterID = clusterID
  56                 return &cc, nil
  57         }
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxBlockEntries      int
  64         MaxCollectionEntries int
  65         MaxCollectionBytes   int64
  66         MaxPermissionEntries int
  67         MaxUUIDEntries       int
  68 }
  69 type Cluster struct {
  70         ClusterID       string `json:"-"`
  71         ManagementToken string
  72         SystemRootToken string
  73         Services        Services
  74         InstanceTypes   InstanceTypeMap
  75         Containers      ContainersConfig
  76         RemoteClusters  map[string]RemoteCluster
  77         PostgreSQL      PostgreSQL
  78
  79         API struct {
  80                 AsyncPermissionsUpdateInterval Duration
  81                 DisabledAPIs                   StringSet
  82                 MaxIndexDatabaseRead           int
  83                 MaxItemsPerResponse            int
  84                 MaxConcurrentRequests          int
  85                 MaxKeepBlockBuffers            int
  86                 MaxRequestAmplification        int
  87                 MaxRequestSize                 int
  88                 RailsSessionSecretToken        string
  89                 RequestTimeout                 Duration
  90                 SendTimeout                    Duration
  91                 WebsocketClientEventQueue      int
  92                 WebsocketServerEventQueue      int
  93                 KeepServiceRequestTimeout      Duration
  94         }
  95         AuditLogs struct {
  96                 MaxAge             Duration
  97                 MaxDeleteBatch     int
  98                 UnloggedAttributes StringSet
  99         }
 100         Collections struct {
 101                 BlobSigning              bool
 102                 BlobSigningKey           string
 103                 BlobSigningTTL           Duration
 104                 BlobTrash                bool
 105                 BlobTrashLifetime        Duration
 106                 BlobTrashCheckInterval   Duration
 107                 BlobTrashConcurrency     int
 108                 BlobDeleteConcurrency    int
 109                 BlobReplicateConcurrency int
 110                 CollectionVersioning     bool
 111                 DefaultTrashLifetime     Duration
 112                 DefaultReplication       int
 113                 ManagedProperties        map[string]struct {
 114                         Value     interface{}
 115                         Function  string
 116                         Protected bool
 117                 }
 118                 PreserveVersionIfIdle Duration
 119                 TrashSweepInterval    Duration
 120                 TrustAllContent       bool
 121
 122                 WebDAVCache WebDAVCacheConfig
 123         }
 124         Git struct {
 125                 GitCommand   string
 126                 GitoliteHome string
 127                 Repositories string
 128         }
 129         Login struct {
 130                 ProviderAppSecret string
 131                 ProviderAppID     string
 132         }
 133         Mail struct {
 134                 MailchimpAPIKey                string
 135                 MailchimpListID                string
 136                 SendUserSetupNotificationEmail bool
 137                 IssueReporterEmailFrom         string
 138                 IssueReporterEmailTo           string
 139                 SupportEmailAddress            string
 140                 EmailFrom                      string
 141         }
 142         SystemLogs struct {
 143                 LogLevel                string
 144                 Format                  string
 145                 MaxRequestLogParamsSize int
 146         }
 147         TLS struct {
 148                 Certificate string
 149                 Key         string
 150                 Insecure    bool
 151         }
 152         Users struct {
 153                 AnonymousUserToken                    string
 154                 AdminNotifierEmailFrom                string
 155                 AutoAdminFirstUser                    bool
 156                 AutoAdminUserWithEmail                string
 157                 AutoSetupNewUsers                     bool
 158                 AutoSetupNewUsersWithRepository       bool
 159                 AutoSetupNewUsersWithVmUUID           string
 160                 AutoSetupUsernameBlacklist            StringSet
 161                 EmailSubjectPrefix                    string
 162                 NewInactiveUserNotificationRecipients StringSet
 163                 NewUserNotificationRecipients         StringSet
 164                 NewUsersAreActive                     bool
 165                 UserNotifierEmailFrom                 string
 166                 UserProfileNotificationAddress        string
 167         }
 168         Volumes   map[string]Volume
 169         Workbench struct {
 170                 ActivationContactLink            string
 171                 APIClientConnectTimeout          Duration
 172                 APIClientReceiveTimeout          Duration
 173                 APIResponseCompression           bool
 174                 ApplicationMimetypesWithViewIcon StringSet
 175                 ArvadosDocsite                   string
 176                 ArvadosPublicDataDocURL          string
 177                 DefaultOpenIdPrefix              string
 178                 EnableGettingStartedPopup        bool
 179                 EnablePublicProjectsPage         bool
 180                 FileViewersConfigURL             string
 181                 LogViewerMaxBytes                ByteSize
 182                 MultiSiteSearch                  string
 183                 ProfilingEnabled                 bool
 184                 Repositories                     bool
 185                 RepositoryCache                  string
 186                 RunningJobLogRecordsToFetch      int
 187                 SecretKeyBase                    string
 188                 ShowRecentCollectionsOnDashboard bool
 189                 ShowUserAgreementInline          bool
 190                 ShowUserNotifications            bool
 191                 SiteName                         string
 192                 Theme                            string
 193                 UserProfileFormFields            map[string]struct {
 194                         Type                 string
 195                         FormFieldTitle       string
 196                         FormFieldDescription string
 197                         Required             bool
 198                         Position             int
 199                         Options              map[string]struct{}
 200                 }
 201                 UserProfileFormMessage string
 202                 VocabularyURL          string
 203         }
 204
 205         EnableBetaController14287 bool
 206 }
 207
 208 type Volume struct {
 209         AccessViaHosts   map[URL]VolumeAccess
 210         ReadOnly         bool
 211         Replication      int
 212         StorageClasses   map[string]bool
 213         Driver           string
 214         DriverParameters json.RawMessage
 215 }
 216
 217 type S3VolumeDriverParameters struct {
 218         AccessKey          string
 219         SecretKey          string
 220         Endpoint           string
 221         Region             string
 222         Bucket             string
 223         LocationConstraint bool
 224         IndexPageSize      int
 225         ConnectTimeout     Duration
 226         ReadTimeout        Duration
 227         RaceWindow         Duration
 228         UnsafeDelete       bool
 229 }
 230
 231 type AzureVolumeDriverParameters struct {
 232         StorageAccountName   string
 233         StorageAccountKey    string
 234         StorageBaseURL       string
 235         ContainerName        string
 236         RequestTimeout       Duration
 237         ListBlobsRetryDelay  Duration
 238         ListBlobsMaxAttempts int
 239 }
 240
 241 type DirectoryVolumeDriverParameters struct {
 242         Root      string
 243         Serialize bool
 244 }
 245
 246 type VolumeAccess struct {
 247         ReadOnly bool
 248 }
 249
 250 type Services struct {
 251         Composer       Service
 252         Controller     Service
 253         DispatchCloud  Service
 254         GitHTTP        Service
 255         GitSSH         Service
 256         Health         Service
 257         Keepbalance    Service
 258         Keepproxy      Service
 259         Keepstore      Service
 260         Nodemanager    Service
 261         RailsAPI       Service
 262         SSO            Service
 263         WebDAVDownload Service
 264         WebDAV         Service
 265         WebShell       Service
 266         Websocket      Service
 267         Workbench1     Service
 268         Workbench2     Service
 269 }
 270
 271 type Service struct {
 272         InternalURLs map[URL]ServiceInstance
 273         ExternalURL  URL
 274 }
 275
 276 // URL is a url.URL that is also usable as a JSON key/value.
 277 type URL url.URL
 278
 279 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 280 // used as a JSON key/value.
 281 func (su *URL) UnmarshalText(text []byte) error {
 282         u, err := url.Parse(string(text))
 283         if err == nil {
 284                 *su = URL(*u)
 285         }
 286         return err
 287 }
 288
 289 func (su URL) MarshalText() ([]byte, error) {
 290         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 291 }
 292
 293 func (su URL) String() string {
 294         return (*url.URL)(&su).String()
 295 }
 296
 297 type ServiceInstance struct {
 298         // Rendezvous is normally empty; when changing the URL of a
 299         // Keepstore service, Rendezvous can be set to the old URL to
 300         // preserve rendezvous ordering.
 301         Rendezvous string `json:",omitempty"`
 302 }
 303
 304 type PostgreSQL struct {
 305         Connection     PostgreSQLConnection
 306         ConnectionPool int
 307 }
 308
 309 type PostgreSQLConnection map[string]string
 310
 311 type RemoteCluster struct {
 312         Host          string
 313         Proxy         bool
 314         Scheme        string
 315         Insecure      bool
 316         ActivateUsers bool
 317 }
 318
 319 type InstanceType struct {
 320         Name            string
 321         ProviderType    string
 322         VCPUs           int
 323         RAM             ByteSize
 324         Scratch         ByteSize
 325         IncludedScratch ByteSize
 326         AddedScratch    ByteSize
 327         Price           float64
 328         Preemptible     bool
 329 }
 330
 331 type ContainersConfig struct {
 332         CloudVMs                    CloudVMsConfig
 333         CrunchRunCommand            string
 334         CrunchRunArgumentsList      []string
 335         DefaultKeepCacheRAM         ByteSize
 336         DispatchPrivateKey          string
 337         LogReuseDecisions           bool
 338         MaxComputeVMs               int
 339         MaxDispatchAttempts         int
 340         MaxRetryAttempts            int
 341         MinRetryPeriod              Duration
 342         ReserveExtraRAM             ByteSize
 343         StaleLockTimeout            Duration
 344         SupportedDockerImageFormats StringSet
 345         UsePreemptibleInstances     bool
 346
 347         JobsAPI struct {
 348                 Enable         string
 349                 GitInternalDir string
 350         }
 351         Logging struct {
 352                 MaxAge                       Duration
 353                 LogBytesPerEvent             int
 354                 LogSecondsBetweenEvents      int
 355                 LogThrottlePeriod            Duration
 356                 LogThrottleBytes             int
 357                 LogThrottleLines             int
 358                 LimitLogBytesPerJob          int
 359                 LogPartialLineThrottlePeriod Duration
 360                 LogUpdatePeriod              Duration
 361                 LogUpdateSize                ByteSize
 362         }
 363         SLURM struct {
 364                 PrioritySpread             int64
 365                 SbatchArgumentsList        []string
 366                 SbatchEnvironmentVariables map[string]string
 367                 Managed                    struct {
 368                         DNSServerConfDir       string
 369                         DNSServerConfTemplate  string
 370                         DNSServerReloadCommand string
 371                         DNSServerUpdateCommand string
 372                         ComputeNodeDomain      string
 373                         ComputeNodeNameservers StringSet
 374                         AssignNodeHostname     string
 375                 }
 376         }
 377 }
 378
 379 type CloudVMsConfig struct {
 380         Enable bool
 381
 382         BootProbeCommand     string
 383         ImageID              string
 384         MaxCloudOpsPerSecond int
 385         MaxProbesPerSecond   int
 386         PollInterval         Duration
 387         ProbeInterval        Duration
 388         SSHPort              string
 389         SyncInterval         Duration
 390         TimeoutBooting       Duration
 391         TimeoutIdle          Duration
 392         TimeoutProbe         Duration
 393         TimeoutShutdown      Duration
 394         TimeoutSignal        Duration
 395         TimeoutTERM          Duration
 396         ResourceTags         map[string]string
 397         TagKeyPrefix         string
 398
 399         Driver           string
 400         DriverParameters json.RawMessage
 401 }
 402
 403 type InstanceTypeMap map[string]InstanceType
 404
 405 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 406
 407 // UnmarshalJSON handles old config files that provide an array of
 408 // instance types instead of a hash.
 409 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 410         if len(data) > 0 && data[0] == '[' {
 411                 var arr []InstanceType
 412                 err := json.Unmarshal(data, &arr)
 413                 if err != nil {
 414                         return err
 415                 }
 416                 if len(arr) == 0 {
 417                         *it = nil
 418                         return nil
 419                 }
 420                 *it = make(map[string]InstanceType, len(arr))
 421                 for _, t := range arr {
 422                         if _, ok := (*it)[t.Name]; ok {
 423                                 return errDuplicateInstanceTypeName
 424                         }
 425                         if t.ProviderType == "" {
 426                                 t.ProviderType = t.Name
 427                         }
 428                         if t.Scratch == 0 {
 429                                 t.Scratch = t.IncludedScratch + t.AddedScratch
 430                         } else if t.AddedScratch == 0 {
 431                                 t.AddedScratch = t.Scratch - t.IncludedScratch
 432                         } else if t.IncludedScratch == 0 {
 433                                 t.IncludedScratch = t.Scratch - t.AddedScratch
 434                         }
 435
 436                         if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 437                                 return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 438                         }
 439                         (*it)[t.Name] = t
 440                 }
 441                 return nil
 442         }
 443         var hash map[string]InstanceType
 444         err := json.Unmarshal(data, &hash)
 445         if err != nil {
 446                 return err
 447         }
 448         // Fill in Name field (and ProviderType field, if not
 449         // specified) using hash key.
 450         *it = InstanceTypeMap(hash)
 451         for name, t := range *it {
 452                 t.Name = name
 453                 if t.ProviderType == "" {
 454                         t.ProviderType = name
 455                 }
 456                 (*it)[name] = t
 457         }
 458         return nil
 459 }
 460
 461 type StringSet map[string]struct{}
 462
 463 // UnmarshalJSON handles old config files that provide an array of
 464 // instance types instead of a hash.
 465 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 466         if len(data) > 0 && data[0] == '[' {
 467                 var arr []string
 468                 err := json.Unmarshal(data, &arr)
 469                 if err != nil {
 470                         return err
 471                 }
 472                 if len(arr) == 0 {
 473                         *ss = nil
 474                         return nil
 475                 }
 476                 *ss = make(map[string]struct{}, len(arr))
 477                 for _, t := range arr {
 478                         (*ss)[t] = struct{}{}
 479                 }
 480                 return nil
 481         }
 482         var hash map[string]struct{}
 483         err := json.Unmarshal(data, &hash)
 484         if err != nil {
 485                 return err
 486         }
 487         *ss = make(map[string]struct{}, len(hash))
 488         for t, _ := range hash {
 489                 (*ss)[t] = struct{}{}
 490         }
 491
 492         return nil
 493 }
 494
 495 type ServiceName string
 496
 497 const (
 498         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 499         ServiceNameController    ServiceName = "arvados-controller"
 500         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 501         ServiceNameHealth        ServiceName = "arvados-health"
 502         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 503         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 504         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 505         ServiceNameWebsocket     ServiceName = "arvados-ws"
 506         ServiceNameKeepbalance   ServiceName = "keep-balance"
 507         ServiceNameKeepweb       ServiceName = "keep-web"
 508         ServiceNameKeepproxy     ServiceName = "keepproxy"
 509         ServiceNameKeepstore     ServiceName = "keepstore"
 510 )
 511
 512 // Map returns all services as a map, suitable for iterating over all
 513 // services or looking up a service by name.
 514 func (svcs Services) Map() map[ServiceName]Service {
 515         return map[ServiceName]Service{
 516                 ServiceNameRailsAPI:      svcs.RailsAPI,
 517                 ServiceNameController:    svcs.Controller,
 518                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 519                 ServiceNameHealth:        svcs.Health,
 520                 ServiceNameNodemanager:   svcs.Nodemanager,
 521                 ServiceNameWorkbench1:    svcs.Workbench1,
 522                 ServiceNameWorkbench2:    svcs.Workbench2,
 523                 ServiceNameWebsocket:     svcs.Websocket,
 524                 ServiceNameKeepbalance:   svcs.Keepbalance,
 525                 ServiceNameKeepweb:       svcs.WebDAV,
 526                 ServiceNameKeepproxy:     svcs.Keepproxy,
 527                 ServiceNameKeepstore:     svcs.Keepstore,
 528         }
 529 }