sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.curoverse.com/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters map[string]Cluster
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         if cc, ok := sc.Clusters[clusterID]; !ok {
  53                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  54         } else {
  55                 cc.ClusterID = clusterID
  56                 return &cc, nil
  57         }
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxBlockEntries      int
  64         MaxCollectionEntries int
  65         MaxCollectionBytes   int64
  66         MaxPermissionEntries int
  67         MaxUUIDEntries       int
  68 }
  69 type Cluster struct {
  70         ClusterID       string `json:"-"`
  71         ManagementToken string
  72         SystemRootToken string
  73         Services        Services
  74         InstanceTypes   InstanceTypeMap
  75         Containers      ContainersConfig
  76         RemoteClusters  map[string]RemoteCluster
  77         PostgreSQL      PostgreSQL
  78
  79         API struct {
  80                 AsyncPermissionsUpdateInterval Duration
  81                 DisabledAPIs                   StringSet
  82                 MaxIndexDatabaseRead           int
  83                 MaxItemsPerResponse            int
  84                 MaxConcurrentRequests          int
  85                 MaxKeepBlobBuffers             int
  86                 MaxRequestAmplification        int
  87                 MaxRequestSize                 int
  88                 RailsSessionSecretToken        string
  89                 RequestTimeout                 Duration
  90                 SendTimeout                    Duration
  91                 WebsocketClientEventQueue      int
  92                 WebsocketServerEventQueue      int
  93                 KeepServiceRequestTimeout      Duration
  94         }
  95         AuditLogs struct {
  96                 MaxAge             Duration
  97                 MaxDeleteBatch     int
  98                 UnloggedAttributes StringSet
  99         }
 100         Collections struct {
 101                 BlobSigning              bool
 102                 BlobSigningKey           string
 103                 BlobSigningTTL           Duration
 104                 BlobTrash                bool
 105                 BlobTrashLifetime        Duration
 106                 BlobTrashCheckInterval   Duration
 107                 BlobTrashConcurrency     int
 108                 BlobDeleteConcurrency    int
 109                 BlobReplicateConcurrency int
 110                 CollectionVersioning     bool
 111                 DefaultTrashLifetime     Duration
 112                 DefaultReplication       int
 113                 ManagedProperties        map[string]struct {
 114                         Value     interface{}
 115                         Function  string
 116                         Protected bool
 117                 }
 118                 PreserveVersionIfIdle Duration
 119                 TrashSweepInterval    Duration
 120                 TrustAllContent       bool
 121
 122                 BlobMissingReport        string
 123                 BalancePeriod            Duration
 124                 BalanceCollectionBatch   int
 125                 BalanceCollectionBuffers int
 126
 127                 WebDAVCache WebDAVCacheConfig
 128         }
 129         Git struct {
 130                 GitCommand   string
 131                 GitoliteHome string
 132                 Repositories string
 133         }
 134         Login struct {
 135                 ProviderAppSecret  string
 136                 ProviderAppID      string
 137                 LoginCluster       string
 138                 RemoteTokenRefresh Duration
 139         }
 140         Mail struct {
 141                 MailchimpAPIKey                string
 142                 MailchimpListID                string
 143                 SendUserSetupNotificationEmail bool
 144                 IssueReporterEmailFrom         string
 145                 IssueReporterEmailTo           string
 146                 SupportEmailAddress            string
 147                 EmailFrom                      string
 148         }
 149         SystemLogs struct {
 150                 LogLevel                string
 151                 Format                  string
 152                 MaxRequestLogParamsSize int
 153         }
 154         TLS struct {
 155                 Certificate string
 156                 Key         string
 157                 Insecure    bool
 158         }
 159         Users struct {
 160                 AnonymousUserToken                    string
 161                 AdminNotifierEmailFrom                string
 162                 AutoAdminFirstUser                    bool
 163                 AutoAdminUserWithEmail                string
 164                 AutoSetupNewUsers                     bool
 165                 AutoSetupNewUsersWithRepository       bool
 166                 AutoSetupNewUsersWithVmUUID           string
 167                 AutoSetupUsernameBlacklist            StringSet
 168                 EmailSubjectPrefix                    string
 169                 NewInactiveUserNotificationRecipients StringSet
 170                 NewUserNotificationRecipients         StringSet
 171                 NewUsersAreActive                     bool
 172                 UserNotifierEmailFrom                 string
 173                 UserProfileNotificationAddress        string
 174         }
 175         Volumes   map[string]Volume
 176         Workbench struct {
 177                 ActivationContactLink            string
 178                 APIClientConnectTimeout          Duration
 179                 APIClientReceiveTimeout          Duration
 180                 APIResponseCompression           bool
 181                 ApplicationMimetypesWithViewIcon StringSet
 182                 ArvadosDocsite                   string
 183                 ArvadosPublicDataDocURL          string
 184                 DefaultOpenIdPrefix              string
 185                 EnableGettingStartedPopup        bool
 186                 EnablePublicProjectsPage         bool
 187                 FileViewersConfigURL             string
 188                 LogViewerMaxBytes                ByteSize
 189                 MultiSiteSearch                  string
 190                 ProfilingEnabled                 bool
 191                 Repositories                     bool
 192                 RepositoryCache                  string
 193                 RunningJobLogRecordsToFetch      int
 194                 SecretKeyBase                    string
 195                 ShowRecentCollectionsOnDashboard bool
 196                 ShowUserAgreementInline          bool
 197                 ShowUserNotifications            bool
 198                 SiteName                         string
 199                 Theme                            string
 200                 UserProfileFormFields            map[string]struct {
 201                         Type                 string
 202                         FormFieldTitle       string
 203                         FormFieldDescription string
 204                         Required             bool
 205                         Position             int
 206                         Options              map[string]struct{}
 207                 }
 208                 UserProfileFormMessage string
 209                 VocabularyURL          string
 210         }
 211
 212         EnableBetaController14287 bool
 213 }
 214
 215 type Volume struct {
 216         AccessViaHosts   map[URL]VolumeAccess
 217         ReadOnly         bool
 218         Replication      int
 219         StorageClasses   map[string]bool
 220         Driver           string
 221         DriverParameters json.RawMessage
 222 }
 223
 224 type S3VolumeDriverParameters struct {
 225         AccessKey          string
 226         SecretKey          string
 227         Endpoint           string
 228         Region             string
 229         Bucket             string
 230         LocationConstraint bool
 231         IndexPageSize      int
 232         ConnectTimeout     Duration
 233         ReadTimeout        Duration
 234         RaceWindow         Duration
 235         UnsafeDelete       bool
 236 }
 237
 238 type AzureVolumeDriverParameters struct {
 239         StorageAccountName   string
 240         StorageAccountKey    string
 241         StorageBaseURL       string
 242         ContainerName        string
 243         RequestTimeout       Duration
 244         ListBlobsRetryDelay  Duration
 245         ListBlobsMaxAttempts int
 246 }
 247
 248 type DirectoryVolumeDriverParameters struct {
 249         Root      string
 250         Serialize bool
 251 }
 252
 253 type VolumeAccess struct {
 254         ReadOnly bool
 255 }
 256
 257 type Services struct {
 258         Composer       Service
 259         Controller     Service
 260         DispatchCloud  Service
 261         GitHTTP        Service
 262         GitSSH         Service
 263         Health         Service
 264         Keepbalance    Service
 265         Keepproxy      Service
 266         Keepstore      Service
 267         Nodemanager    Service
 268         RailsAPI       Service
 269         SSO            Service
 270         WebDAVDownload Service
 271         WebDAV         Service
 272         WebShell       Service
 273         Websocket      Service
 274         Workbench1     Service
 275         Workbench2     Service
 276 }
 277
 278 type Service struct {
 279         InternalURLs map[URL]ServiceInstance
 280         ExternalURL  URL
 281 }
 282
 283 // URL is a url.URL that is also usable as a JSON key/value.
 284 type URL url.URL
 285
 286 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 287 // used as a JSON key/value.
 288 func (su *URL) UnmarshalText(text []byte) error {
 289         u, err := url.Parse(string(text))
 290         if err == nil {
 291                 *su = URL(*u)
 292         }
 293         return err
 294 }
 295
 296 func (su URL) MarshalText() ([]byte, error) {
 297         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 298 }
 299
 300 func (su URL) String() string {
 301         return (*url.URL)(&su).String()
 302 }
 303
 304 type ServiceInstance struct {
 305         Rendezvous string `json:",omitempty"`
 306 }
 307
 308 type PostgreSQL struct {
 309         Connection     PostgreSQLConnection
 310         ConnectionPool int
 311 }
 312
 313 type PostgreSQLConnection map[string]string
 314
 315 type RemoteCluster struct {
 316         Host          string
 317         Proxy         bool
 318         Scheme        string
 319         Insecure      bool
 320         ActivateUsers bool
 321 }
 322
 323 type InstanceType struct {
 324         Name            string
 325         ProviderType    string
 326         VCPUs           int
 327         RAM             ByteSize
 328         Scratch         ByteSize
 329         IncludedScratch ByteSize
 330         AddedScratch    ByteSize
 331         Price           float64
 332         Preemptible     bool
 333 }
 334
 335 type ContainersConfig struct {
 336         CloudVMs                    CloudVMsConfig
 337         CrunchRunCommand            string
 338         CrunchRunArgumentsList      []string
 339         DefaultKeepCacheRAM         ByteSize
 340         DispatchPrivateKey          string
 341         LogReuseDecisions           bool
 342         MaxComputeVMs               int
 343         MaxDispatchAttempts         int
 344         MaxRetryAttempts            int
 345         MinRetryPeriod              Duration
 346         ReserveExtraRAM             ByteSize
 347         StaleLockTimeout            Duration
 348         SupportedDockerImageFormats StringSet
 349         UsePreemptibleInstances     bool
 350
 351         JobsAPI struct {
 352                 Enable         string
 353                 GitInternalDir string
 354         }
 355         Logging struct {
 356                 MaxAge                       Duration
 357                 LogBytesPerEvent             int
 358                 LogSecondsBetweenEvents      int
 359                 LogThrottlePeriod            Duration
 360                 LogThrottleBytes             int
 361                 LogThrottleLines             int
 362                 LimitLogBytesPerJob          int
 363                 LogPartialLineThrottlePeriod Duration
 364                 LogUpdatePeriod              Duration
 365                 LogUpdateSize                ByteSize
 366         }
 367         SLURM struct {
 368                 PrioritySpread             int64
 369                 SbatchArgumentsList        []string
 370                 SbatchEnvironmentVariables map[string]string
 371                 Managed                    struct {
 372                         DNSServerConfDir       string
 373                         DNSServerConfTemplate  string
 374                         DNSServerReloadCommand string
 375                         DNSServerUpdateCommand string
 376                         ComputeNodeDomain      string
 377                         ComputeNodeNameservers StringSet
 378                         AssignNodeHostname     string
 379                 }
 380         }
 381 }
 382
 383 type CloudVMsConfig struct {
 384         Enable bool
 385
 386         BootProbeCommand     string
 387         ImageID              string
 388         MaxCloudOpsPerSecond int
 389         MaxProbesPerSecond   int
 390         PollInterval         Duration
 391         ProbeInterval        Duration
 392         SSHPort              string
 393         SyncInterval         Duration
 394         TimeoutBooting       Duration
 395         TimeoutIdle          Duration
 396         TimeoutProbe         Duration
 397         TimeoutShutdown      Duration
 398         TimeoutSignal        Duration
 399         TimeoutTERM          Duration
 400         ResourceTags         map[string]string
 401         TagKeyPrefix         string
 402
 403         Driver           string
 404         DriverParameters json.RawMessage
 405 }
 406
 407 type InstanceTypeMap map[string]InstanceType
 408
 409 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 410
 411 // UnmarshalJSON handles old config files that provide an array of
 412 // instance types instead of a hash.
 413 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 414         if len(data) > 0 && data[0] == '[' {
 415                 var arr []InstanceType
 416                 err := json.Unmarshal(data, &arr)
 417                 if err != nil {
 418                         return err
 419                 }
 420                 if len(arr) == 0 {
 421                         *it = nil
 422                         return nil
 423                 }
 424                 *it = make(map[string]InstanceType, len(arr))
 425                 for _, t := range arr {
 426                         if _, ok := (*it)[t.Name]; ok {
 427                                 return errDuplicateInstanceTypeName
 428                         }
 429                         if t.ProviderType == "" {
 430                                 t.ProviderType = t.Name
 431                         }
 432                         if t.Scratch == 0 {
 433                                 t.Scratch = t.IncludedScratch + t.AddedScratch
 434                         } else if t.AddedScratch == 0 {
 435                                 t.AddedScratch = t.Scratch - t.IncludedScratch
 436                         } else if t.IncludedScratch == 0 {
 437                                 t.IncludedScratch = t.Scratch - t.AddedScratch
 438                         }
 439
 440                         if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 441                                 return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 442                         }
 443                         (*it)[t.Name] = t
 444                 }
 445                 return nil
 446         }
 447         var hash map[string]InstanceType
 448         err := json.Unmarshal(data, &hash)
 449         if err != nil {
 450                 return err
 451         }
 452         // Fill in Name field (and ProviderType field, if not
 453         // specified) using hash key.
 454         *it = InstanceTypeMap(hash)
 455         for name, t := range *it {
 456                 t.Name = name
 457                 if t.ProviderType == "" {
 458                         t.ProviderType = name
 459                 }
 460                 (*it)[name] = t
 461         }
 462         return nil
 463 }
 464
 465 type StringSet map[string]struct{}
 466
 467 // UnmarshalJSON handles old config files that provide an array of
 468 // instance types instead of a hash.
 469 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 470         if len(data) > 0 && data[0] == '[' {
 471                 var arr []string
 472                 err := json.Unmarshal(data, &arr)
 473                 if err != nil {
 474                         return err
 475                 }
 476                 if len(arr) == 0 {
 477                         *ss = nil
 478                         return nil
 479                 }
 480                 *ss = make(map[string]struct{}, len(arr))
 481                 for _, t := range arr {
 482                         (*ss)[t] = struct{}{}
 483                 }
 484                 return nil
 485         }
 486         var hash map[string]struct{}
 487         err := json.Unmarshal(data, &hash)
 488         if err != nil {
 489                 return err
 490         }
 491         *ss = make(map[string]struct{}, len(hash))
 492         for t, _ := range hash {
 493                 (*ss)[t] = struct{}{}
 494         }
 495
 496         return nil
 497 }
 498
 499 type ServiceName string
 500
 501 const (
 502         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 503         ServiceNameController    ServiceName = "arvados-controller"
 504         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 505         ServiceNameHealth        ServiceName = "arvados-health"
 506         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 507         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 508         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 509         ServiceNameWebsocket     ServiceName = "arvados-ws"
 510         ServiceNameKeepbalance   ServiceName = "keep-balance"
 511         ServiceNameKeepweb       ServiceName = "keep-web"
 512         ServiceNameKeepproxy     ServiceName = "keepproxy"
 513         ServiceNameKeepstore     ServiceName = "keepstore"
 514 )
 515
 516 // Map returns all services as a map, suitable for iterating over all
 517 // services or looking up a service by name.
 518 func (svcs Services) Map() map[ServiceName]Service {
 519         return map[ServiceName]Service{
 520                 ServiceNameRailsAPI:      svcs.RailsAPI,
 521                 ServiceNameController:    svcs.Controller,
 522                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 523                 ServiceNameHealth:        svcs.Health,
 524                 ServiceNameNodemanager:   svcs.Nodemanager,
 525                 ServiceNameWorkbench1:    svcs.Workbench1,
 526                 ServiceNameWorkbench2:    svcs.Workbench2,
 527                 ServiceNameWebsocket:     svcs.Websocket,
 528                 ServiceNameKeepbalance:   svcs.Keepbalance,
 529                 ServiceNameKeepweb:       svcs.WebDAV,
 530                 ServiceNameKeepproxy:     svcs.Keepproxy,
 531                 ServiceNameKeepstore:     svcs.Keepstore,
 532         }
 533 }