sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.curoverse.com/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters map[string]Cluster
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         if cc, ok := sc.Clusters[clusterID]; !ok {
  53                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  54         } else {
  55                 cc.ClusterID = clusterID
  56                 return &cc, nil
  57         }
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxBlockEntries      int
  64         MaxCollectionEntries int
  65         MaxCollectionBytes   int64
  66         MaxPermissionEntries int
  67         MaxUUIDEntries       int
  68 }
  69 type Cluster struct {
  70         ClusterID       string `json:"-"`
  71         ManagementToken string
  72         SystemRootToken string
  73         Services        Services
  74         InstanceTypes   InstanceTypeMap
  75         Containers      ContainersConfig
  76         RemoteClusters  map[string]RemoteCluster
  77         PostgreSQL      PostgreSQL
  78
  79         API struct {
  80                 AsyncPermissionsUpdateInterval Duration
  81                 DisabledAPIs                   StringSet
  82                 MaxIndexDatabaseRead           int
  83                 MaxItemsPerResponse            int
  84                 MaxRequestAmplification        int
  85                 MaxRequestSize                 int
  86                 RailsSessionSecretToken        string
  87                 RequestTimeout                 Duration
  88                 SendTimeout                    Duration
  89                 WebsocketClientEventQueue      int
  90                 WebsocketServerEventQueue      int
  91         }
  92         AuditLogs struct {
  93                 MaxAge             Duration
  94                 MaxDeleteBatch     int
  95                 UnloggedAttributes StringSet
  96         }
  97         Collections struct {
  98                 BlobSigning          bool
  99                 BlobSigningKey       string
 100                 BlobSigningTTL       Duration
 101                 CollectionVersioning bool
 102                 DefaultTrashLifetime Duration
 103                 DefaultReplication   int
 104                 ManagedProperties    map[string]struct {
 105                         Value     interface{}
 106                         Function  string
 107                         Protected bool
 108                 }
 109                 PreserveVersionIfIdle Duration
 110                 TrashSweepInterval    Duration
 111                 TrustAllContent       bool
 112
 113                 WebDAVCache WebDAVCacheConfig
 114         }
 115         Git struct {
 116                 Repositories string
 117         }
 118         Login struct {
 119                 ProviderAppSecret string
 120                 ProviderAppID     string
 121         }
 122         Mail struct {
 123                 MailchimpAPIKey                string
 124                 MailchimpListID                string
 125                 SendUserSetupNotificationEmail bool
 126                 IssueReporterEmailFrom         string
 127                 IssueReporterEmailTo           string
 128                 SupportEmailAddress            string
 129                 EmailFrom                      string
 130         }
 131         SystemLogs struct {
 132                 LogLevel                string
 133                 Format                  string
 134                 MaxRequestLogParamsSize int
 135         }
 136         TLS struct {
 137                 Certificate string
 138                 Key         string
 139                 Insecure    bool
 140         }
 141         Users struct {
 142                 AnonymousUserToken                    string
 143                 AdminNotifierEmailFrom                string
 144                 AutoAdminFirstUser                    bool
 145                 AutoAdminUserWithEmail                string
 146                 AutoSetupNewUsers                     bool
 147                 AutoSetupNewUsersWithRepository       bool
 148                 AutoSetupNewUsersWithVmUUID           string
 149                 AutoSetupUsernameBlacklist            StringSet
 150                 EmailSubjectPrefix                    string
 151                 NewInactiveUserNotificationRecipients StringSet
 152                 NewUserNotificationRecipients         StringSet
 153                 NewUsersAreActive                     bool
 154                 UserNotifierEmailFrom                 string
 155                 UserProfileNotificationAddress        string
 156         }
 157         Workbench struct {
 158                 ActivationContactLink            string
 159                 APIClientConnectTimeout          Duration
 160                 APIClientReceiveTimeout          Duration
 161                 APIResponseCompression           bool
 162                 ApplicationMimetypesWithViewIcon StringSet
 163                 ArvadosDocsite                   string
 164                 ArvadosPublicDataDocURL          string
 165                 DefaultOpenIdPrefix              string
 166                 EnableGettingStartedPopup        bool
 167                 EnablePublicProjectsPage         bool
 168                 FileViewersConfigURL             string
 169                 LogViewerMaxBytes                ByteSize
 170                 MultiSiteSearch                  string
 171                 ProfilingEnabled                 bool
 172                 Repositories                     bool
 173                 RepositoryCache                  string
 174                 RunningJobLogRecordsToFetch      int
 175                 SecretKeyBase                    string
 176                 ShowRecentCollectionsOnDashboard bool
 177                 ShowUserAgreementInline          bool
 178                 ShowUserNotifications            bool
 179                 SiteName                         string
 180                 Theme                            string
 181                 UserProfileFormFields            map[string]struct {
 182                         Type                 string
 183                         FormFieldTitle       string
 184                         FormFieldDescription string
 185                         Required             bool
 186                         Position             int
 187                         Options              map[string]struct{}
 188                 }
 189                 UserProfileFormMessage string
 190                 VocabularyURL          string
 191         }
 192
 193         EnableBetaController14287 bool
 194 }
 195
 196 type Services struct {
 197         Composer       Service
 198         Controller     Service
 199         DispatchCloud  Service
 200         GitHTTP        Service
 201         GitSSH         Service
 202         Health         Service
 203         Keepbalance    Service
 204         Keepproxy      Service
 205         Keepstore      Service
 206         Nodemanager    Service
 207         RailsAPI       Service
 208         SSO            Service
 209         WebDAVDownload Service
 210         WebDAV         Service
 211         WebShell       Service
 212         Websocket      Service
 213         Workbench1     Service
 214         Workbench2     Service
 215 }
 216
 217 type Service struct {
 218         InternalURLs map[URL]ServiceInstance
 219         ExternalURL  URL
 220 }
 221
 222 // URL is a url.URL that is also usable as a JSON key/value.
 223 type URL url.URL
 224
 225 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 226 // used as a JSON key/value.
 227 func (su *URL) UnmarshalText(text []byte) error {
 228         u, err := url.Parse(string(text))
 229         if err == nil {
 230                 *su = URL(*u)
 231         }
 232         return err
 233 }
 234
 235 func (su URL) MarshalText() ([]byte, error) {
 236         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 237 }
 238
 239 type ServiceInstance struct{}
 240
 241 type PostgreSQL struct {
 242         Connection     PostgreSQLConnection
 243         ConnectionPool int
 244 }
 245
 246 type PostgreSQLConnection map[string]string
 247
 248 type RemoteCluster struct {
 249         Host          string
 250         Proxy         bool
 251         Scheme        string
 252         Insecure      bool
 253         ActivateUsers bool
 254 }
 255
 256 type InstanceType struct {
 257         Name            string
 258         ProviderType    string
 259         VCPUs           int
 260         RAM             ByteSize
 261         Scratch         ByteSize
 262         IncludedScratch ByteSize
 263         AddedScratch    ByteSize
 264         Price           float64
 265         Preemptible     bool
 266 }
 267
 268 type ContainersConfig struct {
 269         CloudVMs                    CloudVMsConfig
 270         CrunchRunCommand            string
 271         CrunchRunArgumentsList      []string
 272         DefaultKeepCacheRAM         ByteSize
 273         DispatchPrivateKey          string
 274         LogReuseDecisions           bool
 275         MaxComputeVMs               int
 276         MaxDispatchAttempts         int
 277         MaxRetryAttempts            int
 278         MinRetryPeriod              Duration
 279         ReserveExtraRAM             ByteSize
 280         StaleLockTimeout            Duration
 281         SupportedDockerImageFormats StringSet
 282         UsePreemptibleInstances     bool
 283
 284         JobsAPI struct {
 285                 Enable         string
 286                 GitInternalDir string
 287         }
 288         Logging struct {
 289                 MaxAge                       Duration
 290                 LogBytesPerEvent             int
 291                 LogSecondsBetweenEvents      int
 292                 LogThrottlePeriod            Duration
 293                 LogThrottleBytes             int
 294                 LogThrottleLines             int
 295                 LimitLogBytesPerJob          int
 296                 LogPartialLineThrottlePeriod Duration
 297                 LogUpdatePeriod              Duration
 298                 LogUpdateSize                ByteSize
 299         }
 300         SLURM struct {
 301                 PrioritySpread             int64
 302                 SbatchArgumentsList        []string
 303                 SbatchEnvironmentVariables map[string]string
 304                 Managed                    struct {
 305                         DNSServerConfDir       string
 306                         DNSServerConfTemplate  string
 307                         DNSServerReloadCommand string
 308                         DNSServerUpdateCommand string
 309                         ComputeNodeDomain      string
 310                         ComputeNodeNameservers StringSet
 311                         AssignNodeHostname     string
 312                 }
 313         }
 314 }
 315
 316 type CloudVMsConfig struct {
 317         Enable bool
 318
 319         BootProbeCommand     string
 320         ImageID              string
 321         MaxCloudOpsPerSecond int
 322         MaxProbesPerSecond   int
 323         PollInterval         Duration
 324         ProbeInterval        Duration
 325         SSHPort              string
 326         SyncInterval         Duration
 327         TimeoutBooting       Duration
 328         TimeoutIdle          Duration
 329         TimeoutProbe         Duration
 330         TimeoutShutdown      Duration
 331         TimeoutSignal        Duration
 332         TimeoutTERM          Duration
 333         ResourceTags         map[string]string
 334         TagKeyPrefix         string
 335
 336         Driver           string
 337         DriverParameters json.RawMessage
 338 }
 339
 340 type InstanceTypeMap map[string]InstanceType
 341
 342 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 343
 344 // UnmarshalJSON handles old config files that provide an array of
 345 // instance types instead of a hash.
 346 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 347         if len(data) > 0 && data[0] == '[' {
 348                 var arr []InstanceType
 349                 err := json.Unmarshal(data, &arr)
 350                 if err != nil {
 351                         return err
 352                 }
 353                 if len(arr) == 0 {
 354                         *it = nil
 355                         return nil
 356                 }
 357                 *it = make(map[string]InstanceType, len(arr))
 358                 for _, t := range arr {
 359                         if _, ok := (*it)[t.Name]; ok {
 360                                 return errDuplicateInstanceTypeName
 361                         }
 362                         if t.ProviderType == "" {
 363                                 t.ProviderType = t.Name
 364                         }
 365                         if t.Scratch == 0 {
 366                                 t.Scratch = t.IncludedScratch + t.AddedScratch
 367                         } else if t.AddedScratch == 0 {
 368                                 t.AddedScratch = t.Scratch - t.IncludedScratch
 369                         } else if t.IncludedScratch == 0 {
 370                                 t.IncludedScratch = t.Scratch - t.AddedScratch
 371                         }
 372
 373                         if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 374                                 return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 375                         }
 376                         (*it)[t.Name] = t
 377                 }
 378                 return nil
 379         }
 380         var hash map[string]InstanceType
 381         err := json.Unmarshal(data, &hash)
 382         if err != nil {
 383                 return err
 384         }
 385         // Fill in Name field (and ProviderType field, if not
 386         // specified) using hash key.
 387         *it = InstanceTypeMap(hash)
 388         for name, t := range *it {
 389                 t.Name = name
 390                 if t.ProviderType == "" {
 391                         t.ProviderType = name
 392                 }
 393                 (*it)[name] = t
 394         }
 395         return nil
 396 }
 397
 398 type StringSet map[string]struct{}
 399
 400 // UnmarshalJSON handles old config files that provide an array of
 401 // instance types instead of a hash.
 402 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 403         if len(data) > 0 && data[0] == '[' {
 404                 var arr []string
 405                 err := json.Unmarshal(data, &arr)
 406                 if err != nil {
 407                         return err
 408                 }
 409                 if len(arr) == 0 {
 410                         *ss = nil
 411                         return nil
 412                 }
 413                 *ss = make(map[string]struct{}, len(arr))
 414                 for _, t := range arr {
 415                         (*ss)[t] = struct{}{}
 416                 }
 417                 return nil
 418         }
 419         var hash map[string]struct{}
 420         err := json.Unmarshal(data, &hash)
 421         if err != nil {
 422                 return err
 423         }
 424         *ss = make(map[string]struct{}, len(hash))
 425         for t, _ := range hash {
 426                 (*ss)[t] = struct{}{}
 427         }
 428
 429         return nil
 430 }
 431
 432 type ServiceName string
 433
 434 const (
 435         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 436         ServiceNameController    ServiceName = "arvados-controller"
 437         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 438         ServiceNameHealth        ServiceName = "arvados-health"
 439         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 440         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 441         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 442         ServiceNameWebsocket     ServiceName = "arvados-ws"
 443         ServiceNameKeepbalance   ServiceName = "keep-balance"
 444         ServiceNameKeepweb       ServiceName = "keep-web"
 445         ServiceNameKeepproxy     ServiceName = "keepproxy"
 446         ServiceNameKeepstore     ServiceName = "keepstore"
 447 )
 448
 449 // Map returns all services as a map, suitable for iterating over all
 450 // services or looking up a service by name.
 451 func (svcs Services) Map() map[ServiceName]Service {
 452         return map[ServiceName]Service{
 453                 ServiceNameRailsAPI:      svcs.RailsAPI,
 454                 ServiceNameController:    svcs.Controller,
 455                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 456                 ServiceNameHealth:        svcs.Health,
 457                 ServiceNameNodemanager:   svcs.Nodemanager,
 458                 ServiceNameWorkbench1:    svcs.Workbench1,
 459                 ServiceNameWorkbench2:    svcs.Workbench2,
 460                 ServiceNameWebsocket:     svcs.Websocket,
 461                 ServiceNameKeepbalance:   svcs.Keepbalance,
 462                 ServiceNameKeepweb:       svcs.WebDAV,
 463                 ServiceNameKeepproxy:     svcs.Keepproxy,
 464                 ServiceNameKeepstore:     svcs.Keepstore,
 465         }
 466 }