sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.curoverse.com/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters map[string]Cluster
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         if cc, ok := sc.Clusters[clusterID]; !ok {
  53                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  54         } else {
  55                 cc.ClusterID = clusterID
  56                 return &cc, nil
  57         }
  58 }
  59
  60 type Cluster struct {
  61         ClusterID       string `json:"-"`
  62         ManagementToken string
  63         SystemRootToken string
  64         Services        Services
  65         InstanceTypes   InstanceTypeMap
  66         Containers      ContainersConfig
  67         RemoteClusters  map[string]RemoteCluster
  68         PostgreSQL      PostgreSQL
  69
  70         API struct {
  71                 AsyncPermissionsUpdateInterval Duration
  72                 DisabledAPIs                   []string
  73                 MaxIndexDatabaseRead           int
  74                 MaxItemsPerResponse            int
  75                 MaxRequestAmplification        int
  76                 MaxRequestSize                 int
  77                 RailsSessionSecretToken        string
  78                 RequestTimeout                 Duration
  79         }
  80         AuditLogs struct {
  81                 MaxAge             Duration
  82                 MaxDeleteBatch     int
  83                 UnloggedAttributes []string
  84         }
  85         Collections struct {
  86                 BlobSigning          bool
  87                 BlobSigningKey       string
  88                 BlobSigningTTL       Duration
  89                 CollectionVersioning bool
  90                 DefaultTrashLifetime Duration
  91                 DefaultReplication   int
  92                 ManagedProperties    map[string]struct {
  93                         Value     interface{}
  94                         Function  string
  95                         Protected bool
  96                 }
  97                 PreserveVersionIfIdle Duration
  98                 TrashSweepInterval    Duration
  99                 TrustAllContent       bool
 100         }
 101         Git struct {
 102                 Repositories string
 103         }
 104         Login struct {
 105                 ProviderAppSecret string
 106                 ProviderAppID     string
 107         }
 108         Mail struct {
 109                 MailchimpAPIKey                string
 110                 MailchimpListID                string
 111                 SendUserSetupNotificationEmail bool
 112                 IssueReporterEmailFrom         string
 113                 IssueReporterEmailTo           string
 114                 SupportEmailAddress            string
 115                 EmailFrom                      string
 116         }
 117         SystemLogs struct {
 118                 LogLevel                string
 119                 Format                  string
 120                 MaxRequestLogParamsSize int
 121         }
 122         TLS struct {
 123                 Certificate string
 124                 Key         string
 125                 Insecure    bool
 126         }
 127         Users struct {
 128                 AnonymousUserToken                    string
 129                 AdminNotifierEmailFrom                string
 130                 AutoAdminFirstUser                    bool
 131                 AutoAdminUserWithEmail                string
 132                 AutoSetupNewUsers                     bool
 133                 AutoSetupNewUsersWithRepository       bool
 134                 AutoSetupNewUsersWithVmUUID           string
 135                 AutoSetupUsernameBlacklist            []string
 136                 EmailSubjectPrefix                    string
 137                 NewInactiveUserNotificationRecipients []string
 138                 NewUserNotificationRecipients         []string
 139                 NewUsersAreActive                     bool
 140                 UserNotifierEmailFrom                 string
 141                 UserProfileNotificationAddress        string
 142         }
 143         Workbench struct {
 144                 ActivationContactLink            string
 145                 APIClientConnectTimeout          Duration
 146                 APIClientReceiveTimeout          Duration
 147                 APIResponseCompression           bool
 148                 ApplicationMimetypesWithViewIcon map[string]struct{}
 149                 ArvadosDocsite                   string
 150                 ArvadosPublicDataDocURL          string
 151                 DefaultOpenIdPrefix              string
 152                 EnableGettingStartedPopup        bool
 153                 EnablePublicProjectsPage         bool
 154                 FileViewersConfigURL             string
 155                 LogViewerMaxBytes                ByteSize
 156                 MultiSiteSearch                  string
 157                 ProfilingEnabled                 bool
 158                 Repositories                     bool
 159                 RepositoryCache                  string
 160                 RunningJobLogRecordsToFetch      int
 161                 SecretKeyBase                    string
 162                 ShowRecentCollectionsOnDashboard bool
 163                 ShowUserAgreementInline          bool
 164                 ShowUserNotifications            bool
 165                 SiteName                         string
 166                 Theme                            string
 167                 UserProfileFormFields            map[string]struct {
 168                         Type                 string
 169                         FormFieldTitle       string
 170                         FormFieldDescription string
 171                         Required             bool
 172                         Position             int
 173                         Options              map[string]struct{}
 174                 }
 175                 UserProfileFormMessage string
 176                 VocabularyURL          string
 177         }
 178
 179         EnableBetaController14287 bool
 180 }
 181
 182 type Services struct {
 183         Composer       Service
 184         Controller     Service
 185         DispatchCloud  Service
 186         GitHTTP        Service
 187         GitSSH         Service
 188         Health         Service
 189         Keepbalance    Service
 190         Keepproxy      Service
 191         Keepstore      Service
 192         Nodemanager    Service
 193         RailsAPI       Service
 194         SSO            Service
 195         WebDAVDownload Service
 196         WebDAV         Service
 197         WebShell       Service
 198         Websocket      Service
 199         Workbench1     Service
 200         Workbench2     Service
 201 }
 202
 203 type Service struct {
 204         InternalURLs map[URL]ServiceInstance
 205         ExternalURL  URL
 206 }
 207
 208 // URL is a url.URL that is also usable as a JSON key/value.
 209 type URL url.URL
 210
 211 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 212 // used as a JSON key/value.
 213 func (su *URL) UnmarshalText(text []byte) error {
 214         u, err := url.Parse(string(text))
 215         if err == nil {
 216                 *su = URL(*u)
 217         }
 218         return err
 219 }
 220
 221 func (su URL) MarshalText() ([]byte, error) {
 222         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 223 }
 224
 225 type ServiceInstance struct{}
 226
 227 type PostgreSQL struct {
 228         Connection     PostgreSQLConnection
 229         ConnectionPool int
 230 }
 231
 232 type PostgreSQLConnection map[string]string
 233
 234 type RemoteCluster struct {
 235         Host          string
 236         Proxy         bool
 237         Scheme        string
 238         Insecure      bool
 239         ActivateUsers bool
 240 }
 241
 242 type InstanceType struct {
 243         Name            string
 244         ProviderType    string
 245         VCPUs           int
 246         RAM             ByteSize
 247         Scratch         ByteSize
 248         IncludedScratch ByteSize
 249         AddedScratch    ByteSize
 250         Price           float64
 251         Preemptible     bool
 252 }
 253
 254 type ContainersConfig struct {
 255         CloudVMs                    CloudVMsConfig
 256         DefaultKeepCacheRAM         ByteSize
 257         DispatchPrivateKey          string
 258         LogReuseDecisions           bool
 259         MaxComputeVMs               int
 260         MaxDispatchAttempts         int
 261         MaxRetryAttempts            int
 262         StaleLockTimeout            Duration
 263         SupportedDockerImageFormats []string
 264         UsePreemptibleInstances     bool
 265
 266         JobsAPI struct {
 267                 Enable                  string
 268                 GitInternalDir          string
 269                 DefaultDockerImage      string
 270                 CrunchJobWrapper        string
 271                 CrunchJobUser           string
 272                 CrunchRefreshTrigger    string
 273                 ReuseJobIfOutputsDiffer bool
 274         }
 275         Logging struct {
 276                 MaxAge                       Duration
 277                 LogBytesPerEvent             int
 278                 LogSecondsBetweenEvents      int
 279                 LogThrottlePeriod            Duration
 280                 LogThrottleBytes             int
 281                 LogThrottleLines             int
 282                 LimitLogBytesPerJob          int
 283                 LogPartialLineThrottlePeriod Duration
 284                 LogUpdatePeriod              Duration
 285                 LogUpdateSize                ByteSize
 286         }
 287         SLURM struct {
 288                 Managed struct {
 289                         DNSServerConfDir       string
 290                         DNSServerConfTemplate  string
 291                         DNSServerReloadCommand string
 292                         DNSServerUpdateCommand string
 293                         ComputeNodeDomain      string
 294                         ComputeNodeNameservers []string
 295                         AssignNodeHostname     string
 296                 }
 297         }
 298 }
 299
 300 type CloudVMsConfig struct {
 301         Enable bool
 302
 303         BootProbeCommand     string
 304         ImageID              string
 305         MaxCloudOpsPerSecond int
 306         MaxProbesPerSecond   int
 307         PollInterval         Duration
 308         ProbeInterval        Duration
 309         SSHPort              string
 310         SyncInterval         Duration
 311         TimeoutBooting       Duration
 312         TimeoutIdle          Duration
 313         TimeoutProbe         Duration
 314         TimeoutShutdown      Duration
 315         TimeoutSignal        Duration
 316         TimeoutTERM          Duration
 317         ResourceTags         map[string]string
 318         TagKeyPrefix         string
 319
 320         Driver           string
 321         DriverParameters json.RawMessage
 322 }
 323
 324 type InstanceTypeMap map[string]InstanceType
 325
 326 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 327
 328 // UnmarshalJSON handles old config files that provide an array of
 329 // instance types instead of a hash.
 330 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 331         if len(data) > 0 && data[0] == '[' {
 332                 var arr []InstanceType
 333                 err := json.Unmarshal(data, &arr)
 334                 if err != nil {
 335                         return err
 336                 }
 337                 if len(arr) == 0 {
 338                         *it = nil
 339                         return nil
 340                 }
 341                 *it = make(map[string]InstanceType, len(arr))
 342                 for _, t := range arr {
 343                         if _, ok := (*it)[t.Name]; ok {
 344                                 return errDuplicateInstanceTypeName
 345                         }
 346                         if t.ProviderType == "" {
 347                                 t.ProviderType = t.Name
 348                         }
 349                         if t.Scratch == 0 {
 350                                 t.Scratch = t.IncludedScratch + t.AddedScratch
 351                         } else if t.AddedScratch == 0 {
 352                                 t.AddedScratch = t.Scratch - t.IncludedScratch
 353                         } else if t.IncludedScratch == 0 {
 354                                 t.IncludedScratch = t.Scratch - t.AddedScratch
 355                         }
 356
 357                         if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 358                                 return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 359                         }
 360                         (*it)[t.Name] = t
 361                 }
 362                 return nil
 363         }
 364         var hash map[string]InstanceType
 365         err := json.Unmarshal(data, &hash)
 366         if err != nil {
 367                 return err
 368         }
 369         // Fill in Name field (and ProviderType field, if not
 370         // specified) using hash key.
 371         *it = InstanceTypeMap(hash)
 372         for name, t := range *it {
 373                 t.Name = name
 374                 if t.ProviderType == "" {
 375                         t.ProviderType = name
 376                 }
 377                 (*it)[name] = t
 378         }
 379         return nil
 380 }
 381
 382 type ServiceName string
 383
 384 const (
 385         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 386         ServiceNameController    ServiceName = "arvados-controller"
 387         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 388         ServiceNameHealth        ServiceName = "arvados-health"
 389         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 390         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 391         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 392         ServiceNameWebsocket     ServiceName = "arvados-ws"
 393         ServiceNameKeepbalance   ServiceName = "keep-balance"
 394         ServiceNameKeepweb       ServiceName = "keep-web"
 395         ServiceNameKeepproxy     ServiceName = "keepproxy"
 396         ServiceNameKeepstore     ServiceName = "keepstore"
 397 )
 398
 399 // Map returns all services as a map, suitable for iterating over all
 400 // services or looking up a service by name.
 401 func (svcs Services) Map() map[ServiceName]Service {
 402         return map[ServiceName]Service{
 403                 ServiceNameRailsAPI:      svcs.RailsAPI,
 404                 ServiceNameController:    svcs.Controller,
 405                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 406                 ServiceNameHealth:        svcs.Health,
 407                 ServiceNameNodemanager:   svcs.Nodemanager,
 408                 ServiceNameWorkbench1:    svcs.Workbench1,
 409                 ServiceNameWorkbench2:    svcs.Workbench2,
 410                 ServiceNameWebsocket:     svcs.Websocket,
 411                 ServiceNameKeepbalance:   svcs.Keepbalance,
 412                 ServiceNameKeepweb:       svcs.WebDAV,
 413                 ServiceNameKeepproxy:     svcs.Keepproxy,
 414                 ServiceNameKeepstore:     svcs.Keepstore,
 415         }
 416 }