sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.curoverse.com/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters map[string]Cluster
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         if cc, ok := sc.Clusters[clusterID]; !ok {
  53                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  54         } else {
  55                 cc.ClusterID = clusterID
  56                 return &cc, nil
  57         }
  58 }
  59
  60 type Cluster struct {
  61         ClusterID       string `json:"-"`
  62         ManagementToken string
  63         SystemRootToken string
  64         Services        Services
  65         InstanceTypes   InstanceTypeMap
  66         Containers      ContainersConfig
  67         RemoteClusters  map[string]RemoteCluster
  68         PostgreSQL      PostgreSQL
  69
  70         API struct {
  71                 AsyncPermissionsUpdateInterval Duration
  72                 DisabledAPIs                   StringSet
  73                 MaxIndexDatabaseRead           int
  74                 MaxItemsPerResponse            int
  75                 MaxRequestAmplification        int
  76                 MaxRequestSize                 int
  77                 RailsSessionSecretToken        string
  78                 RequestTimeout                 Duration
  79                 SendTimeout                    Duration
  80                 WebsocketClientEventQueue      int
  81                 WebsocketServerEventQueue      int
  82         }
  83         AuditLogs struct {
  84                 MaxAge             Duration
  85                 MaxDeleteBatch     int
  86                 UnloggedAttributes StringSet
  87         }
  88         Collections struct {
  89                 BlobSigning          bool
  90                 BlobSigningKey       string
  91                 BlobSigningTTL       Duration
  92                 CollectionVersioning bool
  93                 DefaultTrashLifetime Duration
  94                 DefaultReplication   int
  95                 ManagedProperties    map[string]struct {
  96                         Value     interface{}
  97                         Function  string
  98                         Protected bool
  99                 }
 100                 PreserveVersionIfIdle Duration
 101                 TrashSweepInterval    Duration
 102                 TrustAllContent       bool
 103         }
 104         Git struct {
 105                 Repositories string
 106         }
 107         Login struct {
 108                 ProviderAppSecret string
 109                 ProviderAppID     string
 110         }
 111         Mail struct {
 112                 MailchimpAPIKey                string
 113                 MailchimpListID                string
 114                 SendUserSetupNotificationEmail bool
 115                 IssueReporterEmailFrom         string
 116                 IssueReporterEmailTo           string
 117                 SupportEmailAddress            string
 118                 EmailFrom                      string
 119         }
 120         SystemLogs struct {
 121                 LogLevel                string
 122                 Format                  string
 123                 MaxRequestLogParamsSize int
 124         }
 125         TLS struct {
 126                 Certificate string
 127                 Key         string
 128                 Insecure    bool
 129         }
 130         Users struct {
 131                 AnonymousUserToken                    string
 132                 AdminNotifierEmailFrom                string
 133                 AutoAdminFirstUser                    bool
 134                 AutoAdminUserWithEmail                string
 135                 AutoSetupNewUsers                     bool
 136                 AutoSetupNewUsersWithRepository       bool
 137                 AutoSetupNewUsersWithVmUUID           string
 138                 AutoSetupUsernameBlacklist            StringSet
 139                 EmailSubjectPrefix                    string
 140                 NewInactiveUserNotificationRecipients StringSet
 141                 NewUserNotificationRecipients         StringSet
 142                 NewUsersAreActive                     bool
 143                 UserNotifierEmailFrom                 string
 144                 UserProfileNotificationAddress        string
 145         }
 146         Workbench struct {
 147                 ActivationContactLink            string
 148                 APIClientConnectTimeout          Duration
 149                 APIClientReceiveTimeout          Duration
 150                 APIResponseCompression           bool
 151                 ApplicationMimetypesWithViewIcon StringSet
 152                 ArvadosDocsite                   string
 153                 ArvadosPublicDataDocURL          string
 154                 DefaultOpenIdPrefix              string
 155                 EnableGettingStartedPopup        bool
 156                 EnablePublicProjectsPage         bool
 157                 FileViewersConfigURL             string
 158                 LogViewerMaxBytes                ByteSize
 159                 MultiSiteSearch                  string
 160                 ProfilingEnabled                 bool
 161                 Repositories                     bool
 162                 RepositoryCache                  string
 163                 RunningJobLogRecordsToFetch      int
 164                 SecretKeyBase                    string
 165                 ShowRecentCollectionsOnDashboard bool
 166                 ShowUserAgreementInline          bool
 167                 ShowUserNotifications            bool
 168                 SiteName                         string
 169                 Theme                            string
 170                 UserProfileFormFields            map[string]struct {
 171                         Type                 string
 172                         FormFieldTitle       string
 173                         FormFieldDescription string
 174                         Required             bool
 175                         Position             int
 176                         Options              map[string]struct{}
 177                 }
 178                 UserProfileFormMessage string
 179                 VocabularyURL          string
 180         }
 181
 182         EnableBetaController14287 bool
 183 }
 184
 185 type Services struct {
 186         Composer       Service
 187         Controller     Service
 188         DispatchCloud  Service
 189         GitHTTP        Service
 190         GitSSH         Service
 191         Health         Service
 192         Keepbalance    Service
 193         Keepproxy      Service
 194         Keepstore      Service
 195         Nodemanager    Service
 196         RailsAPI       Service
 197         SSO            Service
 198         WebDAVDownload Service
 199         WebDAV         Service
 200         WebShell       Service
 201         Websocket      Service
 202         Workbench1     Service
 203         Workbench2     Service
 204 }
 205
 206 type Service struct {
 207         InternalURLs map[URL]ServiceInstance
 208         ExternalURL  URL
 209 }
 210
 211 // URL is a url.URL that is also usable as a JSON key/value.
 212 type URL url.URL
 213
 214 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 215 // used as a JSON key/value.
 216 func (su *URL) UnmarshalText(text []byte) error {
 217         u, err := url.Parse(string(text))
 218         if err == nil {
 219                 *su = URL(*u)
 220         }
 221         return err
 222 }
 223
 224 func (su URL) MarshalText() ([]byte, error) {
 225         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 226 }
 227
 228 type ServiceInstance struct{}
 229
 230 type PostgreSQL struct {
 231         Connection     PostgreSQLConnection
 232         ConnectionPool int
 233 }
 234
 235 type PostgreSQLConnection map[string]string
 236
 237 type RemoteCluster struct {
 238         Host          string
 239         Proxy         bool
 240         Scheme        string
 241         Insecure      bool
 242         ActivateUsers bool
 243 }
 244
 245 type InstanceType struct {
 246         Name            string
 247         ProviderType    string
 248         VCPUs           int
 249         RAM             ByteSize
 250         Scratch         ByteSize
 251         IncludedScratch ByteSize
 252         AddedScratch    ByteSize
 253         Price           float64
 254         Preemptible     bool
 255 }
 256
 257 type ContainersConfig struct {
 258         CloudVMs                    CloudVMsConfig
 259         CrunchRunCommand            string
 260         CrunchRunArgumentsList      []string
 261         DefaultKeepCacheRAM         ByteSize
 262         DispatchPrivateKey          string
 263         LogReuseDecisions           bool
 264         MaxComputeVMs               int
 265         MaxDispatchAttempts         int
 266         MaxRetryAttempts            int
 267         MinRetryPeriod              Duration
 268         ReserveExtraRAM             ByteSize
 269         StaleLockTimeout            Duration
 270         SupportedDockerImageFormats StringSet
 271         UsePreemptibleInstances     bool
 272
 273         JobsAPI struct {
 274                 Enable         string
 275                 GitInternalDir string
 276         }
 277         Logging struct {
 278                 MaxAge                       Duration
 279                 LogBytesPerEvent             int
 280                 LogSecondsBetweenEvents      int
 281                 LogThrottlePeriod            Duration
 282                 LogThrottleBytes             int
 283                 LogThrottleLines             int
 284                 LimitLogBytesPerJob          int
 285                 LogPartialLineThrottlePeriod Duration
 286                 LogUpdatePeriod              Duration
 287                 LogUpdateSize                ByteSize
 288         }
 289         SLURM struct {
 290                 PrioritySpread             int64
 291                 SbatchArgumentsList        []string
 292                 SbatchEnvironmentVariables map[string]string
 293                 Managed                    struct {
 294                         DNSServerConfDir       string
 295                         DNSServerConfTemplate  string
 296                         DNSServerReloadCommand string
 297                         DNSServerUpdateCommand string
 298                         ComputeNodeDomain      string
 299                         ComputeNodeNameservers StringSet
 300                         AssignNodeHostname     string
 301                 }
 302         }
 303 }
 304
 305 type CloudVMsConfig struct {
 306         Enable bool
 307
 308         BootProbeCommand     string
 309         ImageID              string
 310         MaxCloudOpsPerSecond int
 311         MaxProbesPerSecond   int
 312         PollInterval         Duration
 313         ProbeInterval        Duration
 314         SSHPort              string
 315         SyncInterval         Duration
 316         TimeoutBooting       Duration
 317         TimeoutIdle          Duration
 318         TimeoutProbe         Duration
 319         TimeoutShutdown      Duration
 320         TimeoutSignal        Duration
 321         TimeoutTERM          Duration
 322         ResourceTags         map[string]string
 323         TagKeyPrefix         string
 324
 325         Driver           string
 326         DriverParameters json.RawMessage
 327 }
 328
 329 type InstanceTypeMap map[string]InstanceType
 330
 331 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 332
 333 // UnmarshalJSON handles old config files that provide an array of
 334 // instance types instead of a hash.
 335 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 336         if len(data) > 0 && data[0] == '[' {
 337                 var arr []InstanceType
 338                 err := json.Unmarshal(data, &arr)
 339                 if err != nil {
 340                         return err
 341                 }
 342                 if len(arr) == 0 {
 343                         *it = nil
 344                         return nil
 345                 }
 346                 *it = make(map[string]InstanceType, len(arr))
 347                 for _, t := range arr {
 348                         if _, ok := (*it)[t.Name]; ok {
 349                                 return errDuplicateInstanceTypeName
 350                         }
 351                         if t.ProviderType == "" {
 352                                 t.ProviderType = t.Name
 353                         }
 354                         if t.Scratch == 0 {
 355                                 t.Scratch = t.IncludedScratch + t.AddedScratch
 356                         } else if t.AddedScratch == 0 {
 357                                 t.AddedScratch = t.Scratch - t.IncludedScratch
 358                         } else if t.IncludedScratch == 0 {
 359                                 t.IncludedScratch = t.Scratch - t.AddedScratch
 360                         }
 361
 362                         if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 363                                 return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 364                         }
 365                         (*it)[t.Name] = t
 366                 }
 367                 return nil
 368         }
 369         var hash map[string]InstanceType
 370         err := json.Unmarshal(data, &hash)
 371         if err != nil {
 372                 return err
 373         }
 374         // Fill in Name field (and ProviderType field, if not
 375         // specified) using hash key.
 376         *it = InstanceTypeMap(hash)
 377         for name, t := range *it {
 378                 t.Name = name
 379                 if t.ProviderType == "" {
 380                         t.ProviderType = name
 381                 }
 382                 (*it)[name] = t
 383         }
 384         return nil
 385 }
 386
 387 type StringSet map[string]struct{}
 388
 389 // UnmarshalJSON handles old config files that provide an array of
 390 // instance types instead of a hash.
 391 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 392         if len(data) > 0 && data[0] == '[' {
 393                 var arr []string
 394                 err := json.Unmarshal(data, &arr)
 395                 if err != nil {
 396                         return err
 397                 }
 398                 if len(arr) == 0 {
 399                         *ss = nil
 400                         return nil
 401                 }
 402                 *ss = make(map[string]struct{}, len(arr))
 403                 for _, t := range arr {
 404                         (*ss)[t] = struct{}{}
 405                 }
 406                 return nil
 407         }
 408         var hash map[string]struct{}
 409         err := json.Unmarshal(data, &hash)
 410         if err != nil {
 411                 return err
 412         }
 413         *ss = make(map[string]struct{}, len(hash))
 414         for t, _ := range hash {
 415                 (*ss)[t] = struct{}{}
 416         }
 417
 418         return nil
 419 }
 420
 421 type ServiceName string
 422
 423 const (
 424         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 425         ServiceNameController    ServiceName = "arvados-controller"
 426         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 427         ServiceNameHealth        ServiceName = "arvados-health"
 428         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 429         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 430         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 431         ServiceNameWebsocket     ServiceName = "arvados-ws"
 432         ServiceNameKeepbalance   ServiceName = "keep-balance"
 433         ServiceNameKeepweb       ServiceName = "keep-web"
 434         ServiceNameKeepproxy     ServiceName = "keepproxy"
 435         ServiceNameKeepstore     ServiceName = "keepstore"
 436 )
 437
 438 // Map returns all services as a map, suitable for iterating over all
 439 // services or looking up a service by name.
 440 func (svcs Services) Map() map[ServiceName]Service {
 441         return map[ServiceName]Service{
 442                 ServiceNameRailsAPI:      svcs.RailsAPI,
 443                 ServiceNameController:    svcs.Controller,
 444                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 445                 ServiceNameHealth:        svcs.Health,
 446                 ServiceNameNodemanager:   svcs.Nodemanager,
 447                 ServiceNameWorkbench1:    svcs.Workbench1,
 448                 ServiceNameWorkbench2:    svcs.Workbench2,
 449                 ServiceNameWebsocket:     svcs.Websocket,
 450                 ServiceNameKeepbalance:   svcs.Keepbalance,
 451                 ServiceNameKeepweb:       svcs.WebDAV,
 452                 ServiceNameKeepproxy:     svcs.Keepproxy,
 453                 ServiceNameKeepstore:     svcs.Keepstore,
 454         }
 455 }