sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.curoverse.com/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters map[string]Cluster
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         if cc, ok := sc.Clusters[clusterID]; !ok {
  53                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  54         } else {
  55                 cc.ClusterID = clusterID
  56                 return &cc, nil
  57         }
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxCollectionEntries int
  64         MaxCollectionBytes   int64
  65         MaxPermissionEntries int
  66         MaxUUIDEntries       int
  67 }
  68 type Cluster struct {
  69         ClusterID       string `json:"-"`
  70         ManagementToken string
  71         SystemRootToken string
  72         Services        Services
  73         InstanceTypes   InstanceTypeMap
  74         Containers      ContainersConfig
  75         RemoteClusters  map[string]RemoteCluster
  76         PostgreSQL      PostgreSQL
  77
  78         API struct {
  79                 AsyncPermissionsUpdateInterval Duration
  80                 DisabledAPIs                   StringSet
  81                 MaxIndexDatabaseRead           int
  82                 MaxItemsPerResponse            int
  83                 MaxRequestAmplification        int
  84                 MaxRequestSize                 int
  85                 RailsSessionSecretToken        string
  86                 RequestTimeout                 Duration
  87                 SendTimeout                    Duration
  88                 WebsocketClientEventQueue      int
  89                 WebsocketServerEventQueue      int
  90         }
  91         AuditLogs struct {
  92                 MaxAge             Duration
  93                 MaxDeleteBatch     int
  94                 UnloggedAttributes StringSet
  95         }
  96         Collections struct {
  97                 BlobSigning          bool
  98                 BlobSigningKey       string
  99                 BlobSigningTTL       Duration
 100                 CollectionVersioning bool
 101                 DefaultTrashLifetime Duration
 102                 DefaultReplication   int
 103                 ManagedProperties    map[string]struct {
 104                         Value     interface{}
 105                         Function  string
 106                         Protected bool
 107                 }
 108                 PreserveVersionIfIdle Duration
 109                 TrashSweepInterval    Duration
 110                 TrustAllContent       bool
 111
 112                 WebDAVCache WebDAVCacheConfig
 113         }
 114         Git struct {
 115                 Repositories string
 116         }
 117         Login struct {
 118                 ProviderAppSecret string
 119                 ProviderAppID     string
 120         }
 121         Mail struct {
 122                 MailchimpAPIKey                string
 123                 MailchimpListID                string
 124                 SendUserSetupNotificationEmail bool
 125                 IssueReporterEmailFrom         string
 126                 IssueReporterEmailTo           string
 127                 SupportEmailAddress            string
 128                 EmailFrom                      string
 129         }
 130         SystemLogs struct {
 131                 LogLevel                string
 132                 Format                  string
 133                 MaxRequestLogParamsSize int
 134         }
 135         TLS struct {
 136                 Certificate string
 137                 Key         string
 138                 Insecure    bool
 139         }
 140         Users struct {
 141                 AnonymousUserToken                    string
 142                 AdminNotifierEmailFrom                string
 143                 AutoAdminFirstUser                    bool
 144                 AutoAdminUserWithEmail                string
 145                 AutoSetupNewUsers                     bool
 146                 AutoSetupNewUsersWithRepository       bool
 147                 AutoSetupNewUsersWithVmUUID           string
 148                 AutoSetupUsernameBlacklist            StringSet
 149                 EmailSubjectPrefix                    string
 150                 NewInactiveUserNotificationRecipients StringSet
 151                 NewUserNotificationRecipients         StringSet
 152                 NewUsersAreActive                     bool
 153                 UserNotifierEmailFrom                 string
 154                 UserProfileNotificationAddress        string
 155         }
 156         Workbench struct {
 157                 ActivationContactLink            string
 158                 APIClientConnectTimeout          Duration
 159                 APIClientReceiveTimeout          Duration
 160                 APIResponseCompression           bool
 161                 ApplicationMimetypesWithViewIcon StringSet
 162                 ArvadosDocsite                   string
 163                 ArvadosPublicDataDocURL          string
 164                 DefaultOpenIdPrefix              string
 165                 EnableGettingStartedPopup        bool
 166                 EnablePublicProjectsPage         bool
 167                 FileViewersConfigURL             string
 168                 LogViewerMaxBytes                ByteSize
 169                 MultiSiteSearch                  string
 170                 ProfilingEnabled                 bool
 171                 Repositories                     bool
 172                 RepositoryCache                  string
 173                 RunningJobLogRecordsToFetch      int
 174                 SecretKeyBase                    string
 175                 ShowRecentCollectionsOnDashboard bool
 176                 ShowUserAgreementInline          bool
 177                 ShowUserNotifications            bool
 178                 SiteName                         string
 179                 Theme                            string
 180                 UserProfileFormFields            map[string]struct {
 181                         Type                 string
 182                         FormFieldTitle       string
 183                         FormFieldDescription string
 184                         Required             bool
 185                         Position             int
 186                         Options              map[string]struct{}
 187                 }
 188                 UserProfileFormMessage string
 189                 VocabularyURL          string
 190         }
 191
 192         EnableBetaController14287 bool
 193 }
 194
 195 type Services struct {
 196         Composer       Service
 197         Controller     Service
 198         DispatchCloud  Service
 199         GitHTTP        Service
 200         GitSSH         Service
 201         Health         Service
 202         Keepbalance    Service
 203         Keepproxy      Service
 204         Keepstore      Service
 205         Nodemanager    Service
 206         RailsAPI       Service
 207         SSO            Service
 208         WebDAVDownload Service
 209         WebDAV         Service
 210         WebShell       Service
 211         Websocket      Service
 212         Workbench1     Service
 213         Workbench2     Service
 214 }
 215
 216 type Service struct {
 217         InternalURLs map[URL]ServiceInstance
 218         ExternalURL  URL
 219 }
 220
 221 // URL is a url.URL that is also usable as a JSON key/value.
 222 type URL url.URL
 223
 224 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 225 // used as a JSON key/value.
 226 func (su *URL) UnmarshalText(text []byte) error {
 227         u, err := url.Parse(string(text))
 228         if err == nil {
 229                 *su = URL(*u)
 230         }
 231         return err
 232 }
 233
 234 func (su URL) MarshalText() ([]byte, error) {
 235         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 236 }
 237
 238 type ServiceInstance struct{}
 239
 240 type PostgreSQL struct {
 241         Connection     PostgreSQLConnection
 242         ConnectionPool int
 243 }
 244
 245 type PostgreSQLConnection map[string]string
 246
 247 type RemoteCluster struct {
 248         Host          string
 249         Proxy         bool
 250         Scheme        string
 251         Insecure      bool
 252         ActivateUsers bool
 253 }
 254
 255 type InstanceType struct {
 256         Name            string
 257         ProviderType    string
 258         VCPUs           int
 259         RAM             ByteSize
 260         Scratch         ByteSize
 261         IncludedScratch ByteSize
 262         AddedScratch    ByteSize
 263         Price           float64
 264         Preemptible     bool
 265 }
 266
 267 type ContainersConfig struct {
 268         CloudVMs                    CloudVMsConfig
 269         CrunchRunCommand            string
 270         CrunchRunArgumentsList      []string
 271         DefaultKeepCacheRAM         ByteSize
 272         DispatchPrivateKey          string
 273         LogReuseDecisions           bool
 274         MaxComputeVMs               int
 275         MaxDispatchAttempts         int
 276         MaxRetryAttempts            int
 277         MinRetryPeriod              Duration
 278         ReserveExtraRAM             ByteSize
 279         StaleLockTimeout            Duration
 280         SupportedDockerImageFormats StringSet
 281         UsePreemptibleInstances     bool
 282
 283         JobsAPI struct {
 284                 Enable         string
 285                 GitInternalDir string
 286         }
 287         Logging struct {
 288                 MaxAge                       Duration
 289                 LogBytesPerEvent             int
 290                 LogSecondsBetweenEvents      int
 291                 LogThrottlePeriod            Duration
 292                 LogThrottleBytes             int
 293                 LogThrottleLines             int
 294                 LimitLogBytesPerJob          int
 295                 LogPartialLineThrottlePeriod Duration
 296                 LogUpdatePeriod              Duration
 297                 LogUpdateSize                ByteSize
 298         }
 299         SLURM struct {
 300                 PrioritySpread             int64
 301                 SbatchArgumentsList        []string
 302                 SbatchEnvironmentVariables map[string]string
 303                 Managed                    struct {
 304                         DNSServerConfDir       string
 305                         DNSServerConfTemplate  string
 306                         DNSServerReloadCommand string
 307                         DNSServerUpdateCommand string
 308                         ComputeNodeDomain      string
 309                         ComputeNodeNameservers StringSet
 310                         AssignNodeHostname     string
 311                 }
 312         }
 313 }
 314
 315 type CloudVMsConfig struct {
 316         Enable bool
 317
 318         BootProbeCommand     string
 319         ImageID              string
 320         MaxCloudOpsPerSecond int
 321         MaxProbesPerSecond   int
 322         PollInterval         Duration
 323         ProbeInterval        Duration
 324         SSHPort              string
 325         SyncInterval         Duration
 326         TimeoutBooting       Duration
 327         TimeoutIdle          Duration
 328         TimeoutProbe         Duration
 329         TimeoutShutdown      Duration
 330         TimeoutSignal        Duration
 331         TimeoutTERM          Duration
 332         ResourceTags         map[string]string
 333         TagKeyPrefix         string
 334
 335         Driver           string
 336         DriverParameters json.RawMessage
 337 }
 338
 339 type InstanceTypeMap map[string]InstanceType
 340
 341 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 342
 343 // UnmarshalJSON handles old config files that provide an array of
 344 // instance types instead of a hash.
 345 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 346         if len(data) > 0 && data[0] == '[' {
 347                 var arr []InstanceType
 348                 err := json.Unmarshal(data, &arr)
 349                 if err != nil {
 350                         return err
 351                 }
 352                 if len(arr) == 0 {
 353                         *it = nil
 354                         return nil
 355                 }
 356                 *it = make(map[string]InstanceType, len(arr))
 357                 for _, t := range arr {
 358                         if _, ok := (*it)[t.Name]; ok {
 359                                 return errDuplicateInstanceTypeName
 360                         }
 361                         if t.ProviderType == "" {
 362                                 t.ProviderType = t.Name
 363                         }
 364                         if t.Scratch == 0 {
 365                                 t.Scratch = t.IncludedScratch + t.AddedScratch
 366                         } else if t.AddedScratch == 0 {
 367                                 t.AddedScratch = t.Scratch - t.IncludedScratch
 368                         } else if t.IncludedScratch == 0 {
 369                                 t.IncludedScratch = t.Scratch - t.AddedScratch
 370                         }
 371
 372                         if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 373                                 return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 374                         }
 375                         (*it)[t.Name] = t
 376                 }
 377                 return nil
 378         }
 379         var hash map[string]InstanceType
 380         err := json.Unmarshal(data, &hash)
 381         if err != nil {
 382                 return err
 383         }
 384         // Fill in Name field (and ProviderType field, if not
 385         // specified) using hash key.
 386         *it = InstanceTypeMap(hash)
 387         for name, t := range *it {
 388                 t.Name = name
 389                 if t.ProviderType == "" {
 390                         t.ProviderType = name
 391                 }
 392                 (*it)[name] = t
 393         }
 394         return nil
 395 }
 396
 397 type StringSet map[string]struct{}
 398
 399 // UnmarshalJSON handles old config files that provide an array of
 400 // instance types instead of a hash.
 401 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 402         if len(data) > 0 && data[0] == '[' {
 403                 var arr []string
 404                 err := json.Unmarshal(data, &arr)
 405                 if err != nil {
 406                         return err
 407                 }
 408                 if len(arr) == 0 {
 409                         *ss = nil
 410                         return nil
 411                 }
 412                 *ss = make(map[string]struct{}, len(arr))
 413                 for _, t := range arr {
 414                         (*ss)[t] = struct{}{}
 415                 }
 416                 return nil
 417         }
 418         var hash map[string]struct{}
 419         err := json.Unmarshal(data, &hash)
 420         if err != nil {
 421                 return err
 422         }
 423         *ss = make(map[string]struct{}, len(hash))
 424         for t, _ := range hash {
 425                 (*ss)[t] = struct{}{}
 426         }
 427
 428         return nil
 429 }
 430
 431 type ServiceName string
 432
 433 const (
 434         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 435         ServiceNameController    ServiceName = "arvados-controller"
 436         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 437         ServiceNameHealth        ServiceName = "arvados-health"
 438         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 439         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 440         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 441         ServiceNameWebsocket     ServiceName = "arvados-ws"
 442         ServiceNameKeepbalance   ServiceName = "keep-balance"
 443         ServiceNameKeepweb       ServiceName = "keep-web"
 444         ServiceNameKeepproxy     ServiceName = "keepproxy"
 445         ServiceNameKeepstore     ServiceName = "keepstore"
 446 )
 447
 448 // Map returns all services as a map, suitable for iterating over all
 449 // services or looking up a service by name.
 450 func (svcs Services) Map() map[ServiceName]Service {
 451         return map[ServiceName]Service{
 452                 ServiceNameRailsAPI:      svcs.RailsAPI,
 453                 ServiceNameController:    svcs.Controller,
 454                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 455                 ServiceNameHealth:        svcs.Health,
 456                 ServiceNameNodemanager:   svcs.Nodemanager,
 457                 ServiceNameWorkbench1:    svcs.Workbench1,
 458                 ServiceNameWorkbench2:    svcs.Workbench2,
 459                 ServiceNameWebsocket:     svcs.Websocket,
 460                 ServiceNameKeepbalance:   svcs.Keepbalance,
 461                 ServiceNameKeepweb:       svcs.WebDAV,
 462                 ServiceNameKeepproxy:     svcs.Keepproxy,
 463                 ServiceNameKeepstore:     svcs.Keepstore,
 464         }
 465 }