sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.curoverse.com/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters map[string]Cluster
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         if cc, ok := sc.Clusters[clusterID]; !ok {
  53                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  54         } else {
  55                 cc.ClusterID = clusterID
  56                 return &cc, nil
  57         }
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxBlockEntries      int
  64         MaxCollectionEntries int
  65         MaxCollectionBytes   int64
  66         MaxPermissionEntries int
  67         MaxUUIDEntries       int
  68 }
  69 type Cluster struct {
  70         ClusterID       string `json:"-"`
  71         ManagementToken string
  72         SystemRootToken string
  73         Services        Services
  74         InstanceTypes   InstanceTypeMap
  75         Containers      ContainersConfig
  76         RemoteClusters  map[string]RemoteCluster
  77         PostgreSQL      PostgreSQL
  78
  79         API struct {
  80                 AsyncPermissionsUpdateInterval Duration
  81                 DisabledAPIs                   StringSet
  82                 MaxIndexDatabaseRead           int
  83                 MaxItemsPerResponse            int
  84                 MaxRequestAmplification        int
  85                 MaxRequestSize                 int
  86                 RailsSessionSecretToken        string
  87                 RequestTimeout                 Duration
  88                 SendTimeout                    Duration
  89                 WebsocketClientEventQueue      int
  90                 WebsocketServerEventQueue      int
  91                 KeepServiceRequestTimeout      Duration
  92         }
  93         AuditLogs struct {
  94                 MaxAge             Duration
  95                 MaxDeleteBatch     int
  96                 UnloggedAttributes StringSet
  97         }
  98         Collections struct {
  99                 BlobSigning          bool
 100                 BlobSigningKey       string
 101                 BlobSigningTTL       Duration
 102                 CollectionVersioning bool
 103                 DefaultTrashLifetime Duration
 104                 DefaultReplication   int
 105                 ManagedProperties    map[string]struct {
 106                         Value     interface{}
 107                         Function  string
 108                         Protected bool
 109                 }
 110                 PreserveVersionIfIdle Duration
 111                 TrashSweepInterval    Duration
 112                 TrustAllContent       bool
 113
 114                 WebDAVCache WebDAVCacheConfig
 115         }
 116         Git struct {
 117                 GitCommand   string
 118                 GitoliteHome string
 119                 Repositories string
 120         }
 121         Login struct {
 122                 ProviderAppSecret string
 123                 ProviderAppID     string
 124         }
 125         Mail struct {
 126                 MailchimpAPIKey                string
 127                 MailchimpListID                string
 128                 SendUserSetupNotificationEmail bool
 129                 IssueReporterEmailFrom         string
 130                 IssueReporterEmailTo           string
 131                 SupportEmailAddress            string
 132                 EmailFrom                      string
 133         }
 134         SystemLogs struct {
 135                 LogLevel                string
 136                 Format                  string
 137                 MaxRequestLogParamsSize int
 138         }
 139         TLS struct {
 140                 Certificate string
 141                 Key         string
 142                 Insecure    bool
 143         }
 144         Users struct {
 145                 AnonymousUserToken                    string
 146                 AdminNotifierEmailFrom                string
 147                 AutoAdminFirstUser                    bool
 148                 AutoAdminUserWithEmail                string
 149                 AutoSetupNewUsers                     bool
 150                 AutoSetupNewUsersWithRepository       bool
 151                 AutoSetupNewUsersWithVmUUID           string
 152                 AutoSetupUsernameBlacklist            StringSet
 153                 EmailSubjectPrefix                    string
 154                 NewInactiveUserNotificationRecipients StringSet
 155                 NewUserNotificationRecipients         StringSet
 156                 NewUsersAreActive                     bool
 157                 UserNotifierEmailFrom                 string
 158                 UserProfileNotificationAddress        string
 159         }
 160         Workbench struct {
 161                 ActivationContactLink            string
 162                 APIClientConnectTimeout          Duration
 163                 APIClientReceiveTimeout          Duration
 164                 APIResponseCompression           bool
 165                 ApplicationMimetypesWithViewIcon StringSet
 166                 ArvadosDocsite                   string
 167                 ArvadosPublicDataDocURL          string
 168                 DefaultOpenIdPrefix              string
 169                 EnableGettingStartedPopup        bool
 170                 EnablePublicProjectsPage         bool
 171                 FileViewersConfigURL             string
 172                 LogViewerMaxBytes                ByteSize
 173                 MultiSiteSearch                  string
 174                 ProfilingEnabled                 bool
 175                 Repositories                     bool
 176                 RepositoryCache                  string
 177                 RunningJobLogRecordsToFetch      int
 178                 SecretKeyBase                    string
 179                 ShowRecentCollectionsOnDashboard bool
 180                 ShowUserAgreementInline          bool
 181                 ShowUserNotifications            bool
 182                 SiteName                         string
 183                 Theme                            string
 184                 UserProfileFormFields            map[string]struct {
 185                         Type                 string
 186                         FormFieldTitle       string
 187                         FormFieldDescription string
 188                         Required             bool
 189                         Position             int
 190                         Options              map[string]struct{}
 191                 }
 192                 UserProfileFormMessage string
 193                 VocabularyURL          string
 194         }
 195
 196         EnableBetaController14287 bool
 197 }
 198
 199 type Services struct {
 200         Composer       Service
 201         Controller     Service
 202         DispatchCloud  Service
 203         GitHTTP        Service
 204         GitSSH         Service
 205         Health         Service
 206         Keepbalance    Service
 207         Keepproxy      Service
 208         Keepstore      Service
 209         Nodemanager    Service
 210         RailsAPI       Service
 211         SSO            Service
 212         WebDAVDownload Service
 213         WebDAV         Service
 214         WebShell       Service
 215         Websocket      Service
 216         Workbench1     Service
 217         Workbench2     Service
 218 }
 219
 220 type Service struct {
 221         InternalURLs map[URL]ServiceInstance
 222         ExternalURL  URL
 223 }
 224
 225 // URL is a url.URL that is also usable as a JSON key/value.
 226 type URL url.URL
 227
 228 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 229 // used as a JSON key/value.
 230 func (su *URL) UnmarshalText(text []byte) error {
 231         u, err := url.Parse(string(text))
 232         if err == nil {
 233                 *su = URL(*u)
 234         }
 235         return err
 236 }
 237
 238 func (su URL) MarshalText() ([]byte, error) {
 239         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 240 }
 241
 242 type ServiceInstance struct{}
 243
 244 type PostgreSQL struct {
 245         Connection     PostgreSQLConnection
 246         ConnectionPool int
 247 }
 248
 249 type PostgreSQLConnection map[string]string
 250
 251 type RemoteCluster struct {
 252         Host          string
 253         Proxy         bool
 254         Scheme        string
 255         Insecure      bool
 256         ActivateUsers bool
 257 }
 258
 259 type InstanceType struct {
 260         Name            string
 261         ProviderType    string
 262         VCPUs           int
 263         RAM             ByteSize
 264         Scratch         ByteSize
 265         IncludedScratch ByteSize
 266         AddedScratch    ByteSize
 267         Price           float64
 268         Preemptible     bool
 269 }
 270
 271 type ContainersConfig struct {
 272         CloudVMs                    CloudVMsConfig
 273         CrunchRunCommand            string
 274         CrunchRunArgumentsList      []string
 275         DefaultKeepCacheRAM         ByteSize
 276         DispatchPrivateKey          string
 277         LogReuseDecisions           bool
 278         MaxComputeVMs               int
 279         MaxDispatchAttempts         int
 280         MaxRetryAttempts            int
 281         MinRetryPeriod              Duration
 282         ReserveExtraRAM             ByteSize
 283         StaleLockTimeout            Duration
 284         SupportedDockerImageFormats StringSet
 285         UsePreemptibleInstances     bool
 286
 287         JobsAPI struct {
 288                 Enable         string
 289                 GitInternalDir string
 290         }
 291         Logging struct {
 292                 MaxAge                       Duration
 293                 LogBytesPerEvent             int
 294                 LogSecondsBetweenEvents      int
 295                 LogThrottlePeriod            Duration
 296                 LogThrottleBytes             int
 297                 LogThrottleLines             int
 298                 LimitLogBytesPerJob          int
 299                 LogPartialLineThrottlePeriod Duration
 300                 LogUpdatePeriod              Duration
 301                 LogUpdateSize                ByteSize
 302         }
 303         SLURM struct {
 304                 PrioritySpread             int64
 305                 SbatchArgumentsList        []string
 306                 SbatchEnvironmentVariables map[string]string
 307                 Managed                    struct {
 308                         DNSServerConfDir       string
 309                         DNSServerConfTemplate  string
 310                         DNSServerReloadCommand string
 311                         DNSServerUpdateCommand string
 312                         ComputeNodeDomain      string
 313                         ComputeNodeNameservers StringSet
 314                         AssignNodeHostname     string
 315                 }
 316         }
 317 }
 318
 319 type CloudVMsConfig struct {
 320         Enable bool
 321
 322         BootProbeCommand     string
 323         ImageID              string
 324         MaxCloudOpsPerSecond int
 325         MaxProbesPerSecond   int
 326         PollInterval         Duration
 327         ProbeInterval        Duration
 328         SSHPort              string
 329         SyncInterval         Duration
 330         TimeoutBooting       Duration
 331         TimeoutIdle          Duration
 332         TimeoutProbe         Duration
 333         TimeoutShutdown      Duration
 334         TimeoutSignal        Duration
 335         TimeoutTERM          Duration
 336         ResourceTags         map[string]string
 337         TagKeyPrefix         string
 338
 339         Driver           string
 340         DriverParameters json.RawMessage
 341 }
 342
 343 type InstanceTypeMap map[string]InstanceType
 344
 345 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 346
 347 // UnmarshalJSON handles old config files that provide an array of
 348 // instance types instead of a hash.
 349 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 350         if len(data) > 0 && data[0] == '[' {
 351                 var arr []InstanceType
 352                 err := json.Unmarshal(data, &arr)
 353                 if err != nil {
 354                         return err
 355                 }
 356                 if len(arr) == 0 {
 357                         *it = nil
 358                         return nil
 359                 }
 360                 *it = make(map[string]InstanceType, len(arr))
 361                 for _, t := range arr {
 362                         if _, ok := (*it)[t.Name]; ok {
 363                                 return errDuplicateInstanceTypeName
 364                         }
 365                         if t.ProviderType == "" {
 366                                 t.ProviderType = t.Name
 367                         }
 368                         if t.Scratch == 0 {
 369                                 t.Scratch = t.IncludedScratch + t.AddedScratch
 370                         } else if t.AddedScratch == 0 {
 371                                 t.AddedScratch = t.Scratch - t.IncludedScratch
 372                         } else if t.IncludedScratch == 0 {
 373                                 t.IncludedScratch = t.Scratch - t.AddedScratch
 374                         }
 375
 376                         if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 377                                 return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 378                         }
 379                         (*it)[t.Name] = t
 380                 }
 381                 return nil
 382         }
 383         var hash map[string]InstanceType
 384         err := json.Unmarshal(data, &hash)
 385         if err != nil {
 386                 return err
 387         }
 388         // Fill in Name field (and ProviderType field, if not
 389         // specified) using hash key.
 390         *it = InstanceTypeMap(hash)
 391         for name, t := range *it {
 392                 t.Name = name
 393                 if t.ProviderType == "" {
 394                         t.ProviderType = name
 395                 }
 396                 (*it)[name] = t
 397         }
 398         return nil
 399 }
 400
 401 type StringSet map[string]struct{}
 402
 403 // UnmarshalJSON handles old config files that provide an array of
 404 // instance types instead of a hash.
 405 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 406         if len(data) > 0 && data[0] == '[' {
 407                 var arr []string
 408                 err := json.Unmarshal(data, &arr)
 409                 if err != nil {
 410                         return err
 411                 }
 412                 if len(arr) == 0 {
 413                         *ss = nil
 414                         return nil
 415                 }
 416                 *ss = make(map[string]struct{}, len(arr))
 417                 for _, t := range arr {
 418                         (*ss)[t] = struct{}{}
 419                 }
 420                 return nil
 421         }
 422         var hash map[string]struct{}
 423         err := json.Unmarshal(data, &hash)
 424         if err != nil {
 425                 return err
 426         }
 427         *ss = make(map[string]struct{}, len(hash))
 428         for t, _ := range hash {
 429                 (*ss)[t] = struct{}{}
 430         }
 431
 432         return nil
 433 }
 434
 435 type ServiceName string
 436
 437 const (
 438         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 439         ServiceNameController    ServiceName = "arvados-controller"
 440         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 441         ServiceNameHealth        ServiceName = "arvados-health"
 442         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 443         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 444         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 445         ServiceNameWebsocket     ServiceName = "arvados-ws"
 446         ServiceNameKeepbalance   ServiceName = "keep-balance"
 447         ServiceNameKeepweb       ServiceName = "keep-web"
 448         ServiceNameKeepproxy     ServiceName = "keepproxy"
 449         ServiceNameKeepstore     ServiceName = "keepstore"
 450 )
 451
 452 // Map returns all services as a map, suitable for iterating over all
 453 // services or looking up a service by name.
 454 func (svcs Services) Map() map[ServiceName]Service {
 455         return map[ServiceName]Service{
 456                 ServiceNameRailsAPI:      svcs.RailsAPI,
 457                 ServiceNameController:    svcs.Controller,
 458                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 459                 ServiceNameHealth:        svcs.Health,
 460                 ServiceNameNodemanager:   svcs.Nodemanager,
 461                 ServiceNameWorkbench1:    svcs.Workbench1,
 462                 ServiceNameWorkbench2:    svcs.Workbench2,
 463                 ServiceNameWebsocket:     svcs.Websocket,
 464                 ServiceNameKeepbalance:   svcs.Keepbalance,
 465                 ServiceNameKeepweb:       svcs.WebDAV,
 466                 ServiceNameKeepproxy:     svcs.Keepproxy,
 467                 ServiceNameKeepstore:     svcs.Keepstore,
 468         }
 469 }