sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.curoverse.com/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters map[string]Cluster
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         if cc, ok := sc.Clusters[clusterID]; !ok {
  53                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  54         } else {
  55                 cc.ClusterID = clusterID
  56                 return &cc, nil
  57         }
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxBlockEntries      int
  64         MaxCollectionEntries int
  65         MaxCollectionBytes   int64
  66         MaxPermissionEntries int
  67         MaxUUIDEntries       int
  68 }
  69 type Cluster struct {
  70         ClusterID       string `json:"-"`
  71         ManagementToken string
  72         SystemRootToken string
  73         Services        Services
  74         InstanceTypes   InstanceTypeMap
  75         Containers      ContainersConfig
  76         RemoteClusters  map[string]RemoteCluster
  77         PostgreSQL      PostgreSQL
  78
  79         API struct {
  80                 AsyncPermissionsUpdateInterval Duration
  81                 DisabledAPIs                   StringSet
  82                 MaxIndexDatabaseRead           int
  83                 MaxItemsPerResponse            int
  84                 MaxRequestAmplification        int
  85                 MaxRequestSize                 int
  86                 RailsSessionSecretToken        string
  87                 RequestTimeout                 Duration
  88                 SendTimeout                    Duration
  89                 WebsocketClientEventQueue      int
  90                 WebsocketServerEventQueue      int
  91         }
  92         AuditLogs struct {
  93                 MaxAge             Duration
  94                 MaxDeleteBatch     int
  95                 UnloggedAttributes StringSet
  96         }
  97         Collections struct {
  98                 BlobSigning          bool
  99                 BlobSigningKey       string
 100                 BlobSigningTTL       Duration
 101                 CollectionVersioning bool
 102                 DefaultTrashLifetime Duration
 103                 DefaultReplication   int
 104                 ManagedProperties    map[string]struct {
 105                         Value     interface{}
 106                         Function  string
 107                         Protected bool
 108                 }
 109                 PreserveVersionIfIdle Duration
 110                 TrashSweepInterval    Duration
 111                 TrustAllContent       bool
 112
 113                 WebDAVCache WebDAVCacheConfig
 114         }
 115         Git struct {
 116                 Repositories string
 117         }
 118         Login struct {
 119                 ProviderAppSecret  string
 120                 ProviderAppID      string
 121                 LoginCluster       string
 122                 RemoteTokenRefresh Duration
 123         }
 124         Mail struct {
 125                 MailchimpAPIKey                string
 126                 MailchimpListID                string
 127                 SendUserSetupNotificationEmail bool
 128                 IssueReporterEmailFrom         string
 129                 IssueReporterEmailTo           string
 130                 SupportEmailAddress            string
 131                 EmailFrom                      string
 132         }
 133         SystemLogs struct {
 134                 LogLevel                string
 135                 Format                  string
 136                 MaxRequestLogParamsSize int
 137         }
 138         TLS struct {
 139                 Certificate string
 140                 Key         string
 141                 Insecure    bool
 142         }
 143         Users struct {
 144                 AnonymousUserToken                    string
 145                 AdminNotifierEmailFrom                string
 146                 AutoAdminFirstUser                    bool
 147                 AutoAdminUserWithEmail                string
 148                 AutoSetupNewUsers                     bool
 149                 AutoSetupNewUsersWithRepository       bool
 150                 AutoSetupNewUsersWithVmUUID           string
 151                 AutoSetupUsernameBlacklist            StringSet
 152                 EmailSubjectPrefix                    string
 153                 NewInactiveUserNotificationRecipients StringSet
 154                 NewUserNotificationRecipients         StringSet
 155                 NewUsersAreActive                     bool
 156                 UserNotifierEmailFrom                 string
 157                 UserProfileNotificationAddress        string
 158         }
 159         Workbench struct {
 160                 ActivationContactLink            string
 161                 APIClientConnectTimeout          Duration
 162                 APIClientReceiveTimeout          Duration
 163                 APIResponseCompression           bool
 164                 ApplicationMimetypesWithViewIcon StringSet
 165                 ArvadosDocsite                   string
 166                 ArvadosPublicDataDocURL          string
 167                 DefaultOpenIdPrefix              string
 168                 EnableGettingStartedPopup        bool
 169                 EnablePublicProjectsPage         bool
 170                 FileViewersConfigURL             string
 171                 LogViewerMaxBytes                ByteSize
 172                 MultiSiteSearch                  string
 173                 ProfilingEnabled                 bool
 174                 Repositories                     bool
 175                 RepositoryCache                  string
 176                 RunningJobLogRecordsToFetch      int
 177                 SecretKeyBase                    string
 178                 ShowRecentCollectionsOnDashboard bool
 179                 ShowUserAgreementInline          bool
 180                 ShowUserNotifications            bool
 181                 SiteName                         string
 182                 Theme                            string
 183                 UserProfileFormFields            map[string]struct {
 184                         Type                 string
 185                         FormFieldTitle       string
 186                         FormFieldDescription string
 187                         Required             bool
 188                         Position             int
 189                         Options              map[string]struct{}
 190                 }
 191                 UserProfileFormMessage string
 192                 VocabularyURL          string
 193         }
 194
 195         EnableBetaController14287 bool
 196 }
 197
 198 type Services struct {
 199         Composer       Service
 200         Controller     Service
 201         DispatchCloud  Service
 202         GitHTTP        Service
 203         GitSSH         Service
 204         Health         Service
 205         Keepbalance    Service
 206         Keepproxy      Service
 207         Keepstore      Service
 208         Nodemanager    Service
 209         RailsAPI       Service
 210         SSO            Service
 211         WebDAVDownload Service
 212         WebDAV         Service
 213         WebShell       Service
 214         Websocket      Service
 215         Workbench1     Service
 216         Workbench2     Service
 217 }
 218
 219 type Service struct {
 220         InternalURLs map[URL]ServiceInstance
 221         ExternalURL  URL
 222 }
 223
 224 // URL is a url.URL that is also usable as a JSON key/value.
 225 type URL url.URL
 226
 227 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 228 // used as a JSON key/value.
 229 func (su *URL) UnmarshalText(text []byte) error {
 230         u, err := url.Parse(string(text))
 231         if err == nil {
 232                 *su = URL(*u)
 233         }
 234         return err
 235 }
 236
 237 func (su URL) MarshalText() ([]byte, error) {
 238         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 239 }
 240
 241 type ServiceInstance struct{}
 242
 243 type PostgreSQL struct {
 244         Connection     PostgreSQLConnection
 245         ConnectionPool int
 246 }
 247
 248 type PostgreSQLConnection map[string]string
 249
 250 type RemoteCluster struct {
 251         Host          string
 252         Proxy         bool
 253         Scheme        string
 254         Insecure      bool
 255         ActivateUsers bool
 256 }
 257
 258 type InstanceType struct {
 259         Name            string
 260         ProviderType    string
 261         VCPUs           int
 262         RAM             ByteSize
 263         Scratch         ByteSize
 264         IncludedScratch ByteSize
 265         AddedScratch    ByteSize
 266         Price           float64
 267         Preemptible     bool
 268 }
 269
 270 type ContainersConfig struct {
 271         CloudVMs                    CloudVMsConfig
 272         CrunchRunCommand            string
 273         CrunchRunArgumentsList      []string
 274         DefaultKeepCacheRAM         ByteSize
 275         DispatchPrivateKey          string
 276         LogReuseDecisions           bool
 277         MaxComputeVMs               int
 278         MaxDispatchAttempts         int
 279         MaxRetryAttempts            int
 280         MinRetryPeriod              Duration
 281         ReserveExtraRAM             ByteSize
 282         StaleLockTimeout            Duration
 283         SupportedDockerImageFormats StringSet
 284         UsePreemptibleInstances     bool
 285
 286         JobsAPI struct {
 287                 Enable         string
 288                 GitInternalDir string
 289         }
 290         Logging struct {
 291                 MaxAge                       Duration
 292                 LogBytesPerEvent             int
 293                 LogSecondsBetweenEvents      int
 294                 LogThrottlePeriod            Duration
 295                 LogThrottleBytes             int
 296                 LogThrottleLines             int
 297                 LimitLogBytesPerJob          int
 298                 LogPartialLineThrottlePeriod Duration
 299                 LogUpdatePeriod              Duration
 300                 LogUpdateSize                ByteSize
 301         }
 302         SLURM struct {
 303                 PrioritySpread             int64
 304                 SbatchArgumentsList        []string
 305                 SbatchEnvironmentVariables map[string]string
 306                 Managed                    struct {
 307                         DNSServerConfDir       string
 308                         DNSServerConfTemplate  string
 309                         DNSServerReloadCommand string
 310                         DNSServerUpdateCommand string
 311                         ComputeNodeDomain      string
 312                         ComputeNodeNameservers StringSet
 313                         AssignNodeHostname     string
 314                 }
 315         }
 316 }
 317
 318 type CloudVMsConfig struct {
 319         Enable bool
 320
 321         BootProbeCommand     string
 322         ImageID              string
 323         MaxCloudOpsPerSecond int
 324         MaxProbesPerSecond   int
 325         PollInterval         Duration
 326         ProbeInterval        Duration
 327         SSHPort              string
 328         SyncInterval         Duration
 329         TimeoutBooting       Duration
 330         TimeoutIdle          Duration
 331         TimeoutProbe         Duration
 332         TimeoutShutdown      Duration
 333         TimeoutSignal        Duration
 334         TimeoutTERM          Duration
 335         ResourceTags         map[string]string
 336         TagKeyPrefix         string
 337
 338         Driver           string
 339         DriverParameters json.RawMessage
 340 }
 341
 342 type InstanceTypeMap map[string]InstanceType
 343
 344 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 345
 346 // UnmarshalJSON handles old config files that provide an array of
 347 // instance types instead of a hash.
 348 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 349         if len(data) > 0 && data[0] == '[' {
 350                 var arr []InstanceType
 351                 err := json.Unmarshal(data, &arr)
 352                 if err != nil {
 353                         return err
 354                 }
 355                 if len(arr) == 0 {
 356                         *it = nil
 357                         return nil
 358                 }
 359                 *it = make(map[string]InstanceType, len(arr))
 360                 for _, t := range arr {
 361                         if _, ok := (*it)[t.Name]; ok {
 362                                 return errDuplicateInstanceTypeName
 363                         }
 364                         if t.ProviderType == "" {
 365                                 t.ProviderType = t.Name
 366                         }
 367                         if t.Scratch == 0 {
 368                                 t.Scratch = t.IncludedScratch + t.AddedScratch
 369                         } else if t.AddedScratch == 0 {
 370                                 t.AddedScratch = t.Scratch - t.IncludedScratch
 371                         } else if t.IncludedScratch == 0 {
 372                                 t.IncludedScratch = t.Scratch - t.AddedScratch
 373                         }
 374
 375                         if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 376                                 return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 377                         }
 378                         (*it)[t.Name] = t
 379                 }
 380                 return nil
 381         }
 382         var hash map[string]InstanceType
 383         err := json.Unmarshal(data, &hash)
 384         if err != nil {
 385                 return err
 386         }
 387         // Fill in Name field (and ProviderType field, if not
 388         // specified) using hash key.
 389         *it = InstanceTypeMap(hash)
 390         for name, t := range *it {
 391                 t.Name = name
 392                 if t.ProviderType == "" {
 393                         t.ProviderType = name
 394                 }
 395                 (*it)[name] = t
 396         }
 397         return nil
 398 }
 399
 400 type StringSet map[string]struct{}
 401
 402 // UnmarshalJSON handles old config files that provide an array of
 403 // instance types instead of a hash.
 404 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 405         if len(data) > 0 && data[0] == '[' {
 406                 var arr []string
 407                 err := json.Unmarshal(data, &arr)
 408                 if err != nil {
 409                         return err
 410                 }
 411                 if len(arr) == 0 {
 412                         *ss = nil
 413                         return nil
 414                 }
 415                 *ss = make(map[string]struct{}, len(arr))
 416                 for _, t := range arr {
 417                         (*ss)[t] = struct{}{}
 418                 }
 419                 return nil
 420         }
 421         var hash map[string]struct{}
 422         err := json.Unmarshal(data, &hash)
 423         if err != nil {
 424                 return err
 425         }
 426         *ss = make(map[string]struct{}, len(hash))
 427         for t, _ := range hash {
 428                 (*ss)[t] = struct{}{}
 429         }
 430
 431         return nil
 432 }
 433
 434 type ServiceName string
 435
 436 const (
 437         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 438         ServiceNameController    ServiceName = "arvados-controller"
 439         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 440         ServiceNameHealth        ServiceName = "arvados-health"
 441         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 442         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 443         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 444         ServiceNameWebsocket     ServiceName = "arvados-ws"
 445         ServiceNameKeepbalance   ServiceName = "keep-balance"
 446         ServiceNameKeepweb       ServiceName = "keep-web"
 447         ServiceNameKeepproxy     ServiceName = "keepproxy"
 448         ServiceNameKeepstore     ServiceName = "keepstore"
 449 )
 450
 451 // Map returns all services as a map, suitable for iterating over all
 452 // services or looking up a service by name.
 453 func (svcs Services) Map() map[ServiceName]Service {
 454         return map[ServiceName]Service{
 455                 ServiceNameRailsAPI:      svcs.RailsAPI,
 456                 ServiceNameController:    svcs.Controller,
 457                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 458                 ServiceNameHealth:        svcs.Health,
 459                 ServiceNameNodemanager:   svcs.Nodemanager,
 460                 ServiceNameWorkbench1:    svcs.Workbench1,
 461                 ServiceNameWorkbench2:    svcs.Workbench2,
 462                 ServiceNameWebsocket:     svcs.Websocket,
 463                 ServiceNameKeepbalance:   svcs.Keepbalance,
 464                 ServiceNameKeepweb:       svcs.WebDAV,
 465                 ServiceNameKeepproxy:     svcs.Keepproxy,
 466                 ServiceNameKeepstore:     svcs.Keepstore,
 467         }
 468 }