sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.curoverse.com/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters map[string]Cluster
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         if cc, ok := sc.Clusters[clusterID]; !ok {
  53                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  54         } else {
  55                 cc.ClusterID = clusterID
  56                 return &cc, nil
  57         }
  58 }
  59
  60 type Cluster struct {
  61         ClusterID       string `json:"-"`
  62         ManagementToken string
  63         SystemRootToken string
  64         Services        Services
  65         InstanceTypes   InstanceTypeMap
  66         Containers      ContainersConfig
  67         RemoteClusters  map[string]RemoteCluster
  68         PostgreSQL      PostgreSQL
  69
  70         API struct {
  71                 AsyncPermissionsUpdateInterval Duration
  72                 DisabledAPIs                   StringSet
  73                 MaxIndexDatabaseRead           int
  74                 MaxItemsPerResponse            int
  75                 MaxRequestAmplification        int
  76                 MaxRequestSize                 int
  77                 RailsSessionSecretToken        string
  78                 RequestTimeout                 Duration
  79                 SendTimeout                    Duration
  80                 WebsocketClientEventQueue      int
  81                 WebsocketServerEventQueue      int
  82                 KeepServiceRequestTimeout      Duration
  83         }
  84         AuditLogs struct {
  85                 MaxAge             Duration
  86                 MaxDeleteBatch     int
  87                 UnloggedAttributes StringSet
  88         }
  89         Collections struct {
  90                 BlobSigning          bool
  91                 BlobSigningKey       string
  92                 BlobSigningTTL       Duration
  93                 CollectionVersioning bool
  94                 DefaultTrashLifetime Duration
  95                 DefaultReplication   int
  96                 ManagedProperties    map[string]struct {
  97                         Value     interface{}
  98                         Function  string
  99                         Protected bool
 100                 }
 101                 PreserveVersionIfIdle Duration
 102                 TrashSweepInterval    Duration
 103                 TrustAllContent       bool
 104         }
 105         Git struct {
 106                 Repositories string
 107         }
 108         Login struct {
 109                 ProviderAppSecret string
 110                 ProviderAppID     string
 111         }
 112         Mail struct {
 113                 MailchimpAPIKey                string
 114                 MailchimpListID                string
 115                 SendUserSetupNotificationEmail bool
 116                 IssueReporterEmailFrom         string
 117                 IssueReporterEmailTo           string
 118                 SupportEmailAddress            string
 119                 EmailFrom                      string
 120         }
 121         SystemLogs struct {
 122                 LogLevel                string
 123                 Format                  string
 124                 MaxRequestLogParamsSize int
 125         }
 126         TLS struct {
 127                 Certificate string
 128                 Key         string
 129                 Insecure    bool
 130         }
 131         Users struct {
 132                 AnonymousUserToken                    string
 133                 AdminNotifierEmailFrom                string
 134                 AutoAdminFirstUser                    bool
 135                 AutoAdminUserWithEmail                string
 136                 AutoSetupNewUsers                     bool
 137                 AutoSetupNewUsersWithRepository       bool
 138                 AutoSetupNewUsersWithVmUUID           string
 139                 AutoSetupUsernameBlacklist            StringSet
 140                 EmailSubjectPrefix                    string
 141                 NewInactiveUserNotificationRecipients StringSet
 142                 NewUserNotificationRecipients         StringSet
 143                 NewUsersAreActive                     bool
 144                 UserNotifierEmailFrom                 string
 145                 UserProfileNotificationAddress        string
 146         }
 147         Workbench struct {
 148                 ActivationContactLink            string
 149                 APIClientConnectTimeout          Duration
 150                 APIClientReceiveTimeout          Duration
 151                 APIResponseCompression           bool
 152                 ApplicationMimetypesWithViewIcon StringSet
 153                 ArvadosDocsite                   string
 154                 ArvadosPublicDataDocURL          string
 155                 DefaultOpenIdPrefix              string
 156                 EnableGettingStartedPopup        bool
 157                 EnablePublicProjectsPage         bool
 158                 FileViewersConfigURL             string
 159                 LogViewerMaxBytes                ByteSize
 160                 MultiSiteSearch                  string
 161                 ProfilingEnabled                 bool
 162                 Repositories                     bool
 163                 RepositoryCache                  string
 164                 RunningJobLogRecordsToFetch      int
 165                 SecretKeyBase                    string
 166                 ShowRecentCollectionsOnDashboard bool
 167                 ShowUserAgreementInline          bool
 168                 ShowUserNotifications            bool
 169                 SiteName                         string
 170                 Theme                            string
 171                 UserProfileFormFields            map[string]struct {
 172                         Type                 string
 173                         FormFieldTitle       string
 174                         FormFieldDescription string
 175                         Required             bool
 176                         Position             int
 177                         Options              map[string]struct{}
 178                 }
 179                 UserProfileFormMessage string
 180                 VocabularyURL          string
 181         }
 182
 183         EnableBetaController14287 bool
 184 }
 185
 186 type Services struct {
 187         Composer       Service
 188         Controller     Service
 189         DispatchCloud  Service
 190         GitHTTP        Service
 191         GitSSH         Service
 192         Health         Service
 193         Keepbalance    Service
 194         Keepproxy      Service
 195         Keepstore      Service
 196         Nodemanager    Service
 197         RailsAPI       Service
 198         SSO            Service
 199         WebDAVDownload Service
 200         WebDAV         Service
 201         WebShell       Service
 202         Websocket      Service
 203         Workbench1     Service
 204         Workbench2     Service
 205 }
 206
 207 type Service struct {
 208         InternalURLs map[URL]ServiceInstance
 209         ExternalURL  URL
 210 }
 211
 212 // URL is a url.URL that is also usable as a JSON key/value.
 213 type URL url.URL
 214
 215 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 216 // used as a JSON key/value.
 217 func (su *URL) UnmarshalText(text []byte) error {
 218         u, err := url.Parse(string(text))
 219         if err == nil {
 220                 *su = URL(*u)
 221         }
 222         return err
 223 }
 224
 225 func (su URL) MarshalText() ([]byte, error) {
 226         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 227 }
 228
 229 type ServiceInstance struct{}
 230
 231 type PostgreSQL struct {
 232         Connection     PostgreSQLConnection
 233         ConnectionPool int
 234 }
 235
 236 type PostgreSQLConnection map[string]string
 237
 238 type RemoteCluster struct {
 239         Host          string
 240         Proxy         bool
 241         Scheme        string
 242         Insecure      bool
 243         ActivateUsers bool
 244 }
 245
 246 type InstanceType struct {
 247         Name            string
 248         ProviderType    string
 249         VCPUs           int
 250         RAM             ByteSize
 251         Scratch         ByteSize
 252         IncludedScratch ByteSize
 253         AddedScratch    ByteSize
 254         Price           float64
 255         Preemptible     bool
 256 }
 257
 258 type ContainersConfig struct {
 259         CloudVMs                    CloudVMsConfig
 260         CrunchRunCommand            string
 261         CrunchRunArgumentsList      []string
 262         DefaultKeepCacheRAM         ByteSize
 263         DispatchPrivateKey          string
 264         LogReuseDecisions           bool
 265         MaxComputeVMs               int
 266         MaxDispatchAttempts         int
 267         MaxRetryAttempts            int
 268         MinRetryPeriod              Duration
 269         ReserveExtraRAM             ByteSize
 270         StaleLockTimeout            Duration
 271         SupportedDockerImageFormats StringSet
 272         UsePreemptibleInstances     bool
 273
 274         JobsAPI struct {
 275                 Enable                  string
 276                 GitInternalDir          string
 277                 DefaultDockerImage      string
 278                 CrunchJobWrapper        string
 279                 CrunchJobUser           string
 280                 CrunchRefreshTrigger    string
 281                 ReuseJobIfOutputsDiffer bool
 282         }
 283         Logging struct {
 284                 MaxAge                       Duration
 285                 LogBytesPerEvent             int
 286                 LogSecondsBetweenEvents      int
 287                 LogThrottlePeriod            Duration
 288                 LogThrottleBytes             int
 289                 LogThrottleLines             int
 290                 LimitLogBytesPerJob          int
 291                 LogPartialLineThrottlePeriod Duration
 292                 LogUpdatePeriod              Duration
 293                 LogUpdateSize                ByteSize
 294         }
 295         SLURM struct {
 296                 PrioritySpread             int64
 297                 SbatchArgumentsList        []string
 298                 SbatchEnvironmentVariables map[string]string
 299                 Managed                    struct {
 300                         DNSServerConfDir       string
 301                         DNSServerConfTemplate  string
 302                         DNSServerReloadCommand string
 303                         DNSServerUpdateCommand string
 304                         ComputeNodeDomain      string
 305                         ComputeNodeNameservers StringSet
 306                         AssignNodeHostname     string
 307                 }
 308         }
 309 }
 310
 311 type CloudVMsConfig struct {
 312         Enable bool
 313
 314         BootProbeCommand     string
 315         ImageID              string
 316         MaxCloudOpsPerSecond int
 317         MaxProbesPerSecond   int
 318         PollInterval         Duration
 319         ProbeInterval        Duration
 320         SSHPort              string
 321         SyncInterval         Duration
 322         TimeoutBooting       Duration
 323         TimeoutIdle          Duration
 324         TimeoutProbe         Duration
 325         TimeoutShutdown      Duration
 326         TimeoutSignal        Duration
 327         TimeoutTERM          Duration
 328         ResourceTags         map[string]string
 329         TagKeyPrefix         string
 330
 331         Driver           string
 332         DriverParameters json.RawMessage
 333 }
 334
 335 type InstanceTypeMap map[string]InstanceType
 336
 337 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 338
 339 // UnmarshalJSON handles old config files that provide an array of
 340 // instance types instead of a hash.
 341 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 342         if len(data) > 0 && data[0] == '[' {
 343                 var arr []InstanceType
 344                 err := json.Unmarshal(data, &arr)
 345                 if err != nil {
 346                         return err
 347                 }
 348                 if len(arr) == 0 {
 349                         *it = nil
 350                         return nil
 351                 }
 352                 *it = make(map[string]InstanceType, len(arr))
 353                 for _, t := range arr {
 354                         if _, ok := (*it)[t.Name]; ok {
 355                                 return errDuplicateInstanceTypeName
 356                         }
 357                         if t.ProviderType == "" {
 358                                 t.ProviderType = t.Name
 359                         }
 360                         if t.Scratch == 0 {
 361                                 t.Scratch = t.IncludedScratch + t.AddedScratch
 362                         } else if t.AddedScratch == 0 {
 363                                 t.AddedScratch = t.Scratch - t.IncludedScratch
 364                         } else if t.IncludedScratch == 0 {
 365                                 t.IncludedScratch = t.Scratch - t.AddedScratch
 366                         }
 367
 368                         if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 369                                 return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 370                         }
 371                         (*it)[t.Name] = t
 372                 }
 373                 return nil
 374         }
 375         var hash map[string]InstanceType
 376         err := json.Unmarshal(data, &hash)
 377         if err != nil {
 378                 return err
 379         }
 380         // Fill in Name field (and ProviderType field, if not
 381         // specified) using hash key.
 382         *it = InstanceTypeMap(hash)
 383         for name, t := range *it {
 384                 t.Name = name
 385                 if t.ProviderType == "" {
 386                         t.ProviderType = name
 387                 }
 388                 (*it)[name] = t
 389         }
 390         return nil
 391 }
 392
 393 type StringSet map[string]struct{}
 394
 395 // UnmarshalJSON handles old config files that provide an array of
 396 // instance types instead of a hash.
 397 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 398         if len(data) > 0 && data[0] == '[' {
 399                 var arr []string
 400                 err := json.Unmarshal(data, &arr)
 401                 if err != nil {
 402                         return err
 403                 }
 404                 if len(arr) == 0 {
 405                         *ss = nil
 406                         return nil
 407                 }
 408                 *ss = make(map[string]struct{}, len(arr))
 409                 for _, t := range arr {
 410                         (*ss)[t] = struct{}{}
 411                 }
 412                 return nil
 413         }
 414         var hash map[string]struct{}
 415         err := json.Unmarshal(data, &hash)
 416         if err != nil {
 417                 return err
 418         }
 419         *ss = make(map[string]struct{}, len(hash))
 420         for t, _ := range hash {
 421                 (*ss)[t] = struct{}{}
 422         }
 423
 424         return nil
 425 }
 426
 427 type ServiceName string
 428
 429 const (
 430         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 431         ServiceNameController    ServiceName = "arvados-controller"
 432         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 433         ServiceNameHealth        ServiceName = "arvados-health"
 434         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 435         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 436         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 437         ServiceNameWebsocket     ServiceName = "arvados-ws"
 438         ServiceNameKeepbalance   ServiceName = "keep-balance"
 439         ServiceNameKeepweb       ServiceName = "keep-web"
 440         ServiceNameKeepproxy     ServiceName = "keepproxy"
 441         ServiceNameKeepstore     ServiceName = "keepstore"
 442 )
 443
 444 // Map returns all services as a map, suitable for iterating over all
 445 // services or looking up a service by name.
 446 func (svcs Services) Map() map[ServiceName]Service {
 447         return map[ServiceName]Service{
 448                 ServiceNameRailsAPI:      svcs.RailsAPI,
 449                 ServiceNameController:    svcs.Controller,
 450                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 451                 ServiceNameHealth:        svcs.Health,
 452                 ServiceNameNodemanager:   svcs.Nodemanager,
 453                 ServiceNameWorkbench1:    svcs.Workbench1,
 454                 ServiceNameWorkbench2:    svcs.Workbench2,
 455                 ServiceNameWebsocket:     svcs.Websocket,
 456                 ServiceNameKeepbalance:   svcs.Keepbalance,
 457                 ServiceNameKeepweb:       svcs.WebDAV,
 458                 ServiceNameKeepproxy:     svcs.Keepproxy,
 459                 ServiceNameKeepstore:     svcs.Keepstore,
 460         }
 461 }