sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.curoverse.com/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters map[string]Cluster
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         if cc, ok := sc.Clusters[clusterID]; !ok {
  53                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  54         } else {
  55                 cc.ClusterID = clusterID
  56                 return &cc, nil
  57         }
  58 }
  59
  60 type Cluster struct {
  61         ClusterID       string `json:"-"`
  62         ManagementToken string
  63         SystemRootToken string
  64         Services        Services
  65         InstanceTypes   InstanceTypeMap
  66         Containers      ContainersConfig
  67         RemoteClusters  map[string]RemoteCluster
  68         PostgreSQL      PostgreSQL
  69
  70         API struct {
  71                 AsyncPermissionsUpdateInterval Duration
  72                 DisabledAPIs                   StringSet
  73                 MaxIndexDatabaseRead           int
  74                 MaxItemsPerResponse            int
  75                 MaxRequestAmplification        int
  76                 MaxRequestSize                 int
  77                 RailsSessionSecretToken        string
  78                 RequestTimeout                 Duration
  79                 SendTimeout                    Duration
  80                 WebsocketClientEventQueue      int
  81                 WebsocketServerEventQueue      int
  82         }
  83         AuditLogs struct {
  84                 MaxAge             Duration
  85                 MaxDeleteBatch     int
  86                 UnloggedAttributes StringSet
  87         }
  88         Collections struct {
  89                 BlobSigning          bool
  90                 BlobSigningKey       string
  91                 BlobSigningTTL       Duration
  92                 CollectionVersioning bool
  93                 DefaultTrashLifetime Duration
  94                 DefaultReplication   int
  95                 ManagedProperties    map[string]struct {
  96                         Value     interface{}
  97                         Function  string
  98                         Protected bool
  99                 }
 100                 PreserveVersionIfIdle Duration
 101                 TrashSweepInterval    Duration
 102                 TrustAllContent       bool
 103         }
 104         Git struct {
 105                 Repositories string
 106         }
 107         Login struct {
 108                 ProviderAppSecret string
 109                 ProviderAppID     string
 110         }
 111         Mail struct {
 112                 MailchimpAPIKey                string
 113                 MailchimpListID                string
 114                 SendUserSetupNotificationEmail bool
 115                 IssueReporterEmailFrom         string
 116                 IssueReporterEmailTo           string
 117                 SupportEmailAddress            string
 118                 EmailFrom                      string
 119         }
 120         SystemLogs struct {
 121                 LogLevel                string
 122                 Format                  string
 123                 MaxRequestLogParamsSize int
 124         }
 125         TLS struct {
 126                 Certificate string
 127                 Key         string
 128                 Insecure    bool
 129         }
 130         Users struct {
 131                 AnonymousUserToken                    string
 132                 AdminNotifierEmailFrom                string
 133                 AutoAdminFirstUser                    bool
 134                 AutoAdminUserWithEmail                string
 135                 AutoSetupNewUsers                     bool
 136                 AutoSetupNewUsersWithRepository       bool
 137                 AutoSetupNewUsersWithVmUUID           string
 138                 AutoSetupUsernameBlacklist            StringSet
 139                 EmailSubjectPrefix                    string
 140                 NewInactiveUserNotificationRecipients StringSet
 141                 NewUserNotificationRecipients         StringSet
 142                 NewUsersAreActive                     bool
 143                 UserNotifierEmailFrom                 string
 144                 UserProfileNotificationAddress        string
 145         }
 146         Workbench struct {
 147                 ActivationContactLink            string
 148                 APIClientConnectTimeout          Duration
 149                 APIClientReceiveTimeout          Duration
 150                 APIResponseCompression           bool
 151                 ApplicationMimetypesWithViewIcon StringSet
 152                 ArvadosDocsite                   string
 153                 ArvadosPublicDataDocURL          string
 154                 DefaultOpenIdPrefix              string
 155                 EnableGettingStartedPopup        bool
 156                 EnablePublicProjectsPage         bool
 157                 FileViewersConfigURL             string
 158                 LogViewerMaxBytes                ByteSize
 159                 MultiSiteSearch                  string
 160                 ProfilingEnabled                 bool
 161                 Repositories                     bool
 162                 RepositoryCache                  string
 163                 RunningJobLogRecordsToFetch      int
 164                 SecretKeyBase                    string
 165                 ShowRecentCollectionsOnDashboard bool
 166                 ShowUserAgreementInline          bool
 167                 ShowUserNotifications            bool
 168                 SiteName                         string
 169                 Theme                            string
 170                 UserProfileFormFields            map[string]struct {
 171                         Type                 string
 172                         FormFieldTitle       string
 173                         FormFieldDescription string
 174                         Required             bool
 175                         Position             int
 176                         Options              map[string]struct{}
 177                 }
 178                 UserProfileFormMessage string
 179                 VocabularyURL          string
 180         }
 181
 182         EnableBetaController14287 bool
 183 }
 184
 185 type Services struct {
 186         Composer       Service
 187         Controller     Service
 188         DispatchCloud  Service
 189         GitHTTP        Service
 190         GitSSH         Service
 191         Health         Service
 192         Keepbalance    Service
 193         Keepproxy      Service
 194         Keepstore      Service
 195         Nodemanager    Service
 196         RailsAPI       Service
 197         SSO            Service
 198         WebDAVDownload Service
 199         WebDAV         Service
 200         WebShell       Service
 201         Websocket      Service
 202         Workbench1     Service
 203         Workbench2     Service
 204 }
 205
 206 type Service struct {
 207         InternalURLs map[URL]ServiceInstance
 208         ExternalURL  URL
 209 }
 210
 211 // URL is a url.URL that is also usable as a JSON key/value.
 212 type URL url.URL
 213
 214 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 215 // used as a JSON key/value.
 216 func (su *URL) UnmarshalText(text []byte) error {
 217         u, err := url.Parse(string(text))
 218         if err == nil {
 219                 *su = URL(*u)
 220         }
 221         return err
 222 }
 223
 224 func (su URL) MarshalText() ([]byte, error) {
 225         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 226 }
 227
 228 type ServiceInstance struct{}
 229
 230 type PostgreSQL struct {
 231         Connection     PostgreSQLConnection
 232         ConnectionPool int
 233 }
 234
 235 type PostgreSQLConnection map[string]string
 236
 237 type RemoteCluster struct {
 238         Host          string
 239         Proxy         bool
 240         Scheme        string
 241         Insecure      bool
 242         ActivateUsers bool
 243 }
 244
 245 type InstanceType struct {
 246         Name            string
 247         ProviderType    string
 248         VCPUs           int
 249         RAM             ByteSize
 250         Scratch         ByteSize
 251         IncludedScratch ByteSize
 252         AddedScratch    ByteSize
 253         Price           float64
 254         Preemptible     bool
 255 }
 256
 257 type ContainersConfig struct {
 258         CloudVMs                    CloudVMsConfig
 259         CrunchRunCommand            string
 260         CrunchRunArgumentsList      []string
 261         DefaultKeepCacheRAM         ByteSize
 262         DispatchPrivateKey          string
 263         LogReuseDecisions           bool
 264         MaxComputeVMs               int
 265         MaxDispatchAttempts         int
 266         MaxRetryAttempts            int
 267         MinRetryPeriod              Duration
 268         ReserveExtraRAM             ByteSize
 269         StaleLockTimeout            Duration
 270         SupportedDockerImageFormats StringSet
 271         UsePreemptibleInstances     bool
 272
 273         JobsAPI struct {
 274                 Enable                  string
 275                 GitInternalDir          string
 276                 DefaultDockerImage      string
 277                 CrunchJobWrapper        string
 278                 CrunchJobUser           string
 279                 CrunchRefreshTrigger    string
 280                 ReuseJobIfOutputsDiffer bool
 281         }
 282         Logging struct {
 283                 MaxAge                       Duration
 284                 LogBytesPerEvent             int
 285                 LogSecondsBetweenEvents      int
 286                 LogThrottlePeriod            Duration
 287                 LogThrottleBytes             int
 288                 LogThrottleLines             int
 289                 LimitLogBytesPerJob          int
 290                 LogPartialLineThrottlePeriod Duration
 291                 LogUpdatePeriod              Duration
 292                 LogUpdateSize                ByteSize
 293         }
 294         SLURM struct {
 295                 PrioritySpread      int64
 296                 SbatchArgumentsList []string
 297                 Managed             struct {
 298                         DNSServerConfDir       string
 299                         DNSServerConfTemplate  string
 300                         DNSServerReloadCommand string
 301                         DNSServerUpdateCommand string
 302                         ComputeNodeDomain      string
 303                         ComputeNodeNameservers StringSet
 304                         AssignNodeHostname     string
 305                 }
 306         }
 307 }
 308
 309 type CloudVMsConfig struct {
 310         Enable bool
 311
 312         BootProbeCommand     string
 313         ImageID              string
 314         MaxCloudOpsPerSecond int
 315         MaxProbesPerSecond   int
 316         PollInterval         Duration
 317         ProbeInterval        Duration
 318         SSHPort              string
 319         SyncInterval         Duration
 320         TimeoutBooting       Duration
 321         TimeoutIdle          Duration
 322         TimeoutProbe         Duration
 323         TimeoutShutdown      Duration
 324         TimeoutSignal        Duration
 325         TimeoutTERM          Duration
 326         ResourceTags         map[string]string
 327         TagKeyPrefix         string
 328
 329         Driver           string
 330         DriverParameters json.RawMessage
 331 }
 332
 333 type InstanceTypeMap map[string]InstanceType
 334
 335 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 336
 337 // UnmarshalJSON handles old config files that provide an array of
 338 // instance types instead of a hash.
 339 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 340         if len(data) > 0 && data[0] == '[' {
 341                 var arr []InstanceType
 342                 err := json.Unmarshal(data, &arr)
 343                 if err != nil {
 344                         return err
 345                 }
 346                 if len(arr) == 0 {
 347                         *it = nil
 348                         return nil
 349                 }
 350                 *it = make(map[string]InstanceType, len(arr))
 351                 for _, t := range arr {
 352                         if _, ok := (*it)[t.Name]; ok {
 353                                 return errDuplicateInstanceTypeName
 354                         }
 355                         if t.ProviderType == "" {
 356                                 t.ProviderType = t.Name
 357                         }
 358                         if t.Scratch == 0 {
 359                                 t.Scratch = t.IncludedScratch + t.AddedScratch
 360                         } else if t.AddedScratch == 0 {
 361                                 t.AddedScratch = t.Scratch - t.IncludedScratch
 362                         } else if t.IncludedScratch == 0 {
 363                                 t.IncludedScratch = t.Scratch - t.AddedScratch
 364                         }
 365
 366                         if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 367                                 return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 368                         }
 369                         (*it)[t.Name] = t
 370                 }
 371                 return nil
 372         }
 373         var hash map[string]InstanceType
 374         err := json.Unmarshal(data, &hash)
 375         if err != nil {
 376                 return err
 377         }
 378         // Fill in Name field (and ProviderType field, if not
 379         // specified) using hash key.
 380         *it = InstanceTypeMap(hash)
 381         for name, t := range *it {
 382                 t.Name = name
 383                 if t.ProviderType == "" {
 384                         t.ProviderType = name
 385                 }
 386                 (*it)[name] = t
 387         }
 388         return nil
 389 }
 390
 391 type StringSet map[string]struct{}
 392
 393 // UnmarshalJSON handles old config files that provide an array of
 394 // instance types instead of a hash.
 395 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 396         if len(data) > 0 && data[0] == '[' {
 397                 var arr []string
 398                 err := json.Unmarshal(data, &arr)
 399                 if err != nil {
 400                         return err
 401                 }
 402                 if len(arr) == 0 {
 403                         *ss = nil
 404                         return nil
 405                 }
 406                 *ss = make(map[string]struct{}, len(arr))
 407                 for _, t := range arr {
 408                         (*ss)[t] = struct{}{}
 409                 }
 410                 return nil
 411         }
 412         var hash map[string]struct{}
 413         err := json.Unmarshal(data, &hash)
 414         if err != nil {
 415                 return err
 416         }
 417         *ss = make(map[string]struct{}, len(hash))
 418         for t, _ := range hash {
 419                 (*ss)[t] = struct{}{}
 420         }
 421
 422         return nil
 423 }
 424
 425 type ServiceName string
 426
 427 const (
 428         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 429         ServiceNameController    ServiceName = "arvados-controller"
 430         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 431         ServiceNameHealth        ServiceName = "arvados-health"
 432         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 433         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 434         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 435         ServiceNameWebsocket     ServiceName = "arvados-ws"
 436         ServiceNameKeepbalance   ServiceName = "keep-balance"
 437         ServiceNameKeepweb       ServiceName = "keep-web"
 438         ServiceNameKeepproxy     ServiceName = "keepproxy"
 439         ServiceNameKeepstore     ServiceName = "keepstore"
 440 )
 441
 442 // Map returns all services as a map, suitable for iterating over all
 443 // services or looking up a service by name.
 444 func (svcs Services) Map() map[ServiceName]Service {
 445         return map[ServiceName]Service{
 446                 ServiceNameRailsAPI:      svcs.RailsAPI,
 447                 ServiceNameController:    svcs.Controller,
 448                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 449                 ServiceNameHealth:        svcs.Health,
 450                 ServiceNameNodemanager:   svcs.Nodemanager,
 451                 ServiceNameWorkbench1:    svcs.Workbench1,
 452                 ServiceNameWorkbench2:    svcs.Workbench2,
 453                 ServiceNameWebsocket:     svcs.Websocket,
 454                 ServiceNameKeepbalance:   svcs.Keepbalance,
 455                 ServiceNameKeepweb:       svcs.WebDAV,
 456                 ServiceNameKeepproxy:     svcs.Keepproxy,
 457                 ServiceNameKeepstore:     svcs.Keepstore,
 458         }
 459 }