sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.curoverse.com/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters map[string]Cluster
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         if cc, ok := sc.Clusters[clusterID]; !ok {
  53                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  54         } else {
  55                 cc.ClusterID = clusterID
  56                 return &cc, nil
  57         }
  58 }
  59
  60 type Cluster struct {
  61         ClusterID       string `json:"-"`
  62         ManagementToken string
  63         SystemRootToken string
  64         Services        Services
  65         InstanceTypes   InstanceTypeMap
  66         Containers      ContainersConfig
  67         RemoteClusters  map[string]RemoteCluster
  68         PostgreSQL      PostgreSQL
  69
  70         API struct {
  71                 AsyncPermissionsUpdateInterval Duration
  72                 DisabledAPIs                   StringSet
  73                 MaxIndexDatabaseRead           int
  74                 MaxItemsPerResponse            int
  75                 MaxRequestAmplification        int
  76                 MaxRequestSize                 int
  77                 RailsSessionSecretToken        string
  78                 RequestTimeout                 Duration
  79                 SendTimeout                    Duration
  80                 WebsocketClientEventQueue      int
  81                 WebsocketServerEventQueue      int
  82         }
  83         AuditLogs struct {
  84                 MaxAge             Duration
  85                 MaxDeleteBatch     int
  86                 UnloggedAttributes StringSet
  87         }
  88         Collections struct {
  89                 BlobSigning          bool
  90                 BlobSigningKey       string
  91                 BlobSigningTTL       Duration
  92                 CollectionVersioning bool
  93                 DefaultTrashLifetime Duration
  94                 DefaultReplication   int
  95                 ManagedProperties    map[string]struct {
  96                         Value     interface{}
  97                         Function  string
  98                         Protected bool
  99                 }
 100                 PreserveVersionIfIdle Duration
 101                 TrashSweepInterval    Duration
 102                 TrustAllContent       bool
 103         }
 104         Git struct {
 105                 Repositories string
 106         }
 107         Login struct {
 108                 ProviderAppSecret string
 109                 ProviderAppID     string
 110         }
 111         Mail struct {
 112                 MailchimpAPIKey                string
 113                 MailchimpListID                string
 114                 SendUserSetupNotificationEmail bool
 115                 IssueReporterEmailFrom         string
 116                 IssueReporterEmailTo           string
 117                 SupportEmailAddress            string
 118                 EmailFrom                      string
 119         }
 120         SystemLogs struct {
 121                 LogLevel                string
 122                 Format                  string
 123                 MaxRequestLogParamsSize int
 124         }
 125         TLS struct {
 126                 Certificate string
 127                 Key         string
 128                 Insecure    bool
 129         }
 130         Users struct {
 131                 AnonymousUserToken                    string
 132                 AdminNotifierEmailFrom                string
 133                 AutoAdminFirstUser                    bool
 134                 AutoAdminUserWithEmail                string
 135                 AutoSetupNewUsers                     bool
 136                 AutoSetupNewUsersWithRepository       bool
 137                 AutoSetupNewUsersWithVmUUID           string
 138                 AutoSetupUsernameBlacklist            StringSet
 139                 EmailSubjectPrefix                    string
 140                 NewInactiveUserNotificationRecipients StringSet
 141                 NewUserNotificationRecipients         StringSet
 142                 NewUsersAreActive                     bool
 143                 UserNotifierEmailFrom                 string
 144                 UserProfileNotificationAddress        string
 145         }
 146         Workbench struct {
 147                 ActivationContactLink            string
 148                 APIClientConnectTimeout          Duration
 149                 APIClientReceiveTimeout          Duration
 150                 APIResponseCompression           bool
 151                 ApplicationMimetypesWithViewIcon StringSet
 152                 ArvadosDocsite                   string
 153                 ArvadosPublicDataDocURL          string
 154                 DefaultOpenIdPrefix              string
 155                 EnableGettingStartedPopup        bool
 156                 EnablePublicProjectsPage         bool
 157                 FileViewersConfigURL             string
 158                 LogViewerMaxBytes                ByteSize
 159                 MultiSiteSearch                  string
 160                 ProfilingEnabled                 bool
 161                 Repositories                     bool
 162                 RepositoryCache                  string
 163                 RunningJobLogRecordsToFetch      int
 164                 SecretKeyBase                    string
 165                 ShowRecentCollectionsOnDashboard bool
 166                 ShowUserAgreementInline          bool
 167                 ShowUserNotifications            bool
 168                 SiteName                         string
 169                 Theme                            string
 170                 UserProfileFormFields            map[string]struct {
 171                         Type                 string
 172                         FormFieldTitle       string
 173                         FormFieldDescription string
 174                         Required             bool
 175                         Position             int
 176                         Options              map[string]struct{}
 177                 }
 178                 UserProfileFormMessage string
 179                 VocabularyURL          string
 180         }
 181
 182         EnableBetaController14287 bool
 183 }
 184
 185 type Services struct {
 186         Composer       Service
 187         Controller     Service
 188         DispatchCloud  Service
 189         GitHTTP        Service
 190         GitSSH         Service
 191         Health         Service
 192         Keepbalance    Service
 193         Keepproxy      Service
 194         Keepstore      Service
 195         Nodemanager    Service
 196         RailsAPI       Service
 197         SSO            Service
 198         WebDAVDownload Service
 199         WebDAV         Service
 200         WebShell       Service
 201         Websocket      Service
 202         Workbench1     Service
 203         Workbench2     Service
 204 }
 205
 206 type Service struct {
 207         InternalURLs map[URL]ServiceInstance
 208         ExternalURL  URL
 209 }
 210
 211 // URL is a url.URL that is also usable as a JSON key/value.
 212 type URL url.URL
 213
 214 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 215 // used as a JSON key/value.
 216 func (su *URL) UnmarshalText(text []byte) error {
 217         u, err := url.Parse(string(text))
 218         if err == nil {
 219                 *su = URL(*u)
 220         }
 221         return err
 222 }
 223
 224 func (su URL) MarshalText() ([]byte, error) {
 225         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 226 }
 227
 228 type ServiceInstance struct{}
 229
 230 type PostgreSQL struct {
 231         Connection     PostgreSQLConnection
 232         ConnectionPool int
 233 }
 234
 235 type PostgreSQLConnection map[string]string
 236
 237 type RemoteCluster struct {
 238         Host          string
 239         Proxy         bool
 240         Scheme        string
 241         Insecure      bool
 242         ActivateUsers bool
 243 }
 244
 245 type InstanceType struct {
 246         Name            string
 247         ProviderType    string
 248         VCPUs           int
 249         RAM             ByteSize
 250         Scratch         ByteSize
 251         IncludedScratch ByteSize
 252         AddedScratch    ByteSize
 253         Price           float64
 254         Preemptible     bool
 255 }
 256
 257 type ContainersConfig struct {
 258         CloudVMs                    CloudVMsConfig
 259         CrunchRunCommand            string
 260         CrunchRunArgumentsList      []string
 261         DefaultKeepCacheRAM         ByteSize
 262         DispatchPrivateKey          string
 263         LogReuseDecisions           bool
 264         MaxComputeVMs               int
 265         MaxDispatchAttempts         int
 266         MaxRetryAttempts            int
 267         MinRetryPeriod              Duration
 268         ReserveExtraRAM             ByteSize
 269         StaleLockTimeout            Duration
 270         SupportedDockerImageFormats StringSet
 271         UsePreemptibleInstances     bool
 272
 273         JobsAPI struct {
 274                 Enable                  string
 275                 GitInternalDir          string
 276                 DefaultDockerImage      string
 277                 CrunchJobWrapper        string
 278                 CrunchJobUser           string
 279                 CrunchRefreshTrigger    string
 280                 ReuseJobIfOutputsDiffer bool
 281         }
 282         Logging struct {
 283                 MaxAge                       Duration
 284                 LogBytesPerEvent             int
 285                 LogSecondsBetweenEvents      int
 286                 LogThrottlePeriod            Duration
 287                 LogThrottleBytes             int
 288                 LogThrottleLines             int
 289                 LimitLogBytesPerJob          int
 290                 LogPartialLineThrottlePeriod Duration
 291                 LogUpdatePeriod              Duration
 292                 LogUpdateSize                ByteSize
 293         }
 294         SLURM struct {
 295                 PrioritySpread      int64
 296                 SbatchArgumentsList []string
 297                 KeepServices        map[string]Service
 298                 Managed             struct {
 299                         DNSServerConfDir       string
 300                         DNSServerConfTemplate  string
 301                         DNSServerReloadCommand string
 302                         DNSServerUpdateCommand string
 303                         ComputeNodeDomain      string
 304                         ComputeNodeNameservers StringSet
 305                         AssignNodeHostname     string
 306                 }
 307         }
 308 }
 309
 310 type CloudVMsConfig struct {
 311         Enable bool
 312
 313         BootProbeCommand     string
 314         ImageID              string
 315         MaxCloudOpsPerSecond int
 316         MaxProbesPerSecond   int
 317         PollInterval         Duration
 318         ProbeInterval        Duration
 319         SSHPort              string
 320         SyncInterval         Duration
 321         TimeoutBooting       Duration
 322         TimeoutIdle          Duration
 323         TimeoutProbe         Duration
 324         TimeoutShutdown      Duration
 325         TimeoutSignal        Duration
 326         TimeoutTERM          Duration
 327         ResourceTags         map[string]string
 328         TagKeyPrefix         string
 329
 330         Driver           string
 331         DriverParameters json.RawMessage
 332 }
 333
 334 type InstanceTypeMap map[string]InstanceType
 335
 336 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 337
 338 // UnmarshalJSON handles old config files that provide an array of
 339 // instance types instead of a hash.
 340 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 341         if len(data) > 0 && data[0] == '[' {
 342                 var arr []InstanceType
 343                 err := json.Unmarshal(data, &arr)
 344                 if err != nil {
 345                         return err
 346                 }
 347                 if len(arr) == 0 {
 348                         *it = nil
 349                         return nil
 350                 }
 351                 *it = make(map[string]InstanceType, len(arr))
 352                 for _, t := range arr {
 353                         if _, ok := (*it)[t.Name]; ok {
 354                                 return errDuplicateInstanceTypeName
 355                         }
 356                         if t.ProviderType == "" {
 357                                 t.ProviderType = t.Name
 358                         }
 359                         if t.Scratch == 0 {
 360                                 t.Scratch = t.IncludedScratch + t.AddedScratch
 361                         } else if t.AddedScratch == 0 {
 362                                 t.AddedScratch = t.Scratch - t.IncludedScratch
 363                         } else if t.IncludedScratch == 0 {
 364                                 t.IncludedScratch = t.Scratch - t.AddedScratch
 365                         }
 366
 367                         if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 368                                 return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 369                         }
 370                         (*it)[t.Name] = t
 371                 }
 372                 return nil
 373         }
 374         var hash map[string]InstanceType
 375         err := json.Unmarshal(data, &hash)
 376         if err != nil {
 377                 return err
 378         }
 379         // Fill in Name field (and ProviderType field, if not
 380         // specified) using hash key.
 381         *it = InstanceTypeMap(hash)
 382         for name, t := range *it {
 383                 t.Name = name
 384                 if t.ProviderType == "" {
 385                         t.ProviderType = name
 386                 }
 387                 (*it)[name] = t
 388         }
 389         return nil
 390 }
 391
 392 type StringSet map[string]struct{}
 393
 394 // UnmarshalJSON handles old config files that provide an array of
 395 // instance types instead of a hash.
 396 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 397         if len(data) > 0 && data[0] == '[' {
 398                 var arr []string
 399                 err := json.Unmarshal(data, &arr)
 400                 if err != nil {
 401                         return err
 402                 }
 403                 if len(arr) == 0 {
 404                         *ss = nil
 405                         return nil
 406                 }
 407                 *ss = make(map[string]struct{}, len(arr))
 408                 for _, t := range arr {
 409                         (*ss)[t] = struct{}{}
 410                 }
 411                 return nil
 412         }
 413         var hash map[string]struct{}
 414         err := json.Unmarshal(data, &hash)
 415         if err != nil {
 416                 return err
 417         }
 418         *ss = make(map[string]struct{}, len(hash))
 419         for t, _ := range hash {
 420                 (*ss)[t] = struct{}{}
 421         }
 422
 423         return nil
 424 }
 425
 426 type ServiceName string
 427
 428 const (
 429         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 430         ServiceNameController    ServiceName = "arvados-controller"
 431         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 432         ServiceNameHealth        ServiceName = "arvados-health"
 433         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 434         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 435         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 436         ServiceNameWebsocket     ServiceName = "arvados-ws"
 437         ServiceNameKeepbalance   ServiceName = "keep-balance"
 438         ServiceNameKeepweb       ServiceName = "keep-web"
 439         ServiceNameKeepproxy     ServiceName = "keepproxy"
 440         ServiceNameKeepstore     ServiceName = "keepstore"
 441 )
 442
 443 // Map returns all services as a map, suitable for iterating over all
 444 // services or looking up a service by name.
 445 func (svcs Services) Map() map[ServiceName]Service {
 446         return map[ServiceName]Service{
 447                 ServiceNameRailsAPI:      svcs.RailsAPI,
 448                 ServiceNameController:    svcs.Controller,
 449                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 450                 ServiceNameHealth:        svcs.Health,
 451                 ServiceNameNodemanager:   svcs.Nodemanager,
 452                 ServiceNameWorkbench1:    svcs.Workbench1,
 453                 ServiceNameWorkbench2:    svcs.Workbench2,
 454                 ServiceNameWebsocket:     svcs.Websocket,
 455                 ServiceNameKeepbalance:   svcs.Keepbalance,
 456                 ServiceNameKeepweb:       svcs.WebDAV,
 457                 ServiceNameKeepproxy:     svcs.Keepproxy,
 458                 ServiceNameKeepstore:     svcs.Keepstore,
 459         }
 460 }