sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.curoverse.com/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters map[string]Cluster
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         if cc, ok := sc.Clusters[clusterID]; !ok {
  53                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  54         } else {
  55                 cc.ClusterID = clusterID
  56                 return &cc, nil
  57         }
  58 }
  59
  60 type Cluster struct {
  61         ClusterID       string `json:"-"`
  62         ManagementToken string
  63         SystemRootToken string
  64         Services        Services
  65         InstanceTypes   InstanceTypeMap
  66         Containers      ContainersConfig
  67         RemoteClusters  map[string]RemoteCluster
  68         PostgreSQL      PostgreSQL
  69
  70         API struct {
  71                 AsyncPermissionsUpdateInterval Duration
  72                 DisabledAPIs                   StringSet
  73                 MaxIndexDatabaseRead           int
  74                 MaxItemsPerResponse            int
  75                 MaxRequestAmplification        int
  76                 MaxRequestSize                 int
  77                 RailsSessionSecretToken        string
  78                 RequestTimeout                 Duration
  79                 SendTimeout                    Duration
  80                 WebsocketClientEventQueue      int
  81                 WebsocketServerEventQueue      int
  82         }
  83         AuditLogs struct {
  84                 MaxAge             Duration
  85                 MaxDeleteBatch     int
  86                 UnloggedAttributes StringSet
  87         }
  88         Collections struct {
  89                 BlobSigning          bool
  90                 BlobSigningKey       string
  91                 BlobSigningTTL       Duration
  92                 CollectionVersioning bool
  93                 DefaultTrashLifetime Duration
  94                 DefaultReplication   int
  95                 ManagedProperties    map[string]struct {
  96                         Value     interface{}
  97                         Function  string
  98                         Protected bool
  99                 }
 100                 PreserveVersionIfIdle Duration
 101                 TrashSweepInterval    Duration
 102                 TrustAllContent       bool
 103
 104                 WebDAVCache struct {
 105                         TTL                  Duration
 106                         UUIDTTL              Duration
 107                         MaxCollectionEntries int
 108                         MaxCollectionBytes   int64
 109                         MaxPermissionEntries int
 110                         MaxUUIDEntries       int
 111                 }
 112         }
 113         Git struct {
 114                 Repositories string
 115         }
 116         Login struct {
 117                 ProviderAppSecret string
 118                 ProviderAppID     string
 119         }
 120         Mail struct {
 121                 MailchimpAPIKey                string
 122                 MailchimpListID                string
 123                 SendUserSetupNotificationEmail bool
 124                 IssueReporterEmailFrom         string
 125                 IssueReporterEmailTo           string
 126                 SupportEmailAddress            string
 127                 EmailFrom                      string
 128         }
 129         SystemLogs struct {
 130                 LogLevel                string
 131                 Format                  string
 132                 MaxRequestLogParamsSize int
 133         }
 134         TLS struct {
 135                 Certificate string
 136                 Key         string
 137                 Insecure    bool
 138         }
 139         Users struct {
 140                 AnonymousUserToken                    string
 141                 AdminNotifierEmailFrom                string
 142                 AutoAdminFirstUser                    bool
 143                 AutoAdminUserWithEmail                string
 144                 AnonymousUserToken                    string
 145                 AutoSetupNewUsers                     bool
 146                 AutoSetupNewUsersWithRepository       bool
 147                 AutoSetupNewUsersWithVmUUID           string
 148                 AutoSetupUsernameBlacklist            StringSet
 149                 EmailSubjectPrefix                    string
 150                 NewInactiveUserNotificationRecipients StringSet
 151                 NewUserNotificationRecipients         StringSet
 152                 NewUsersAreActive                     bool
 153                 UserNotifierEmailFrom                 string
 154                 UserProfileNotificationAddress        string
 155         }
 156         Workbench struct {
 157                 ActivationContactLink            string
 158                 APIClientConnectTimeout          Duration
 159                 APIClientReceiveTimeout          Duration
 160                 APIResponseCompression           bool
 161                 ApplicationMimetypesWithViewIcon StringSet
 162                 ArvadosDocsite                   string
 163                 ArvadosPublicDataDocURL          string
 164                 DefaultOpenIdPrefix              string
 165                 EnableGettingStartedPopup        bool
 166                 EnablePublicProjectsPage         bool
 167                 FileViewersConfigURL             string
 168                 LogViewerMaxBytes                ByteSize
 169                 MultiSiteSearch                  string
 170                 ProfilingEnabled                 bool
 171                 Repositories                     bool
 172                 RepositoryCache                  string
 173                 RunningJobLogRecordsToFetch      int
 174                 SecretKeyBase                    string
 175                 ShowRecentCollectionsOnDashboard bool
 176                 ShowUserAgreementInline          bool
 177                 ShowUserNotifications            bool
 178                 SiteName                         string
 179                 Theme                            string
 180                 UserProfileFormFields            map[string]struct {
 181                         Type                 string
 182                         FormFieldTitle       string
 183                         FormFieldDescription string
 184                         Required             bool
 185                         Position             int
 186                         Options              map[string]struct{}
 187                 }
 188                 UserProfileFormMessage string
 189                 VocabularyURL          string
 190         }
 191
 192         EnableBetaController14287 bool
 193 }
 194
 195 type Services struct {
 196         Composer       Service
 197         Controller     Service
 198         DispatchCloud  Service
 199         GitHTTP        Service
 200         GitSSH         Service
 201         Health         Service
 202         Keepbalance    Service
 203         Keepproxy      Service
 204         Keepstore      Service
 205         Nodemanager    Service
 206         RailsAPI       Service
 207         SSO            Service
 208         WebDAVDownload Service
 209         WebDAV         Service
 210         WebShell       Service
 211         Websocket      Service
 212         Workbench1     Service
 213         Workbench2     Service
 214 }
 215
 216 type Service struct {
 217         InternalURLs map[URL]ServiceInstance
 218         ExternalURL  URL
 219 }
 220
 221 // URL is a url.URL that is also usable as a JSON key/value.
 222 type URL url.URL
 223
 224 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 225 // used as a JSON key/value.
 226 func (su *URL) UnmarshalText(text []byte) error {
 227         u, err := url.Parse(string(text))
 228         if err == nil {
 229                 *su = URL(*u)
 230         }
 231         return err
 232 }
 233
 234 func (su URL) MarshalText() ([]byte, error) {
 235         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 236 }
 237
 238 type ServiceInstance struct{}
 239
 240 type PostgreSQL struct {
 241         Connection     PostgreSQLConnection
 242         ConnectionPool int
 243 }
 244
 245 type PostgreSQLConnection map[string]string
 246
 247 type RemoteCluster struct {
 248         Host          string
 249         Proxy         bool
 250         Scheme        string
 251         Insecure      bool
 252         ActivateUsers bool
 253 }
 254
 255 type InstanceType struct {
 256         Name            string
 257         ProviderType    string
 258         VCPUs           int
 259         RAM             ByteSize
 260         Scratch         ByteSize
 261         IncludedScratch ByteSize
 262         AddedScratch    ByteSize
 263         Price           float64
 264         Preemptible     bool
 265 }
 266
 267 type ContainersConfig struct {
 268         CloudVMs                    CloudVMsConfig
 269         CrunchRunCommand            string
 270         CrunchRunArgumentsList      []string
 271         DefaultKeepCacheRAM         ByteSize
 272         DispatchPrivateKey          string
 273         LogReuseDecisions           bool
 274         MaxComputeVMs               int
 275         MaxDispatchAttempts         int
 276         MaxRetryAttempts            int
 277         MinRetryPeriod              Duration
 278         ReserveExtraRAM             ByteSize
 279         StaleLockTimeout            Duration
 280         SupportedDockerImageFormats StringSet
 281         UsePreemptibleInstances     bool
 282
 283         JobsAPI struct {
 284                 Enable                  string
 285                 GitInternalDir          string
 286                 DefaultDockerImage      string
 287                 CrunchJobWrapper        string
 288                 CrunchJobUser           string
 289                 CrunchRefreshTrigger    string
 290                 ReuseJobIfOutputsDiffer bool
 291         }
 292         Logging struct {
 293                 MaxAge                       Duration
 294                 LogBytesPerEvent             int
 295                 LogSecondsBetweenEvents      int
 296                 LogThrottlePeriod            Duration
 297                 LogThrottleBytes             int
 298                 LogThrottleLines             int
 299                 LimitLogBytesPerJob          int
 300                 LogPartialLineThrottlePeriod Duration
 301                 LogUpdatePeriod              Duration
 302                 LogUpdateSize                ByteSize
 303         }
 304         SLURM struct {
 305                 PrioritySpread      int64
 306                 SbatchArgumentsList []string
 307                 Managed             struct {
 308                         DNSServerConfDir       string
 309                         DNSServerConfTemplate  string
 310                         DNSServerReloadCommand string
 311                         DNSServerUpdateCommand string
 312                         ComputeNodeDomain      string
 313                         ComputeNodeNameservers StringSet
 314                         AssignNodeHostname     string
 315                 }
 316         }
 317 }
 318
 319 type CloudVMsConfig struct {
 320         Enable bool
 321
 322         BootProbeCommand     string
 323         ImageID              string
 324         MaxCloudOpsPerSecond int
 325         MaxProbesPerSecond   int
 326         PollInterval         Duration
 327         ProbeInterval        Duration
 328         SSHPort              string
 329         SyncInterval         Duration
 330         TimeoutBooting       Duration
 331         TimeoutIdle          Duration
 332         TimeoutProbe         Duration
 333         TimeoutShutdown      Duration
 334         TimeoutSignal        Duration
 335         TimeoutTERM          Duration
 336         ResourceTags         map[string]string
 337         TagKeyPrefix         string
 338
 339         Driver           string
 340         DriverParameters json.RawMessage
 341 }
 342
 343 type InstanceTypeMap map[string]InstanceType
 344
 345 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 346
 347 // UnmarshalJSON handles old config files that provide an array of
 348 // instance types instead of a hash.
 349 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 350         if len(data) > 0 && data[0] == '[' {
 351                 var arr []InstanceType
 352                 err := json.Unmarshal(data, &arr)
 353                 if err != nil {
 354                         return err
 355                 }
 356                 if len(arr) == 0 {
 357                         *it = nil
 358                         return nil
 359                 }
 360                 *it = make(map[string]InstanceType, len(arr))
 361                 for _, t := range arr {
 362                         if _, ok := (*it)[t.Name]; ok {
 363                                 return errDuplicateInstanceTypeName
 364                         }
 365                         if t.ProviderType == "" {
 366                                 t.ProviderType = t.Name
 367                         }
 368                         if t.Scratch == 0 {
 369                                 t.Scratch = t.IncludedScratch + t.AddedScratch
 370                         } else if t.AddedScratch == 0 {
 371                                 t.AddedScratch = t.Scratch - t.IncludedScratch
 372                         } else if t.IncludedScratch == 0 {
 373                                 t.IncludedScratch = t.Scratch - t.AddedScratch
 374                         }
 375
 376                         if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 377                                 return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 378                         }
 379                         (*it)[t.Name] = t
 380                 }
 381                 return nil
 382         }
 383         var hash map[string]InstanceType
 384         err := json.Unmarshal(data, &hash)
 385         if err != nil {
 386                 return err
 387         }
 388         // Fill in Name field (and ProviderType field, if not
 389         // specified) using hash key.
 390         *it = InstanceTypeMap(hash)
 391         for name, t := range *it {
 392                 t.Name = name
 393                 if t.ProviderType == "" {
 394                         t.ProviderType = name
 395                 }
 396                 (*it)[name] = t
 397         }
 398         return nil
 399 }
 400
 401 type StringSet map[string]struct{}
 402
 403 // UnmarshalJSON handles old config files that provide an array of
 404 // instance types instead of a hash.
 405 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 406         if len(data) > 0 && data[0] == '[' {
 407                 var arr []string
 408                 err := json.Unmarshal(data, &arr)
 409                 if err != nil {
 410                         return err
 411                 }
 412                 if len(arr) == 0 {
 413                         *ss = nil
 414                         return nil
 415                 }
 416                 *ss = make(map[string]struct{}, len(arr))
 417                 for _, t := range arr {
 418                         (*ss)[t] = struct{}{}
 419                 }
 420                 return nil
 421         }
 422         var hash map[string]struct{}
 423         err := json.Unmarshal(data, &hash)
 424         if err != nil {
 425                 return err
 426         }
 427         *ss = make(map[string]struct{}, len(hash))
 428         for t, _ := range hash {
 429                 (*ss)[t] = struct{}{}
 430         }
 431
 432         return nil
 433 }
 434
 435 type ServiceName string
 436
 437 const (
 438         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 439         ServiceNameController    ServiceName = "arvados-controller"
 440         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 441         ServiceNameHealth        ServiceName = "arvados-health"
 442         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 443         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 444         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 445         ServiceNameWebsocket     ServiceName = "arvados-ws"
 446         ServiceNameKeepbalance   ServiceName = "keep-balance"
 447         ServiceNameKeepweb       ServiceName = "keep-web"
 448         ServiceNameKeepproxy     ServiceName = "keepproxy"
 449         ServiceNameKeepstore     ServiceName = "keepstore"
 450 )
 451
 452 // Map returns all services as a map, suitable for iterating over all
 453 // services or looking up a service by name.
 454 func (svcs Services) Map() map[ServiceName]Service {
 455         return map[ServiceName]Service{
 456                 ServiceNameRailsAPI:      svcs.RailsAPI,
 457                 ServiceNameController:    svcs.Controller,
 458                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 459                 ServiceNameHealth:        svcs.Health,
 460                 ServiceNameNodemanager:   svcs.Nodemanager,
 461                 ServiceNameWorkbench1:    svcs.Workbench1,
 462                 ServiceNameWorkbench2:    svcs.Workbench2,
 463                 ServiceNameWebsocket:     svcs.Websocket,
 464                 ServiceNameKeepbalance:   svcs.Keepbalance,
 465                 ServiceNameKeepweb:       svcs.WebDAV,
 466                 ServiceNameKeepproxy:     svcs.Keepproxy,
 467                 ServiceNameKeepstore:     svcs.Keepstore,
 468         }
 469 }