sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.curoverse.com/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters map[string]Cluster
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         if cc, ok := sc.Clusters[clusterID]; !ok {
  53                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  54         } else {
  55                 cc.ClusterID = clusterID
  56                 return &cc, nil
  57         }
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxBlockEntries      int
  64         MaxCollectionEntries int
  65         MaxCollectionBytes   int64
  66         MaxPermissionEntries int
  67         MaxUUIDEntries       int
  68 }
  69 type Cluster struct {
  70         ClusterID       string `json:"-"`
  71         ManagementToken string
  72         SystemRootToken string
  73         Services        Services
  74         InstanceTypes   InstanceTypeMap
  75         Containers      ContainersConfig
  76         RemoteClusters  map[string]RemoteCluster
  77         PostgreSQL      PostgreSQL
  78
  79         API struct {
  80                 AsyncPermissionsUpdateInterval Duration
  81                 DisabledAPIs                   StringSet
  82                 MaxIndexDatabaseRead           int
  83                 MaxItemsPerResponse            int
  84                 MaxRequestAmplification        int
  85                 MaxRequestSize                 int
  86                 RailsSessionSecretToken        string
  87                 RequestTimeout                 Duration
  88                 SendTimeout                    Duration
  89                 WebsocketClientEventQueue      int
  90                 WebsocketServerEventQueue      int
  91                 KeepServiceRequestTimeout      Duration
  92         }
  93         AuditLogs struct {
  94                 MaxAge             Duration
  95                 MaxDeleteBatch     int
  96                 UnloggedAttributes StringSet
  97         }
  98         Collections struct {
  99                 BlobSigning          bool
 100                 BlobSigningKey       string
 101                 BlobSigningTTL       Duration
 102                 CollectionVersioning bool
 103                 DefaultTrashLifetime Duration
 104                 DefaultReplication   int
 105                 ManagedProperties    map[string]struct {
 106                         Value     interface{}
 107                         Function  string
 108                         Protected bool
 109                 }
 110                 PreserveVersionIfIdle Duration
 111                 TrashSweepInterval    Duration
 112                 TrustAllContent       bool
 113
 114                 WebDAVCache WebDAVCacheConfig
 115         }
 116         Git struct {
 117                 GitCommand   string
 118                 GitoliteHome string
 119                 Repositories string
 120         }
 121         Login struct {
 122                 ProviderAppSecret  string
 123                 ProviderAppID      string
 124                 LoginCluster       string
 125                 RemoteTokenRefresh Duration
 126         }
 127         Mail struct {
 128                 MailchimpAPIKey                string
 129                 MailchimpListID                string
 130                 SendUserSetupNotificationEmail bool
 131                 IssueReporterEmailFrom         string
 132                 IssueReporterEmailTo           string
 133                 SupportEmailAddress            string
 134                 EmailFrom                      string
 135         }
 136         SystemLogs struct {
 137                 LogLevel                string
 138                 Format                  string
 139                 MaxRequestLogParamsSize int
 140         }
 141         TLS struct {
 142                 Certificate string
 143                 Key         string
 144                 Insecure    bool
 145         }
 146         Users struct {
 147                 AnonymousUserToken                    string
 148                 AdminNotifierEmailFrom                string
 149                 AutoAdminFirstUser                    bool
 150                 AutoAdminUserWithEmail                string
 151                 AutoSetupNewUsers                     bool
 152                 AutoSetupNewUsersWithRepository       bool
 153                 AutoSetupNewUsersWithVmUUID           string
 154                 AutoSetupUsernameBlacklist            StringSet
 155                 EmailSubjectPrefix                    string
 156                 NewInactiveUserNotificationRecipients StringSet
 157                 NewUserNotificationRecipients         StringSet
 158                 NewUsersAreActive                     bool
 159                 UserNotifierEmailFrom                 string
 160                 UserProfileNotificationAddress        string
 161         }
 162         Workbench struct {
 163                 ActivationContactLink            string
 164                 APIClientConnectTimeout          Duration
 165                 APIClientReceiveTimeout          Duration
 166                 APIResponseCompression           bool
 167                 ApplicationMimetypesWithViewIcon StringSet
 168                 ArvadosDocsite                   string
 169                 ArvadosPublicDataDocURL          string
 170                 DefaultOpenIdPrefix              string
 171                 EnableGettingStartedPopup        bool
 172                 EnablePublicProjectsPage         bool
 173                 FileViewersConfigURL             string
 174                 LogViewerMaxBytes                ByteSize
 175                 MultiSiteSearch                  string
 176                 ProfilingEnabled                 bool
 177                 Repositories                     bool
 178                 RepositoryCache                  string
 179                 RunningJobLogRecordsToFetch      int
 180                 SecretKeyBase                    string
 181                 ShowRecentCollectionsOnDashboard bool
 182                 ShowUserAgreementInline          bool
 183                 ShowUserNotifications            bool
 184                 SiteName                         string
 185                 Theme                            string
 186                 UserProfileFormFields            map[string]struct {
 187                         Type                 string
 188                         FormFieldTitle       string
 189                         FormFieldDescription string
 190                         Required             bool
 191                         Position             int
 192                         Options              map[string]struct{}
 193                 }
 194                 UserProfileFormMessage string
 195                 VocabularyURL          string
 196         }
 197
 198         EnableBetaController14287 bool
 199 }
 200
 201 type Services struct {
 202         Composer       Service
 203         Controller     Service
 204         DispatchCloud  Service
 205         GitHTTP        Service
 206         GitSSH         Service
 207         Health         Service
 208         Keepbalance    Service
 209         Keepproxy      Service
 210         Keepstore      Service
 211         Nodemanager    Service
 212         RailsAPI       Service
 213         SSO            Service
 214         WebDAVDownload Service
 215         WebDAV         Service
 216         WebShell       Service
 217         Websocket      Service
 218         Workbench1     Service
 219         Workbench2     Service
 220 }
 221
 222 type Service struct {
 223         InternalURLs map[URL]ServiceInstance
 224         ExternalURL  URL
 225 }
 226
 227 // URL is a url.URL that is also usable as a JSON key/value.
 228 type URL url.URL
 229
 230 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 231 // used as a JSON key/value.
 232 func (su *URL) UnmarshalText(text []byte) error {
 233         u, err := url.Parse(string(text))
 234         if err == nil {
 235                 *su = URL(*u)
 236         }
 237         return err
 238 }
 239
 240 func (su URL) MarshalText() ([]byte, error) {
 241         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 242 }
 243
 244 type ServiceInstance struct{}
 245
 246 type PostgreSQL struct {
 247         Connection     PostgreSQLConnection
 248         ConnectionPool int
 249 }
 250
 251 type PostgreSQLConnection map[string]string
 252
 253 type RemoteCluster struct {
 254         Host          string
 255         Proxy         bool
 256         Scheme        string
 257         Insecure      bool
 258         ActivateUsers bool
 259 }
 260
 261 type InstanceType struct {
 262         Name            string
 263         ProviderType    string
 264         VCPUs           int
 265         RAM             ByteSize
 266         Scratch         ByteSize
 267         IncludedScratch ByteSize
 268         AddedScratch    ByteSize
 269         Price           float64
 270         Preemptible     bool
 271 }
 272
 273 type ContainersConfig struct {
 274         CloudVMs                    CloudVMsConfig
 275         CrunchRunCommand            string
 276         CrunchRunArgumentsList      []string
 277         DefaultKeepCacheRAM         ByteSize
 278         DispatchPrivateKey          string
 279         LogReuseDecisions           bool
 280         MaxComputeVMs               int
 281         MaxDispatchAttempts         int
 282         MaxRetryAttempts            int
 283         MinRetryPeriod              Duration
 284         ReserveExtraRAM             ByteSize
 285         StaleLockTimeout            Duration
 286         SupportedDockerImageFormats StringSet
 287         UsePreemptibleInstances     bool
 288
 289         JobsAPI struct {
 290                 Enable         string
 291                 GitInternalDir string
 292         }
 293         Logging struct {
 294                 MaxAge                       Duration
 295                 LogBytesPerEvent             int
 296                 LogSecondsBetweenEvents      int
 297                 LogThrottlePeriod            Duration
 298                 LogThrottleBytes             int
 299                 LogThrottleLines             int
 300                 LimitLogBytesPerJob          int
 301                 LogPartialLineThrottlePeriod Duration
 302                 LogUpdatePeriod              Duration
 303                 LogUpdateSize                ByteSize
 304         }
 305         SLURM struct {
 306                 PrioritySpread             int64
 307                 SbatchArgumentsList        []string
 308                 SbatchEnvironmentVariables map[string]string
 309                 Managed                    struct {
 310                         DNSServerConfDir       string
 311                         DNSServerConfTemplate  string
 312                         DNSServerReloadCommand string
 313                         DNSServerUpdateCommand string
 314                         ComputeNodeDomain      string
 315                         ComputeNodeNameservers StringSet
 316                         AssignNodeHostname     string
 317                 }
 318         }
 319 }
 320
 321 type CloudVMsConfig struct {
 322         Enable bool
 323
 324         BootProbeCommand     string
 325         ImageID              string
 326         MaxCloudOpsPerSecond int
 327         MaxProbesPerSecond   int
 328         PollInterval         Duration
 329         ProbeInterval        Duration
 330         SSHPort              string
 331         SyncInterval         Duration
 332         TimeoutBooting       Duration
 333         TimeoutIdle          Duration
 334         TimeoutProbe         Duration
 335         TimeoutShutdown      Duration
 336         TimeoutSignal        Duration
 337         TimeoutTERM          Duration
 338         ResourceTags         map[string]string
 339         TagKeyPrefix         string
 340
 341         Driver           string
 342         DriverParameters json.RawMessage
 343 }
 344
 345 type InstanceTypeMap map[string]InstanceType
 346
 347 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 348
 349 // UnmarshalJSON handles old config files that provide an array of
 350 // instance types instead of a hash.
 351 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 352         if len(data) > 0 && data[0] == '[' {
 353                 var arr []InstanceType
 354                 err := json.Unmarshal(data, &arr)
 355                 if err != nil {
 356                         return err
 357                 }
 358                 if len(arr) == 0 {
 359                         *it = nil
 360                         return nil
 361                 }
 362                 *it = make(map[string]InstanceType, len(arr))
 363                 for _, t := range arr {
 364                         if _, ok := (*it)[t.Name]; ok {
 365                                 return errDuplicateInstanceTypeName
 366                         }
 367                         if t.ProviderType == "" {
 368                                 t.ProviderType = t.Name
 369                         }
 370                         if t.Scratch == 0 {
 371                                 t.Scratch = t.IncludedScratch + t.AddedScratch
 372                         } else if t.AddedScratch == 0 {
 373                                 t.AddedScratch = t.Scratch - t.IncludedScratch
 374                         } else if t.IncludedScratch == 0 {
 375                                 t.IncludedScratch = t.Scratch - t.AddedScratch
 376                         }
 377
 378                         if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 379                                 return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 380                         }
 381                         (*it)[t.Name] = t
 382                 }
 383                 return nil
 384         }
 385         var hash map[string]InstanceType
 386         err := json.Unmarshal(data, &hash)
 387         if err != nil {
 388                 return err
 389         }
 390         // Fill in Name field (and ProviderType field, if not
 391         // specified) using hash key.
 392         *it = InstanceTypeMap(hash)
 393         for name, t := range *it {
 394                 t.Name = name
 395                 if t.ProviderType == "" {
 396                         t.ProviderType = name
 397                 }
 398                 (*it)[name] = t
 399         }
 400         return nil
 401 }
 402
 403 type StringSet map[string]struct{}
 404
 405 // UnmarshalJSON handles old config files that provide an array of
 406 // instance types instead of a hash.
 407 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 408         if len(data) > 0 && data[0] == '[' {
 409                 var arr []string
 410                 err := json.Unmarshal(data, &arr)
 411                 if err != nil {
 412                         return err
 413                 }
 414                 if len(arr) == 0 {
 415                         *ss = nil
 416                         return nil
 417                 }
 418                 *ss = make(map[string]struct{}, len(arr))
 419                 for _, t := range arr {
 420                         (*ss)[t] = struct{}{}
 421                 }
 422                 return nil
 423         }
 424         var hash map[string]struct{}
 425         err := json.Unmarshal(data, &hash)
 426         if err != nil {
 427                 return err
 428         }
 429         *ss = make(map[string]struct{}, len(hash))
 430         for t, _ := range hash {
 431                 (*ss)[t] = struct{}{}
 432         }
 433
 434         return nil
 435 }
 436
 437 type ServiceName string
 438
 439 const (
 440         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 441         ServiceNameController    ServiceName = "arvados-controller"
 442         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 443         ServiceNameHealth        ServiceName = "arvados-health"
 444         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 445         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 446         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 447         ServiceNameWebsocket     ServiceName = "arvados-ws"
 448         ServiceNameKeepbalance   ServiceName = "keep-balance"
 449         ServiceNameKeepweb       ServiceName = "keep-web"
 450         ServiceNameKeepproxy     ServiceName = "keepproxy"
 451         ServiceNameKeepstore     ServiceName = "keepstore"
 452 )
 453
 454 // Map returns all services as a map, suitable for iterating over all
 455 // services or looking up a service by name.
 456 func (svcs Services) Map() map[ServiceName]Service {
 457         return map[ServiceName]Service{
 458                 ServiceNameRailsAPI:      svcs.RailsAPI,
 459                 ServiceNameController:    svcs.Controller,
 460                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 461                 ServiceNameHealth:        svcs.Health,
 462                 ServiceNameNodemanager:   svcs.Nodemanager,
 463                 ServiceNameWorkbench1:    svcs.Workbench1,
 464                 ServiceNameWorkbench2:    svcs.Workbench2,
 465                 ServiceNameWebsocket:     svcs.Websocket,
 466                 ServiceNameKeepbalance:   svcs.Keepbalance,
 467                 ServiceNameKeepweb:       svcs.WebDAV,
 468                 ServiceNameKeepproxy:     svcs.Keepproxy,
 469                 ServiceNameKeepstore:     svcs.Keepstore,
 470         }
 471 }