sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.curoverse.com/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters map[string]Cluster
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         if cc, ok := sc.Clusters[clusterID]; !ok {
  53                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  54         } else {
  55                 cc.ClusterID = clusterID
  56                 return &cc, nil
  57         }
  58 }
  59
  60 type WebDAVCacheConfig struct {
  61         TTL                  Duration
  62         UUIDTTL              Duration
  63         MaxBlockEntries      int
  64         MaxCollectionEntries int
  65         MaxCollectionBytes   int64
  66         MaxPermissionEntries int
  67         MaxUUIDEntries       int
  68 }
  69 type Cluster struct {
  70         ClusterID       string `json:"-"`
  71         ManagementToken string
  72         SystemRootToken string
  73         Services        Services
  74         InstanceTypes   InstanceTypeMap
  75         Containers      ContainersConfig
  76         RemoteClusters  map[string]RemoteCluster
  77         PostgreSQL      PostgreSQL
  78
  79         API struct {
  80                 AsyncPermissionsUpdateInterval Duration
  81                 DisabledAPIs                   StringSet
  82                 MaxIndexDatabaseRead           int
  83                 MaxItemsPerResponse            int
  84                 MaxRequestAmplification        int
  85                 MaxRequestSize                 int
  86                 RailsSessionSecretToken        string
  87                 RequestTimeout                 Duration
  88                 SendTimeout                    Duration
  89                 WebsocketClientEventQueue      int
  90                 WebsocketServerEventQueue      int
  91                 KeepServiceRequestTimeout      Duration
  92         }
  93         AuditLogs struct {
  94                 MaxAge             Duration
  95                 MaxDeleteBatch     int
  96                 UnloggedAttributes StringSet
  97         }
  98         Collections struct {
  99                 BlobSigning          bool
 100                 BlobSigningKey       string
 101                 BlobSigningTTL       Duration
 102                 CollectionVersioning bool
 103                 DefaultTrashLifetime Duration
 104                 DefaultReplication   int
 105                 ManagedProperties    map[string]struct {
 106                         Value     interface{}
 107                         Function  string
 108                         Protected bool
 109                 }
 110                 PreserveVersionIfIdle Duration
 111                 TrashSweepInterval    Duration
 112                 TrustAllContent       bool
 113
 114                 WebDAVCache WebDAVCacheConfig
 115         }
 116         Git struct {
 117                 Repositories string
 118         }
 119         Login struct {
 120                 ProviderAppSecret string
 121                 ProviderAppID     string
 122         }
 123         Mail struct {
 124                 MailchimpAPIKey                string
 125                 MailchimpListID                string
 126                 SendUserSetupNotificationEmail bool
 127                 IssueReporterEmailFrom         string
 128                 IssueReporterEmailTo           string
 129                 SupportEmailAddress            string
 130                 EmailFrom                      string
 131         }
 132         SystemLogs struct {
 133                 LogLevel                string
 134                 Format                  string
 135                 MaxRequestLogParamsSize int
 136         }
 137         TLS struct {
 138                 Certificate string
 139                 Key         string
 140                 Insecure    bool
 141         }
 142         Users struct {
 143                 AnonymousUserToken                    string
 144                 AdminNotifierEmailFrom                string
 145                 AutoAdminFirstUser                    bool
 146                 AutoAdminUserWithEmail                string
 147                 AutoSetupNewUsers                     bool
 148                 AutoSetupNewUsersWithRepository       bool
 149                 AutoSetupNewUsersWithVmUUID           string
 150                 AutoSetupUsernameBlacklist            StringSet
 151                 EmailSubjectPrefix                    string
 152                 NewInactiveUserNotificationRecipients StringSet
 153                 NewUserNotificationRecipients         StringSet
 154                 NewUsersAreActive                     bool
 155                 UserNotifierEmailFrom                 string
 156                 UserProfileNotificationAddress        string
 157         }
 158         Workbench struct {
 159                 ActivationContactLink            string
 160                 APIClientConnectTimeout          Duration
 161                 APIClientReceiveTimeout          Duration
 162                 APIResponseCompression           bool
 163                 ApplicationMimetypesWithViewIcon StringSet
 164                 ArvadosDocsite                   string
 165                 ArvadosPublicDataDocURL          string
 166                 DefaultOpenIdPrefix              string
 167                 EnableGettingStartedPopup        bool
 168                 EnablePublicProjectsPage         bool
 169                 FileViewersConfigURL             string
 170                 LogViewerMaxBytes                ByteSize
 171                 MultiSiteSearch                  string
 172                 ProfilingEnabled                 bool
 173                 Repositories                     bool
 174                 RepositoryCache                  string
 175                 RunningJobLogRecordsToFetch      int
 176                 SecretKeyBase                    string
 177                 ShowRecentCollectionsOnDashboard bool
 178                 ShowUserAgreementInline          bool
 179                 ShowUserNotifications            bool
 180                 SiteName                         string
 181                 Theme                            string
 182                 UserProfileFormFields            map[string]struct {
 183                         Type                 string
 184                         FormFieldTitle       string
 185                         FormFieldDescription string
 186                         Required             bool
 187                         Position             int
 188                         Options              map[string]struct{}
 189                 }
 190                 UserProfileFormMessage string
 191                 VocabularyURL          string
 192         }
 193
 194         EnableBetaController14287 bool
 195 }
 196
 197 type Services struct {
 198         Composer       Service
 199         Controller     Service
 200         DispatchCloud  Service
 201         GitHTTP        Service
 202         GitSSH         Service
 203         Health         Service
 204         Keepbalance    Service
 205         Keepproxy      Service
 206         Keepstore      Service
 207         Nodemanager    Service
 208         RailsAPI       Service
 209         SSO            Service
 210         WebDAVDownload Service
 211         WebDAV         Service
 212         WebShell       Service
 213         Websocket      Service
 214         Workbench1     Service
 215         Workbench2     Service
 216 }
 217
 218 type Service struct {
 219         InternalURLs map[URL]ServiceInstance
 220         ExternalURL  URL
 221 }
 222
 223 // URL is a url.URL that is also usable as a JSON key/value.
 224 type URL url.URL
 225
 226 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 227 // used as a JSON key/value.
 228 func (su *URL) UnmarshalText(text []byte) error {
 229         u, err := url.Parse(string(text))
 230         if err == nil {
 231                 *su = URL(*u)
 232         }
 233         return err
 234 }
 235
 236 func (su URL) MarshalText() ([]byte, error) {
 237         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 238 }
 239
 240 type ServiceInstance struct{}
 241
 242 type PostgreSQL struct {
 243         Connection     PostgreSQLConnection
 244         ConnectionPool int
 245 }
 246
 247 type PostgreSQLConnection map[string]string
 248
 249 type RemoteCluster struct {
 250         Host          string
 251         Proxy         bool
 252         Scheme        string
 253         Insecure      bool
 254         ActivateUsers bool
 255 }
 256
 257 type InstanceType struct {
 258         Name            string
 259         ProviderType    string
 260         VCPUs           int
 261         RAM             ByteSize
 262         Scratch         ByteSize
 263         IncludedScratch ByteSize
 264         AddedScratch    ByteSize
 265         Price           float64
 266         Preemptible     bool
 267 }
 268
 269 type ContainersConfig struct {
 270         CloudVMs                    CloudVMsConfig
 271         CrunchRunCommand            string
 272         CrunchRunArgumentsList      []string
 273         DefaultKeepCacheRAM         ByteSize
 274         DispatchPrivateKey          string
 275         LogReuseDecisions           bool
 276         MaxComputeVMs               int
 277         MaxDispatchAttempts         int
 278         MaxRetryAttempts            int
 279         MinRetryPeriod              Duration
 280         ReserveExtraRAM             ByteSize
 281         StaleLockTimeout            Duration
 282         SupportedDockerImageFormats StringSet
 283         UsePreemptibleInstances     bool
 284
 285         JobsAPI struct {
 286                 Enable         string
 287                 GitInternalDir string
 288         }
 289         Logging struct {
 290                 MaxAge                       Duration
 291                 LogBytesPerEvent             int
 292                 LogSecondsBetweenEvents      int
 293                 LogThrottlePeriod            Duration
 294                 LogThrottleBytes             int
 295                 LogThrottleLines             int
 296                 LimitLogBytesPerJob          int
 297                 LogPartialLineThrottlePeriod Duration
 298                 LogUpdatePeriod              Duration
 299                 LogUpdateSize                ByteSize
 300         }
 301         SLURM struct {
 302                 PrioritySpread             int64
 303                 SbatchArgumentsList        []string
 304                 SbatchEnvironmentVariables map[string]string
 305                 Managed                    struct {
 306                         DNSServerConfDir       string
 307                         DNSServerConfTemplate  string
 308                         DNSServerReloadCommand string
 309                         DNSServerUpdateCommand string
 310                         ComputeNodeDomain      string
 311                         ComputeNodeNameservers StringSet
 312                         AssignNodeHostname     string
 313                 }
 314         }
 315 }
 316
 317 type CloudVMsConfig struct {
 318         Enable bool
 319
 320         BootProbeCommand     string
 321         ImageID              string
 322         MaxCloudOpsPerSecond int
 323         MaxProbesPerSecond   int
 324         PollInterval         Duration
 325         ProbeInterval        Duration
 326         SSHPort              string
 327         SyncInterval         Duration
 328         TimeoutBooting       Duration
 329         TimeoutIdle          Duration
 330         TimeoutProbe         Duration
 331         TimeoutShutdown      Duration
 332         TimeoutSignal        Duration
 333         TimeoutTERM          Duration
 334         ResourceTags         map[string]string
 335         TagKeyPrefix         string
 336
 337         Driver           string
 338         DriverParameters json.RawMessage
 339 }
 340
 341 type InstanceTypeMap map[string]InstanceType
 342
 343 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 344
 345 // UnmarshalJSON handles old config files that provide an array of
 346 // instance types instead of a hash.
 347 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 348         if len(data) > 0 && data[0] == '[' {
 349                 var arr []InstanceType
 350                 err := json.Unmarshal(data, &arr)
 351                 if err != nil {
 352                         return err
 353                 }
 354                 if len(arr) == 0 {
 355                         *it = nil
 356                         return nil
 357                 }
 358                 *it = make(map[string]InstanceType, len(arr))
 359                 for _, t := range arr {
 360                         if _, ok := (*it)[t.Name]; ok {
 361                                 return errDuplicateInstanceTypeName
 362                         }
 363                         if t.ProviderType == "" {
 364                                 t.ProviderType = t.Name
 365                         }
 366                         if t.Scratch == 0 {
 367                                 t.Scratch = t.IncludedScratch + t.AddedScratch
 368                         } else if t.AddedScratch == 0 {
 369                                 t.AddedScratch = t.Scratch - t.IncludedScratch
 370                         } else if t.IncludedScratch == 0 {
 371                                 t.IncludedScratch = t.Scratch - t.AddedScratch
 372                         }
 373
 374                         if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 375                                 return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 376                         }
 377                         (*it)[t.Name] = t
 378                 }
 379                 return nil
 380         }
 381         var hash map[string]InstanceType
 382         err := json.Unmarshal(data, &hash)
 383         if err != nil {
 384                 return err
 385         }
 386         // Fill in Name field (and ProviderType field, if not
 387         // specified) using hash key.
 388         *it = InstanceTypeMap(hash)
 389         for name, t := range *it {
 390                 t.Name = name
 391                 if t.ProviderType == "" {
 392                         t.ProviderType = name
 393                 }
 394                 (*it)[name] = t
 395         }
 396         return nil
 397 }
 398
 399 type StringSet map[string]struct{}
 400
 401 // UnmarshalJSON handles old config files that provide an array of
 402 // instance types instead of a hash.
 403 func (ss *StringSet) UnmarshalJSON(data []byte) error {
 404         if len(data) > 0 && data[0] == '[' {
 405                 var arr []string
 406                 err := json.Unmarshal(data, &arr)
 407                 if err != nil {
 408                         return err
 409                 }
 410                 if len(arr) == 0 {
 411                         *ss = nil
 412                         return nil
 413                 }
 414                 *ss = make(map[string]struct{}, len(arr))
 415                 for _, t := range arr {
 416                         (*ss)[t] = struct{}{}
 417                 }
 418                 return nil
 419         }
 420         var hash map[string]struct{}
 421         err := json.Unmarshal(data, &hash)
 422         if err != nil {
 423                 return err
 424         }
 425         *ss = make(map[string]struct{}, len(hash))
 426         for t, _ := range hash {
 427                 (*ss)[t] = struct{}{}
 428         }
 429
 430         return nil
 431 }
 432
 433 type ServiceName string
 434
 435 const (
 436         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 437         ServiceNameController    ServiceName = "arvados-controller"
 438         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 439         ServiceNameHealth        ServiceName = "arvados-health"
 440         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 441         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 442         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 443         ServiceNameWebsocket     ServiceName = "arvados-ws"
 444         ServiceNameKeepbalance   ServiceName = "keep-balance"
 445         ServiceNameKeepweb       ServiceName = "keep-web"
 446         ServiceNameKeepproxy     ServiceName = "keepproxy"
 447         ServiceNameKeepstore     ServiceName = "keepstore"
 448 )
 449
 450 // Map returns all services as a map, suitable for iterating over all
 451 // services or looking up a service by name.
 452 func (svcs Services) Map() map[ServiceName]Service {
 453         return map[ServiceName]Service{
 454                 ServiceNameRailsAPI:      svcs.RailsAPI,
 455                 ServiceNameController:    svcs.Controller,
 456                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 457                 ServiceNameHealth:        svcs.Health,
 458                 ServiceNameNodemanager:   svcs.Nodemanager,
 459                 ServiceNameWorkbench1:    svcs.Workbench1,
 460                 ServiceNameWorkbench2:    svcs.Workbench2,
 461                 ServiceNameWebsocket:     svcs.Websocket,
 462                 ServiceNameKeepbalance:   svcs.Keepbalance,
 463                 ServiceNameKeepweb:       svcs.WebDAV,
 464                 ServiceNameKeepproxy:     svcs.Keepproxy,
 465                 ServiceNameKeepstore:     svcs.Keepstore,
 466         }
 467 }