sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12         "os"
  13
  14         "git.curoverse.com/arvados.git/sdk/go/config"
  15 )
  16
  17 var DefaultConfigFile = func() string {
  18         if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
  19                 return path
  20         } else {
  21                 return "/etc/arvados/config.yml"
  22         }
  23 }()
  24
  25 type Config struct {
  26         Clusters map[string]Cluster
  27 }
  28
  29 // GetConfig returns the current system config, loading it from
  30 // configFile if needed.
  31 func GetConfig(configFile string) (*Config, error) {
  32         var cfg Config
  33         err := config.LoadFile(&cfg, configFile)
  34         return &cfg, err
  35 }
  36
  37 // GetCluster returns the cluster ID and config for the given
  38 // cluster, or the default/only configured cluster if clusterID is "".
  39 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  40         if clusterID == "" {
  41                 if len(sc.Clusters) == 0 {
  42                         return nil, fmt.Errorf("no clusters configured")
  43                 } else if len(sc.Clusters) > 1 {
  44                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  45                 } else {
  46                         for id, cc := range sc.Clusters {
  47                                 cc.ClusterID = id
  48                                 return &cc, nil
  49                         }
  50                 }
  51         }
  52         if cc, ok := sc.Clusters[clusterID]; !ok {
  53                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  54         } else {
  55                 cc.ClusterID = clusterID
  56                 return &cc, nil
  57         }
  58 }
  59
  60 type Cluster struct {
  61         ClusterID       string `json:"-"`
  62         ManagementToken string
  63         SystemRootToken string
  64         Services        Services
  65         InstanceTypes   InstanceTypeMap
  66         Containers      ContainersConfig
  67         RemoteClusters  map[string]RemoteCluster
  68         PostgreSQL      PostgreSQL
  69
  70         API struct {
  71                 AsyncPermissionsUpdateInterval Duration
  72                 DisabledAPIs                   []string
  73                 MaxIndexDatabaseRead           int
  74                 MaxItemsPerResponse            int
  75                 MaxRequestAmplification        int
  76                 MaxRequestSize                 int
  77                 RailsSessionSecretToken        string
  78                 RequestTimeout                 Duration
  79         }
  80         AuditLogs struct {
  81                 MaxAge             Duration
  82                 MaxDeleteBatch     int
  83                 UnloggedAttributes []string
  84         }
  85         Collections struct {
  86                 BlobSigning          bool
  87                 BlobSigningKey       string
  88                 BlobSigningTTL       Duration
  89                 CollectionVersioning bool
  90                 DefaultTrashLifetime Duration
  91                 DefaultReplication   int
  92                 ManagedProperties    map[string]struct {
  93                         Value     interface{}
  94                         Function  string
  95                         Protected bool
  96                 }
  97                 PreserveVersionIfIdle Duration
  98                 TrashSweepInterval    Duration
  99                 TrustAllContent       bool
 100         }
 101         Git struct {
 102                 Repositories string
 103         }
 104         Login struct {
 105                 ProviderAppSecret string
 106                 ProviderAppID     string
 107         }
 108         Mail struct {
 109                 MailchimpAPIKey                string
 110                 MailchimpListID                string
 111                 SendUserSetupNotificationEmail bool
 112                 IssueReporterEmailFrom         string
 113                 IssueReporterEmailTo           string
 114                 SupportEmailAddress            string
 115                 EmailFrom                      string
 116         }
 117         SystemLogs struct {
 118                 LogLevel                string
 119                 Format                  string
 120                 MaxRequestLogParamsSize int
 121         }
 122         TLS struct {
 123                 Certificate string
 124                 Key         string
 125                 Insecure    bool
 126         }
 127         Users struct {
 128                 AnonymousUserToken                    string
 129                 AdminNotifierEmailFrom                string
 130                 AutoAdminFirstUser                    bool
 131                 AutoAdminUserWithEmail                string
 132                 AutoSetupNewUsers                     bool
 133                 AutoSetupNewUsersWithRepository       bool
 134                 AutoSetupNewUsersWithVmUUID           string
 135                 AutoSetupUsernameBlacklist            []string
 136                 EmailSubjectPrefix                    string
 137                 NewInactiveUserNotificationRecipients []string
 138                 NewUserNotificationRecipients         []string
 139                 NewUsersAreActive                     bool
 140                 UserNotifierEmailFrom                 string
 141                 UserProfileNotificationAddress        string
 142         }
 143         Workbench struct {
 144                 ActivationContactLink            string
 145                 APIClientConnectTimeout          Duration
 146                 APIClientReceiveTimeout          Duration
 147                 APIResponseCompression           bool
 148                 ApplicationMimetypesWithViewIcon map[string]struct{}
 149                 ArvadosDocsite                   string
 150                 ArvadosPublicDataDocURL          string
 151                 DefaultOpenIdPrefix              string
 152                 EnableGettingStartedPopup        bool
 153                 EnablePublicProjectsPage         bool
 154                 FileViewersConfigURL             string
 155                 LogViewerMaxBytes                ByteSize
 156                 MultiSiteSearch                  string
 157                 ProfilingEnabled                 bool
 158                 Repositories                     bool
 159                 RepositoryCache                  string
 160                 RunningJobLogRecordsToFetch      int
 161                 SecretKeyBase                    string
 162                 ShowRecentCollectionsOnDashboard bool
 163                 ShowUserAgreementInline          bool
 164                 ShowUserNotifications            bool
 165                 SiteName                         string
 166                 Theme                            string
 167                 UserProfileFormFields            map[string]struct {
 168                         Type                 string
 169                         FormFieldTitle       string
 170                         FormFieldDescription string
 171                         Required             bool
 172                         Position             int
 173                         Options              map[string]struct{}
 174                 }
 175                 UserProfileFormMessage string
 176                 VocabularyURL          string
 177         }
 178
 179         EnableBetaController14287 bool
 180 }
 181
 182 type Services struct {
 183         Composer       Service
 184         Controller     Service
 185         DispatchCloud  Service
 186         GitHTTP        Service
 187         GitSSH         Service
 188         Health         Service
 189         Keepbalance    Service
 190         Keepproxy      Service
 191         Keepstore      Service
 192         Nodemanager    Service
 193         RailsAPI       Service
 194         SSO            Service
 195         WebDAVDownload Service
 196         WebDAV         Service
 197         WebShell       Service
 198         Websocket      Service
 199         Workbench1     Service
 200         Workbench2     Service
 201 }
 202
 203 type Service struct {
 204         InternalURLs map[URL]ServiceInstance
 205         ExternalURL  URL
 206 }
 207
 208 // URL is a url.URL that is also usable as a JSON key/value.
 209 type URL url.URL
 210
 211 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 212 // used as a JSON key/value.
 213 func (su *URL) UnmarshalText(text []byte) error {
 214         u, err := url.Parse(string(text))
 215         if err == nil {
 216                 *su = URL(*u)
 217         }
 218         return err
 219 }
 220
 221 func (su URL) MarshalText() ([]byte, error) {
 222         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 223 }
 224
 225 type ServiceInstance struct{}
 226
 227 type PostgreSQL struct {
 228         Connection     PostgreSQLConnection
 229         ConnectionPool int
 230 }
 231
 232 type PostgreSQLConnection map[string]string
 233
 234 type RemoteCluster struct {
 235         Host          string
 236         Proxy         bool
 237         Scheme        string
 238         Insecure      bool
 239         ActivateUsers bool
 240 }
 241
 242 type InstanceType struct {
 243         Name            string
 244         ProviderType    string
 245         VCPUs           int
 246         RAM             ByteSize
 247         Scratch         ByteSize
 248         IncludedScratch ByteSize
 249         AddedScratch    ByteSize
 250         Price           float64
 251         Preemptible     bool
 252 }
 253
 254 type ContainersConfig struct {
 255         CloudVMs                    CloudVMsConfig
 256         CrunchRunCommand            string
 257         CrunchRunArgumentsList      []string
 258         DefaultKeepCacheRAM         ByteSize
 259         DispatchPrivateKey          string
 260         LogReuseDecisions           bool
 261         MaxComputeVMs               int
 262         MaxDispatchAttempts         int
 263         MaxRetryAttempts            int
 264         MinRetryPeriod              Duration
 265         ReserveExtraRAM             ByteSize
 266         StaleLockTimeout            Duration
 267         SupportedDockerImageFormats []string
 268         UsePreemptibleInstances     bool
 269
 270         JobsAPI struct {
 271                 Enable                  string
 272                 GitInternalDir          string
 273                 DefaultDockerImage      string
 274                 CrunchJobWrapper        string
 275                 CrunchJobUser           string
 276                 CrunchRefreshTrigger    string
 277                 ReuseJobIfOutputsDiffer bool
 278         }
 279         Logging struct {
 280                 MaxAge                       Duration
 281                 LogBytesPerEvent             int
 282                 LogSecondsBetweenEvents      int
 283                 LogThrottlePeriod            Duration
 284                 LogThrottleBytes             int
 285                 LogThrottleLines             int
 286                 LimitLogBytesPerJob          int
 287                 LogPartialLineThrottlePeriod Duration
 288                 LogUpdatePeriod              Duration
 289                 LogUpdateSize                ByteSize
 290         }
 291         SLURM struct {
 292                 PrioritySpread      int64
 293                 SbatchArgumentsList []string
 294                 Managed             struct {
 295                         DNSServerConfDir       string
 296                         DNSServerConfTemplate  string
 297                         DNSServerReloadCommand string
 298                         DNSServerUpdateCommand string
 299                         ComputeNodeDomain      string
 300                         ComputeNodeNameservers []string
 301                         AssignNodeHostname     string
 302                 }
 303         }
 304 }
 305
 306 type CloudVMsConfig struct {
 307         Enable bool
 308
 309         BootProbeCommand     string
 310         ImageID              string
 311         MaxCloudOpsPerSecond int
 312         MaxProbesPerSecond   int
 313         PollInterval         Duration
 314         ProbeInterval        Duration
 315         SSHPort              string
 316         SyncInterval         Duration
 317         TimeoutBooting       Duration
 318         TimeoutIdle          Duration
 319         TimeoutProbe         Duration
 320         TimeoutShutdown      Duration
 321         TimeoutSignal        Duration
 322         TimeoutTERM          Duration
 323         ResourceTags         map[string]string
 324         TagKeyPrefix         string
 325
 326         Driver           string
 327         DriverParameters json.RawMessage
 328 }
 329
 330 type InstanceTypeMap map[string]InstanceType
 331
 332 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 333
 334 // UnmarshalJSON handles old config files that provide an array of
 335 // instance types instead of a hash.
 336 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 337         if len(data) > 0 && data[0] == '[' {
 338                 var arr []InstanceType
 339                 err := json.Unmarshal(data, &arr)
 340                 if err != nil {
 341                         return err
 342                 }
 343                 if len(arr) == 0 {
 344                         *it = nil
 345                         return nil
 346                 }
 347                 *it = make(map[string]InstanceType, len(arr))
 348                 for _, t := range arr {
 349                         if _, ok := (*it)[t.Name]; ok {
 350                                 return errDuplicateInstanceTypeName
 351                         }
 352                         if t.ProviderType == "" {
 353                                 t.ProviderType = t.Name
 354                         }
 355                         if t.Scratch == 0 {
 356                                 t.Scratch = t.IncludedScratch + t.AddedScratch
 357                         } else if t.AddedScratch == 0 {
 358                                 t.AddedScratch = t.Scratch - t.IncludedScratch
 359                         } else if t.IncludedScratch == 0 {
 360                                 t.IncludedScratch = t.Scratch - t.AddedScratch
 361                         }
 362
 363                         if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 364                                 return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 365                         }
 366                         (*it)[t.Name] = t
 367                 }
 368                 return nil
 369         }
 370         var hash map[string]InstanceType
 371         err := json.Unmarshal(data, &hash)
 372         if err != nil {
 373                 return err
 374         }
 375         // Fill in Name field (and ProviderType field, if not
 376         // specified) using hash key.
 377         *it = InstanceTypeMap(hash)
 378         for name, t := range *it {
 379                 t.Name = name
 380                 if t.ProviderType == "" {
 381                         t.ProviderType = name
 382                 }
 383                 (*it)[name] = t
 384         }
 385         return nil
 386 }
 387
 388 type ServiceName string
 389
 390 const (
 391         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 392         ServiceNameController    ServiceName = "arvados-controller"
 393         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 394         ServiceNameHealth        ServiceName = "arvados-health"
 395         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 396         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 397         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 398         ServiceNameWebsocket     ServiceName = "arvados-ws"
 399         ServiceNameKeepbalance   ServiceName = "keep-balance"
 400         ServiceNameKeepweb       ServiceName = "keep-web"
 401         ServiceNameKeepproxy     ServiceName = "keepproxy"
 402         ServiceNameKeepstore     ServiceName = "keepstore"
 403 )
 404
 405 // Map returns all services as a map, suitable for iterating over all
 406 // services or looking up a service by name.
 407 func (svcs Services) Map() map[ServiceName]Service {
 408         return map[ServiceName]Service{
 409                 ServiceNameRailsAPI:      svcs.RailsAPI,
 410                 ServiceNameController:    svcs.Controller,
 411                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 412                 ServiceNameHealth:        svcs.Health,
 413                 ServiceNameNodemanager:   svcs.Nodemanager,
 414                 ServiceNameWorkbench1:    svcs.Workbench1,
 415                 ServiceNameWorkbench2:    svcs.Workbench2,
 416                 ServiceNameWebsocket:     svcs.Websocket,
 417                 ServiceNameKeepbalance:   svcs.Keepbalance,
 418                 ServiceNameKeepweb:       svcs.WebDAV,
 419                 ServiceNameKeepproxy:     svcs.Keepproxy,
 420                 ServiceNameKeepstore:     svcs.Keepstore,
 421         }
 422 }