sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12
  13         "git.curoverse.com/arvados.git/sdk/go/config"
  14 )
  15
  16 const DefaultConfigFile = "/etc/arvados/config.yml"
  17
  18 type Config struct {
  19         Clusters map[string]Cluster
  20 }
  21
  22 // GetConfig returns the current system config, loading it from
  23 // configFile if needed.
  24 func GetConfig(configFile string) (*Config, error) {
  25         var cfg Config
  26         err := config.LoadFile(&cfg, configFile)
  27         return &cfg, err
  28 }
  29
  30 // GetCluster returns the cluster ID and config for the given
  31 // cluster, or the default/only configured cluster if clusterID is "".
  32 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  33         if clusterID == "" {
  34                 if len(sc.Clusters) == 0 {
  35                         return nil, fmt.Errorf("no clusters configured")
  36                 } else if len(sc.Clusters) > 1 {
  37                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  38                 } else {
  39                         for id, cc := range sc.Clusters {
  40                                 cc.ClusterID = id
  41                                 return &cc, nil
  42                         }
  43                 }
  44         }
  45         if cc, ok := sc.Clusters[clusterID]; !ok {
  46                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  47         } else {
  48                 cc.ClusterID = clusterID
  49                 return &cc, nil
  50         }
  51 }
  52
  53 type Cluster struct {
  54         ClusterID       string `json:"-"`
  55         ManagementToken string
  56         SystemRootToken string
  57         Services        Services
  58         InstanceTypes   InstanceTypeMap
  59         Containers      ContainersConfig
  60         RemoteClusters  map[string]RemoteCluster
  61         PostgreSQL      PostgreSQL
  62
  63         API struct {
  64                 AsyncPermissionsUpdateInterval Duration
  65                 DisabledAPIs                   []string
  66                 MaxIndexDatabaseRead           int
  67                 MaxItemsPerResponse            int
  68                 MaxRequestAmplification        int
  69                 MaxRequestSize                 int
  70                 RailsSessionSecretToken        string
  71                 RequestTimeout                 Duration
  72         }
  73         AuditLogs struct {
  74                 MaxAge             Duration
  75                 MaxDeleteBatch     int
  76                 UnloggedAttributes []string
  77         }
  78         Collections struct {
  79                 BlobSigning           bool
  80                 BlobSigningKey        string
  81                 DefaultReplication    int
  82                 BlobSigningTTL        Duration
  83                 DefaultTrashLifetime  Duration
  84                 TrashSweepInterval    Duration
  85                 CollectionVersioning  bool
  86                 PreserveVersionIfIdle Duration
  87         }
  88         Git struct {
  89                 Repositories string
  90         }
  91         Login struct {
  92                 ProviderAppSecret string
  93                 ProviderAppID     string
  94         }
  95         Mail struct {
  96                 MailchimpAPIKey                string
  97                 MailchimpListID                string
  98                 SendUserSetupNotificationEmail string
  99                 IssueReporterEmailFrom         string
 100                 IssueReporterEmailTo           string
 101                 SupportEmailAddress            string
 102                 EmailFrom                      string
 103         }
 104         SystemLogs struct {
 105                 LogLevel                string
 106                 Format                  string
 107                 MaxRequestLogParamsSize int
 108         }
 109         TLS struct {
 110                 Certificate string
 111                 Key         string
 112                 Insecure    bool
 113         }
 114         Users struct {
 115                 AdminNotifierEmailFrom                string
 116                 AutoAdminFirstUser                    bool
 117                 AutoAdminUserWithEmail                string
 118                 AutoSetupNewUsers                     bool
 119                 AutoSetupNewUsersWithRepository       bool
 120                 AutoSetupNewUsersWithVmUUID           string
 121                 AutoSetupUsernameBlacklist            []string
 122                 EmailSubjectPrefix                    string
 123                 NewInactiveUserNotificationRecipients []string
 124                 NewUserNotificationRecipients         []string
 125                 NewUsersAreActive                     bool
 126                 UserNotifierEmailFrom                 string
 127                 UserProfileNotificationAddress        string
 128         }
 129         Workbench struct {
 130                 ActivationContactLink            string
 131                 APIClientConnectTimeout          Duration
 132                 APIClientReceiveTimeout          Duration
 133                 APIResponseCompression           bool
 134                 ApplicationMimetypesWithViewIcon map[string]struct{}
 135                 ArvadosDocsite                   string
 136                 ArvadosPublicDataDocURL          string
 137                 EnableGettingStartedPopup        bool
 138                 EnablePublicProjectsPage         bool
 139                 FileViewersConfigURL             string
 140                 LogViewerMaxBytes                ByteSize
 141                 MultiSiteSearch                  bool
 142                 Repositories                     bool
 143                 RepositoryCache                  string
 144                 RunningJobLogRecordsToFetch      int
 145                 SecretKeyBase                    string
 146                 SecretToken                      string
 147                 ShowRecentCollectionsOnDashboard bool
 148                 ShowUserAgreementInline          bool
 149                 ShowUserNotifications            bool
 150                 SiteName                         string
 151                 Theme                            string
 152                 UserProfileFormFields            map[string]struct {
 153                         Type                 string
 154                         FormFieldTitle       string
 155                         FormFieldDescription string
 156                         Required             bool
 157                 }
 158                 UserProfileFormMessage string
 159                 VocabularyURL          string
 160         }
 161 }
 162
 163 type Services struct {
 164         Composer       Service
 165         Controller     Service
 166         DispatchCloud  Service
 167         GitHTTP        Service
 168         GitSSH         Service
 169         Health         Service
 170         Keepbalance    Service
 171         Keepproxy      Service
 172         Keepstore      Service
 173         Nodemanager    Service
 174         RailsAPI       Service
 175         SSO            Service
 176         WebDAVDownload Service
 177         WebDAV         Service
 178         WebShell       Service
 179         Websocket      Service
 180         Workbench1     Service
 181         Workbench2     Service
 182 }
 183
 184 type Service struct {
 185         InternalURLs map[URL]ServiceInstance
 186         ExternalURL  URL
 187 }
 188
 189 // URL is a url.URL that is also usable as a JSON key/value.
 190 type URL url.URL
 191
 192 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 193 // used as a JSON key/value.
 194 func (su *URL) UnmarshalText(text []byte) error {
 195         u, err := url.Parse(string(text))
 196         if err == nil {
 197                 *su = URL(*u)
 198         }
 199         return err
 200 }
 201
 202 func (su URL) MarshalText() ([]byte, error) {
 203         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 204 }
 205
 206 type ServiceInstance struct{}
 207
 208 type PostgreSQL struct {
 209         Connection     PostgreSQLConnection
 210         ConnectionPool int
 211 }
 212
 213 type PostgreSQLConnection map[string]string
 214
 215 type RemoteCluster struct {
 216         Host          string
 217         Proxy         bool
 218         Scheme        string
 219         Insecure      bool
 220         ActivateUsers bool
 221 }
 222
 223 type InstanceType struct {
 224         Name            string
 225         ProviderType    string
 226         VCPUs           int
 227         RAM             ByteSize
 228         Scratch         ByteSize
 229         IncludedScratch ByteSize
 230         AddedScratch    ByteSize
 231         Price           float64
 232         Preemptible     bool
 233 }
 234
 235 type ContainersConfig struct {
 236         CloudVMs                    CloudVMsConfig
 237         DefaultKeepCacheRAM         ByteSize
 238         DispatchPrivateKey          string
 239         LogReuseDecisions           bool
 240         MaxComputeVMs               int
 241         MaxDispatchAttempts         int
 242         MaxRetryAttempts            int
 243         StaleLockTimeout            Duration
 244         SupportedDockerImageFormats []string
 245         UsePreemptibleInstances     bool
 246
 247         JobsAPI struct {
 248                 Enable                  string
 249                 GitInternalDir          string
 250                 DefaultDockerImage      string
 251                 CrunchJobWrapper        string
 252                 CrunchJobUser           string
 253                 CrunchRefreshTrigger    string
 254                 ReuseJobIfOutputsDiffer bool
 255         }
 256         Logging struct {
 257                 MaxAge                       Duration
 258                 LogBytesPerEvent             int
 259                 LogSecondsBetweenEvents      int
 260                 LogThrottlePeriod            Duration
 261                 LogThrottleBytes             int
 262                 LogThrottleLines             int
 263                 LimitLogBytesPerJob          int
 264                 LogPartialLineThrottlePeriod Duration
 265                 LogUpdatePeriod              Duration
 266                 LogUpdateSize                ByteSize
 267         }
 268         SLURM struct {
 269                 Managed struct {
 270                         DNSServerConfDir       string
 271                         DNSServerConfTemplate  string
 272                         DNSServerReloadCommand string
 273                         DNSServerUpdateCommand string
 274                         ComputeNodeDomain      string
 275                         ComputeNodeNameservers []string
 276                         AssignNodeHostname     string
 277                 }
 278         }
 279 }
 280
 281 type CloudVMsConfig struct {
 282         Enable bool
 283
 284         BootProbeCommand     string
 285         ImageID              string
 286         MaxCloudOpsPerSecond int
 287         MaxProbesPerSecond   int
 288         PollInterval         Duration
 289         ProbeInterval        Duration
 290         SSHPort              string
 291         SyncInterval         Duration
 292         TimeoutBooting       Duration
 293         TimeoutIdle          Duration
 294         TimeoutProbe         Duration
 295         TimeoutShutdown      Duration
 296         TimeoutSignal        Duration
 297         TimeoutTERM          Duration
 298         ResourceTags         map[string]string
 299         TagKeyPrefix         string
 300
 301         Driver           string
 302         DriverParameters json.RawMessage
 303 }
 304
 305 type InstanceTypeMap map[string]InstanceType
 306
 307 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 308
 309 // UnmarshalJSON handles old config files that provide an array of
 310 // instance types instead of a hash.
 311 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 312         if len(data) > 0 && data[0] == '[' {
 313                 var arr []InstanceType
 314                 err := json.Unmarshal(data, &arr)
 315                 if err != nil {
 316                         return err
 317                 }
 318                 if len(arr) == 0 {
 319                         *it = nil
 320                         return nil
 321                 }
 322                 *it = make(map[string]InstanceType, len(arr))
 323                 for _, t := range arr {
 324                         if _, ok := (*it)[t.Name]; ok {
 325                                 return errDuplicateInstanceTypeName
 326                         }
 327                         if t.ProviderType == "" {
 328                                 t.ProviderType = t.Name
 329                         }
 330                         if t.Scratch == 0 {
 331                                 t.Scratch = t.IncludedScratch + t.AddedScratch
 332                         } else if t.AddedScratch == 0 {
 333                                 t.AddedScratch = t.Scratch - t.IncludedScratch
 334                         } else if t.IncludedScratch == 0 {
 335                                 t.IncludedScratch = t.Scratch - t.AddedScratch
 336                         }
 337
 338                         if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 339                                 return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 340                         }
 341                         (*it)[t.Name] = t
 342                 }
 343                 return nil
 344         }
 345         var hash map[string]InstanceType
 346         err := json.Unmarshal(data, &hash)
 347         if err != nil {
 348                 return err
 349         }
 350         // Fill in Name field (and ProviderType field, if not
 351         // specified) using hash key.
 352         *it = InstanceTypeMap(hash)
 353         for name, t := range *it {
 354                 t.Name = name
 355                 if t.ProviderType == "" {
 356                         t.ProviderType = name
 357                 }
 358                 (*it)[name] = t
 359         }
 360         return nil
 361 }
 362
 363 type ServiceName string
 364
 365 const (
 366         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 367         ServiceNameController    ServiceName = "arvados-controller"
 368         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 369         ServiceNameHealth        ServiceName = "arvados-health"
 370         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 371         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 372         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 373         ServiceNameWebsocket     ServiceName = "arvados-ws"
 374         ServiceNameKeepbalance   ServiceName = "keep-balance"
 375         ServiceNameKeepweb       ServiceName = "keep-web"
 376         ServiceNameKeepproxy     ServiceName = "keepproxy"
 377         ServiceNameKeepstore     ServiceName = "keepstore"
 378 )
 379
 380 // Map returns all services as a map, suitable for iterating over all
 381 // services or looking up a service by name.
 382 func (svcs Services) Map() map[ServiceName]Service {
 383         return map[ServiceName]Service{
 384                 ServiceNameRailsAPI:      svcs.RailsAPI,
 385                 ServiceNameController:    svcs.Controller,
 386                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 387                 ServiceNameHealth:        svcs.Health,
 388                 ServiceNameNodemanager:   svcs.Nodemanager,
 389                 ServiceNameWorkbench1:    svcs.Workbench1,
 390                 ServiceNameWorkbench2:    svcs.Workbench2,
 391                 ServiceNameWebsocket:     svcs.Websocket,
 392                 ServiceNameKeepbalance:   svcs.Keepbalance,
 393                 ServiceNameKeepweb:       svcs.WebDAV,
 394                 ServiceNameKeepproxy:     svcs.Keepproxy,
 395                 ServiceNameKeepstore:     svcs.Keepstore,
 396         }
 397 }