sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12
  13         "git.curoverse.com/arvados.git/sdk/go/config"
  14 )
  15
  16 const DefaultConfigFile = "/etc/arvados/config.yml"
  17
  18 type Config struct {
  19         Clusters map[string]Cluster
  20 }
  21
  22 // GetConfig returns the current system config, loading it from
  23 // configFile if needed.
  24 func GetConfig(configFile string) (*Config, error) {
  25         var cfg Config
  26         err := config.LoadFile(&cfg, configFile)
  27         return &cfg, err
  28 }
  29
  30 // GetCluster returns the cluster ID and config for the given
  31 // cluster, or the default/only configured cluster if clusterID is "".
  32 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  33         if clusterID == "" {
  34                 if len(sc.Clusters) == 0 {
  35                         return nil, fmt.Errorf("no clusters configured")
  36                 } else if len(sc.Clusters) > 1 {
  37                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  38                 } else {
  39                         for id, cc := range sc.Clusters {
  40                                 cc.ClusterID = id
  41                                 return &cc, nil
  42                         }
  43                 }
  44         }
  45         if cc, ok := sc.Clusters[clusterID]; !ok {
  46                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  47         } else {
  48                 cc.ClusterID = clusterID
  49                 return &cc, nil
  50         }
  51 }
  52
  53 type Cluster struct {
  54         ClusterID       string `json:"-"`
  55         ManagementToken string
  56         SystemRootToken string
  57         Services        Services
  58         InstanceTypes   InstanceTypeMap
  59         Containers      ContainersConfig
  60         RemoteClusters  map[string]RemoteCluster
  61         PostgreSQL      PostgreSQL
  62
  63         API struct {
  64                 AsyncPermissionsUpdateInterval Duration
  65                 DisabledAPIs                   []string
  66                 MaxIndexDatabaseRead           int
  67                 MaxItemsPerResponse            int
  68                 MaxRequestAmplification        int
  69                 MaxRequestSize                 int
  70                 RailsSessionSecretToken        string
  71                 RequestTimeout                 Duration
  72         }
  73         AuditLogs struct {
  74                 MaxAge             Duration
  75                 MaxDeleteBatch     int
  76                 UnloggedAttributes []string
  77         }
  78         Collections struct {
  79                 BlobSigning           bool
  80                 BlobSigningKey        string
  81                 DefaultReplication    int
  82                 BlobSigningTTL        Duration
  83                 DefaultTrashLifetime  Duration
  84                 TrashSweepInterval    Duration
  85                 CollectionVersioning  bool
  86                 PreserveVersionIfIdle Duration
  87         }
  88         Git struct {
  89                 Repositories string
  90         }
  91         Login struct {
  92                 ProviderAppSecret string
  93                 ProviderAppID     string
  94         }
  95         Mail struct {
  96                 MailchimpAPIKey                string
  97                 MailchimpListID                string
  98                 SendUserSetupNotificationEmail string
  99                 IssueReporterEmailFrom         string
 100                 IssueReporterEmailTo           string
 101                 SupportEmailAddress            string
 102                 EmailFrom                      string
 103         }
 104         SystemLogs struct {
 105                 LogLevel                string
 106                 Format                  string
 107                 MaxRequestLogParamsSize int
 108         }
 109         TLS struct {
 110                 Certificate string
 111                 Key         string
 112                 Insecure    bool
 113         }
 114         Users struct {
 115                 AdminNotifierEmailFrom                string
 116                 AutoAdminFirstUser                    bool
 117                 AutoAdminUserWithEmail                string
 118                 AutoSetupNewUsers                     bool
 119                 AutoSetupNewUsersWithRepository       bool
 120                 AutoSetupNewUsersWithVmUUID           string
 121                 AutoSetupUsernameBlacklist            []string
 122                 EmailSubjectPrefix                    string
 123                 NewInactiveUserNotificationRecipients []string
 124                 NewUserNotificationRecipients         []string
 125                 NewUsersAreActive                     bool
 126                 UserNotifierEmailFrom                 string
 127                 UserProfileNotificationAddress        string
 128         }
 129         Workbench struct {
 130                 ActivationContactLink            string
 131                 APIClientConnectTimeout          Duration
 132                 APIClientReceiveTimeout          Duration
 133                 APIResponseCompression           bool
 134                 ApplicationMimetypesWithViewIcon map[string]struct{}
 135                 ArvadosDocsite                   string
 136                 ArvadosPublicDataDocURL          string
 137                 EnableGettingStartedPopup        bool
 138                 EnablePublicProjectsPage         bool
 139                 FileViewersConfigURL             string
 140                 LogViewerMaxBytes                ByteSize
 141                 MultiSiteSearch                  bool
 142                 Repositories                     bool
 143                 RepositoryCache                  string
 144                 RunningJobLogRecordsToFetch      int
 145                 SecretKeyBase                    string
 146                 SecretToken                      string
 147                 ShowRecentCollectionsOnDashboard bool
 148                 ShowUserAgreementInline          bool
 149                 ShowUserNotifications            bool
 150                 SiteName                         string
 151                 Theme                            string
 152                 UserProfileFormFields            map[string]struct {
 153                         Type                 string
 154                         FormFieldTitle       string
 155                         FormFieldDescription string
 156                         Required             bool
 157                 }
 158                 UserProfileFormMessage string
 159                 VocabularyURL          string
 160         }
 161
 162         EnableBetaController14287 bool
 163 }
 164
 165 type Services struct {
 166         Composer       Service
 167         Controller     Service
 168         DispatchCloud  Service
 169         GitHTTP        Service
 170         GitSSH         Service
 171         Health         Service
 172         Keepbalance    Service
 173         Keepproxy      Service
 174         Keepstore      Service
 175         Nodemanager    Service
 176         RailsAPI       Service
 177         SSO            Service
 178         WebDAVDownload Service
 179         WebDAV         Service
 180         WebShell       Service
 181         Websocket      Service
 182         Workbench1     Service
 183         Workbench2     Service
 184 }
 185
 186 type Service struct {
 187         InternalURLs map[URL]ServiceInstance
 188         ExternalURL  URL
 189 }
 190
 191 // URL is a url.URL that is also usable as a JSON key/value.
 192 type URL url.URL
 193
 194 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 195 // used as a JSON key/value.
 196 func (su *URL) UnmarshalText(text []byte) error {
 197         u, err := url.Parse(string(text))
 198         if err == nil {
 199                 *su = URL(*u)
 200         }
 201         return err
 202 }
 203
 204 func (su URL) MarshalText() ([]byte, error) {
 205         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 206 }
 207
 208 type ServiceInstance struct{}
 209
 210 type PostgreSQL struct {
 211         Connection     PostgreSQLConnection
 212         ConnectionPool int
 213 }
 214
 215 type PostgreSQLConnection map[string]string
 216
 217 type RemoteCluster struct {
 218         Host          string
 219         Proxy         bool
 220         Scheme        string
 221         Insecure      bool
 222         ActivateUsers bool
 223 }
 224
 225 type InstanceType struct {
 226         Name            string
 227         ProviderType    string
 228         VCPUs           int
 229         RAM             ByteSize
 230         Scratch         ByteSize
 231         IncludedScratch ByteSize
 232         AddedScratch    ByteSize
 233         Price           float64
 234         Preemptible     bool
 235 }
 236
 237 type ContainersConfig struct {
 238         CloudVMs                    CloudVMsConfig
 239         DefaultKeepCacheRAM         ByteSize
 240         DispatchPrivateKey          string
 241         LogReuseDecisions           bool
 242         MaxComputeVMs               int
 243         MaxDispatchAttempts         int
 244         MaxRetryAttempts            int
 245         StaleLockTimeout            Duration
 246         SupportedDockerImageFormats []string
 247         UsePreemptibleInstances     bool
 248
 249         JobsAPI struct {
 250                 Enable                  string
 251                 GitInternalDir          string
 252                 DefaultDockerImage      string
 253                 CrunchJobWrapper        string
 254                 CrunchJobUser           string
 255                 CrunchRefreshTrigger    string
 256                 ReuseJobIfOutputsDiffer bool
 257         }
 258         Logging struct {
 259                 MaxAge                       Duration
 260                 LogBytesPerEvent             int
 261                 LogSecondsBetweenEvents      int
 262                 LogThrottlePeriod            Duration
 263                 LogThrottleBytes             int
 264                 LogThrottleLines             int
 265                 LimitLogBytesPerJob          int
 266                 LogPartialLineThrottlePeriod Duration
 267                 LogUpdatePeriod              Duration
 268                 LogUpdateSize                ByteSize
 269         }
 270         SLURM struct {
 271                 Managed struct {
 272                         DNSServerConfDir       string
 273                         DNSServerConfTemplate  string
 274                         DNSServerReloadCommand string
 275                         DNSServerUpdateCommand string
 276                         ComputeNodeDomain      string
 277                         ComputeNodeNameservers []string
 278                         AssignNodeHostname     string
 279                 }
 280         }
 281 }
 282
 283 type CloudVMsConfig struct {
 284         Enable bool
 285
 286         BootProbeCommand     string
 287         ImageID              string
 288         MaxCloudOpsPerSecond int
 289         MaxProbesPerSecond   int
 290         PollInterval         Duration
 291         ProbeInterval        Duration
 292         SSHPort              string
 293         SyncInterval         Duration
 294         TimeoutBooting       Duration
 295         TimeoutIdle          Duration
 296         TimeoutProbe         Duration
 297         TimeoutShutdown      Duration
 298         TimeoutSignal        Duration
 299         TimeoutTERM          Duration
 300         ResourceTags         map[string]string
 301         TagKeyPrefix         string
 302
 303         Driver           string
 304         DriverParameters json.RawMessage
 305 }
 306
 307 type InstanceTypeMap map[string]InstanceType
 308
 309 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 310
 311 // UnmarshalJSON handles old config files that provide an array of
 312 // instance types instead of a hash.
 313 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 314         if len(data) > 0 && data[0] == '[' {
 315                 var arr []InstanceType
 316                 err := json.Unmarshal(data, &arr)
 317                 if err != nil {
 318                         return err
 319                 }
 320                 if len(arr) == 0 {
 321                         *it = nil
 322                         return nil
 323                 }
 324                 *it = make(map[string]InstanceType, len(arr))
 325                 for _, t := range arr {
 326                         if _, ok := (*it)[t.Name]; ok {
 327                                 return errDuplicateInstanceTypeName
 328                         }
 329                         if t.ProviderType == "" {
 330                                 t.ProviderType = t.Name
 331                         }
 332                         if t.Scratch == 0 {
 333                                 t.Scratch = t.IncludedScratch + t.AddedScratch
 334                         } else if t.AddedScratch == 0 {
 335                                 t.AddedScratch = t.Scratch - t.IncludedScratch
 336                         } else if t.IncludedScratch == 0 {
 337                                 t.IncludedScratch = t.Scratch - t.AddedScratch
 338                         }
 339
 340                         if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 341                                 return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 342                         }
 343                         (*it)[t.Name] = t
 344                 }
 345                 return nil
 346         }
 347         var hash map[string]InstanceType
 348         err := json.Unmarshal(data, &hash)
 349         if err != nil {
 350                 return err
 351         }
 352         // Fill in Name field (and ProviderType field, if not
 353         // specified) using hash key.
 354         *it = InstanceTypeMap(hash)
 355         for name, t := range *it {
 356                 t.Name = name
 357                 if t.ProviderType == "" {
 358                         t.ProviderType = name
 359                 }
 360                 (*it)[name] = t
 361         }
 362         return nil
 363 }
 364
 365 type ServiceName string
 366
 367 const (
 368         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 369         ServiceNameController    ServiceName = "arvados-controller"
 370         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 371         ServiceNameHealth        ServiceName = "arvados-health"
 372         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 373         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 374         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 375         ServiceNameWebsocket     ServiceName = "arvados-ws"
 376         ServiceNameKeepbalance   ServiceName = "keep-balance"
 377         ServiceNameKeepweb       ServiceName = "keep-web"
 378         ServiceNameKeepproxy     ServiceName = "keepproxy"
 379         ServiceNameKeepstore     ServiceName = "keepstore"
 380 )
 381
 382 // Map returns all services as a map, suitable for iterating over all
 383 // services or looking up a service by name.
 384 func (svcs Services) Map() map[ServiceName]Service {
 385         return map[ServiceName]Service{
 386                 ServiceNameRailsAPI:      svcs.RailsAPI,
 387                 ServiceNameController:    svcs.Controller,
 388                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 389                 ServiceNameHealth:        svcs.Health,
 390                 ServiceNameNodemanager:   svcs.Nodemanager,
 391                 ServiceNameWorkbench1:    svcs.Workbench1,
 392                 ServiceNameWorkbench2:    svcs.Workbench2,
 393                 ServiceNameWebsocket:     svcs.Websocket,
 394                 ServiceNameKeepbalance:   svcs.Keepbalance,
 395                 ServiceNameKeepweb:       svcs.WebDAV,
 396                 ServiceNameKeepproxy:     svcs.Keepproxy,
 397                 ServiceNameKeepstore:     svcs.Keepstore,
 398         }
 399 }