sdk/go/arvados/config.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package arvados
   6
   7 import (
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "net/url"
  12
  13         "git.curoverse.com/arvados.git/sdk/go/config"
  14 )
  15
  16 const DefaultConfigFile = "/etc/arvados/config.yml"
  17
  18 type Config struct {
  19         Clusters map[string]Cluster
  20 }
  21
  22 // GetConfig returns the current system config, loading it from
  23 // configFile if needed.
  24 func GetConfig(configFile string) (*Config, error) {
  25         var cfg Config
  26         err := config.LoadFile(&cfg, configFile)
  27         return &cfg, err
  28 }
  29
  30 // GetCluster returns the cluster ID and config for the given
  31 // cluster, or the default/only configured cluster if clusterID is "".
  32 func (sc *Config) GetCluster(clusterID string) (*Cluster, error) {
  33         if clusterID == "" {
  34                 if len(sc.Clusters) == 0 {
  35                         return nil, fmt.Errorf("no clusters configured")
  36                 } else if len(sc.Clusters) > 1 {
  37                         return nil, fmt.Errorf("multiple clusters configured, cannot choose")
  38                 } else {
  39                         for id, cc := range sc.Clusters {
  40                                 cc.ClusterID = id
  41                                 return &cc, nil
  42                         }
  43                 }
  44         }
  45         if cc, ok := sc.Clusters[clusterID]; !ok {
  46                 return nil, fmt.Errorf("cluster %q is not configured", clusterID)
  47         } else {
  48                 cc.ClusterID = clusterID
  49                 return &cc, nil
  50         }
  51 }
  52
  53 type Cluster struct {
  54         ClusterID       string `json:"-"`
  55         ManagementToken string
  56         SystemRootToken string
  57         Services        Services
  58         InstanceTypes   InstanceTypeMap
  59         Containers      ContainersConfig
  60         RemoteClusters  map[string]RemoteCluster
  61         PostgreSQL      PostgreSQL
  62
  63         API struct {
  64                 AsyncPermissionsUpdateInterval Duration
  65                 DisabledAPIs                   []string
  66                 MaxIndexDatabaseRead           int
  67                 MaxItemsPerResponse            int
  68                 MaxRequestAmplification        int
  69                 MaxRequestSize                 int
  70                 RailsSessionSecretToken        string
  71                 RequestTimeout                 Duration
  72         }
  73         AuditLogs struct {
  74                 MaxAge             Duration
  75                 MaxDeleteBatch     int
  76                 UnloggedAttributes []string
  77         }
  78         Collections struct {
  79                 BlobSigning           bool
  80                 BlobSigningKey        string
  81                 DefaultReplication    int
  82                 BlobSigningTTL        Duration
  83                 DefaultTrashLifetime  Duration
  84                 TrashSweepInterval    Duration
  85                 CollectionVersioning  bool
  86                 PreserveVersionIfIdle Duration
  87         }
  88         Git struct {
  89                 Repositories string
  90         }
  91         Login struct {
  92                 ProviderAppSecret string
  93                 ProviderAppID     string
  94         }
  95         Mail struct {
  96                 MailchimpAPIKey                string
  97                 MailchimpListID                string
  98                 SendUserSetupNotificationEmail string
  99                 IssueReporterEmailFrom         string
 100                 IssueReporterEmailTo           string
 101                 SupportEmailAddress            string
 102                 EmailFrom                      string
 103         }
 104         SystemLogs struct {
 105                 LogLevel                string
 106                 Format                  string
 107                 MaxRequestLogParamsSize int
 108         }
 109         TLS struct {
 110                 Certificate string
 111                 Key         string
 112                 Insecure    bool
 113         }
 114         Users struct {
 115                 AdminNotifierEmailFrom                string
 116                 AutoAdminFirstUser                    bool
 117                 AutoAdminUserWithEmail                string
 118                 AutoSetupNewUsers                     bool
 119                 AutoSetupNewUsersWithRepository       bool
 120                 AutoSetupNewUsersWithVmUUID           string
 121                 AutoSetupUsernameBlacklist            []string
 122                 EmailSubjectPrefix                    string
 123                 NewInactiveUserNotificationRecipients []string
 124                 NewUserNotificationRecipients         []string
 125                 NewUsersAreActive                     bool
 126                 UserNotifierEmailFrom                 string
 127                 UserProfileNotificationAddress        string
 128         }
 129 }
 130
 131 type Services struct {
 132         Composer       Service
 133         Controller     Service
 134         DispatchCloud  Service
 135         GitHTTP        Service
 136         GitSSH         Service
 137         Health         Service
 138         Keepbalance    Service
 139         Keepproxy      Service
 140         Keepstore      Service
 141         Nodemanager    Service
 142         RailsAPI       Service
 143         SSO            Service
 144         WebDAVDownload Service
 145         WebDAV         Service
 146         WebShell       Service
 147         Websocket      Service
 148         Workbench1     Service
 149         Workbench2     Service
 150 }
 151
 152 type Service struct {
 153         InternalURLs map[URL]ServiceInstance
 154         ExternalURL  URL
 155 }
 156
 157 // URL is a url.URL that is also usable as a JSON key/value.
 158 type URL url.URL
 159
 160 // UnmarshalText implements encoding.TextUnmarshaler so URL can be
 161 // used as a JSON key/value.
 162 func (su *URL) UnmarshalText(text []byte) error {
 163         u, err := url.Parse(string(text))
 164         if err == nil {
 165                 *su = URL(*u)
 166         }
 167         return err
 168 }
 169
 170 func (su URL) MarshalText() ([]byte, error) {
 171         return []byte(fmt.Sprintf("%s", (*url.URL)(&su).String())), nil
 172 }
 173
 174 type ServiceInstance struct{}
 175
 176 type PostgreSQL struct {
 177         Connection     PostgreSQLConnection
 178         ConnectionPool int
 179 }
 180
 181 type PostgreSQLConnection map[string]string
 182
 183 type RemoteCluster struct {
 184         Host          string
 185         Proxy         bool
 186         Scheme        string
 187         Insecure      bool
 188         ActivateUsers bool
 189 }
 190
 191 type InstanceType struct {
 192         Name            string
 193         ProviderType    string
 194         VCPUs           int
 195         RAM             ByteSize
 196         Scratch         ByteSize
 197         IncludedScratch ByteSize
 198         AddedScratch    ByteSize
 199         Price           float64
 200         Preemptible     bool
 201 }
 202
 203 type ContainersConfig struct {
 204         CloudVMs                    CloudVMsConfig
 205         DefaultKeepCacheRAM         ByteSize
 206         DispatchPrivateKey          string
 207         LogReuseDecisions           bool
 208         MaxComputeVMs               int
 209         MaxDispatchAttempts         int
 210         MaxRetryAttempts            int
 211         StaleLockTimeout            Duration
 212         SupportedDockerImageFormats []string
 213         UsePreemptibleInstances     bool
 214
 215         JobsAPI struct {
 216                 Enable                  string
 217                 GitInternalDir          string
 218                 DefaultDockerImage      string
 219                 CrunchJobWrapper        string
 220                 CrunchJobUser           string
 221                 CrunchRefreshTrigger    string
 222                 ReuseJobIfOutputsDiffer bool
 223         }
 224         Logging struct {
 225                 MaxAge                       Duration
 226                 LogBytesPerEvent             int
 227                 LogSecondsBetweenEvents      int
 228                 LogThrottlePeriod            Duration
 229                 LogThrottleBytes             int
 230                 LogThrottleLines             int
 231                 LimitLogBytesPerJob          int
 232                 LogPartialLineThrottlePeriod Duration
 233                 LogUpdatePeriod              Duration
 234                 LogUpdateSize                ByteSize
 235         }
 236         SLURM struct {
 237                 Managed struct {
 238                         DNSServerConfDir       string
 239                         DNSServerConfTemplate  string
 240                         DNSServerReloadCommand string
 241                         DNSServerUpdateCommand string
 242                         ComputeNodeDomain      string
 243                         ComputeNodeNameservers []string
 244                         AssignNodeHostname     string
 245                 }
 246         }
 247 }
 248
 249 type CloudVMsConfig struct {
 250         Enable bool
 251
 252         BootProbeCommand     string
 253         ImageID              string
 254         MaxCloudOpsPerSecond int
 255         MaxProbesPerSecond   int
 256         PollInterval         Duration
 257         ProbeInterval        Duration
 258         SSHPort              string
 259         SyncInterval         Duration
 260         TimeoutBooting       Duration
 261         TimeoutIdle          Duration
 262         TimeoutProbe         Duration
 263         TimeoutShutdown      Duration
 264         TimeoutSignal        Duration
 265         TimeoutTERM          Duration
 266         ResourceTags         map[string]string
 267         TagKeyPrefix         string
 268
 269         Driver           string
 270         DriverParameters json.RawMessage
 271 }
 272
 273 type InstanceTypeMap map[string]InstanceType
 274
 275 var errDuplicateInstanceTypeName = errors.New("duplicate instance type name")
 276
 277 // UnmarshalJSON handles old config files that provide an array of
 278 // instance types instead of a hash.
 279 func (it *InstanceTypeMap) UnmarshalJSON(data []byte) error {
 280         if len(data) > 0 && data[0] == '[' {
 281                 var arr []InstanceType
 282                 err := json.Unmarshal(data, &arr)
 283                 if err != nil {
 284                         return err
 285                 }
 286                 if len(arr) == 0 {
 287                         *it = nil
 288                         return nil
 289                 }
 290                 *it = make(map[string]InstanceType, len(arr))
 291                 for _, t := range arr {
 292                         if _, ok := (*it)[t.Name]; ok {
 293                                 return errDuplicateInstanceTypeName
 294                         }
 295                         if t.ProviderType == "" {
 296                                 t.ProviderType = t.Name
 297                         }
 298                         if t.Scratch == 0 {
 299                                 t.Scratch = t.IncludedScratch + t.AddedScratch
 300                         } else if t.AddedScratch == 0 {
 301                                 t.AddedScratch = t.Scratch - t.IncludedScratch
 302                         } else if t.IncludedScratch == 0 {
 303                                 t.IncludedScratch = t.Scratch - t.AddedScratch
 304                         }
 305
 306                         if t.Scratch != (t.IncludedScratch + t.AddedScratch) {
 307                                 return fmt.Errorf("%v: Scratch != (IncludedScratch + AddedScratch)", t.Name)
 308                         }
 309                         (*it)[t.Name] = t
 310                 }
 311                 return nil
 312         }
 313         var hash map[string]InstanceType
 314         err := json.Unmarshal(data, &hash)
 315         if err != nil {
 316                 return err
 317         }
 318         // Fill in Name field (and ProviderType field, if not
 319         // specified) using hash key.
 320         *it = InstanceTypeMap(hash)
 321         for name, t := range *it {
 322                 t.Name = name
 323                 if t.ProviderType == "" {
 324                         t.ProviderType = name
 325                 }
 326                 (*it)[name] = t
 327         }
 328         return nil
 329 }
 330
 331 type ServiceName string
 332
 333 const (
 334         ServiceNameRailsAPI      ServiceName = "arvados-api-server"
 335         ServiceNameController    ServiceName = "arvados-controller"
 336         ServiceNameDispatchCloud ServiceName = "arvados-dispatch-cloud"
 337         ServiceNameHealth        ServiceName = "arvados-health"
 338         ServiceNameNodemanager   ServiceName = "arvados-node-manager"
 339         ServiceNameWorkbench1    ServiceName = "arvados-workbench1"
 340         ServiceNameWorkbench2    ServiceName = "arvados-workbench2"
 341         ServiceNameWebsocket     ServiceName = "arvados-ws"
 342         ServiceNameKeepbalance   ServiceName = "keep-balance"
 343         ServiceNameKeepweb       ServiceName = "keep-web"
 344         ServiceNameKeepproxy     ServiceName = "keepproxy"
 345         ServiceNameKeepstore     ServiceName = "keepstore"
 346 )
 347
 348 // Map returns all services as a map, suitable for iterating over all
 349 // services or looking up a service by name.
 350 func (svcs Services) Map() map[ServiceName]Service {
 351         return map[ServiceName]Service{
 352                 ServiceNameRailsAPI:      svcs.RailsAPI,
 353                 ServiceNameController:    svcs.Controller,
 354                 ServiceNameDispatchCloud: svcs.DispatchCloud,
 355                 ServiceNameHealth:        svcs.Health,
 356                 ServiceNameNodemanager:   svcs.Nodemanager,
 357                 ServiceNameWorkbench1:    svcs.Workbench1,
 358                 ServiceNameWorkbench2:    svcs.Workbench2,
 359                 ServiceNameWebsocket:     svcs.Websocket,
 360                 ServiceNameKeepbalance:   svcs.Keepbalance,
 361                 ServiceNameKeepweb:       svcs.WebDAV,
 362                 ServiceNameKeepproxy:     svcs.Keepproxy,
 363                 ServiceNameKeepstore:     svcs.Keepstore,
 364         }
 365 }