Merge branch '21815-trigrams-exclude-ids'

[arvados.git] / lib / config / load.go

diff --git a/lib/config/load.go b/lib/config/load.go
index abb3a804baab7cb447f9bc55e4bc39d8e8b376ca..93c175062e896cdea74669f21df28358129e489e 100644 (file)
--- a/lib/config/load.go
+++ b/lib/config/load.go
@@ -21,11 +21,12 @@ import (
        "strings"
        "time"
 
+       "dario.cat/mergo"
        "git.arvados.org/arvados.git/sdk/go/arvados"
        "github.com/ghodss/yaml"
-       "github.com/imdario/mergo"
        "github.com/prometheus/client_golang/prometheus"
        "github.com/sirupsen/logrus"
+       "golang.org/x/crypto/ssh"
        "golang.org/x/sys/unix"
 )
 
@@ -47,7 +48,6 @@ type Loader struct {
        CrunchDispatchSlurmPath string
        WebsocketPath           string
        KeepproxyPath           string
-       GitHttpdPath            string
        KeepBalancePath         string
 
        configdata []byte
@@ -87,7 +87,6 @@ func (ldr *Loader) SetupFlags(flagset *flag.FlagSet) {
                flagset.StringVar(&ldr.CrunchDispatchSlurmPath, "legacy-crunch-dispatch-slurm-config", defaultCrunchDispatchSlurmConfigPath, "Legacy crunch-dispatch-slurm configuration `file`")
                flagset.StringVar(&ldr.WebsocketPath, "legacy-ws-config", defaultWebsocketConfigPath, "Legacy arvados-ws configuration `file`")
                flagset.StringVar(&ldr.KeepproxyPath, "legacy-keepproxy-config", defaultKeepproxyConfigPath, "Legacy keepproxy configuration `file`")
-               flagset.StringVar(&ldr.GitHttpdPath, "legacy-git-httpd-config", defaultGitHttpdConfigPath, "Legacy arvados-git-httpd configuration `file`")
                flagset.StringVar(&ldr.KeepBalancePath, "legacy-keepbalance-config", defaultKeepBalanceConfigPath, "Legacy keep-balance configuration `file`")
                flagset.BoolVar(&ldr.SkipLegacy, "skip-legacy", false, "Don't load legacy config files")
        }
@@ -167,9 +166,6 @@ func (ldr *Loader) MungeLegacyConfigArgs(lgr logrus.FieldLogger, args []string,
        if legacyConfigArg != "-legacy-keepproxy-config" {
                ldr.KeepproxyPath = ""
        }
-       if legacyConfigArg != "-legacy-git-httpd-config" {
-               ldr.GitHttpdPath = ""
-       }
        if legacyConfigArg != "-legacy-keepbalance-config" {
                ldr.KeepBalancePath = ""
        }
@@ -295,7 +291,6 @@ func (ldr *Loader) Load() (*arvados.Config, error) {
                        ldr.loadOldCrunchDispatchSlurmConfig,
                        ldr.loadOldWebsocketConfig,
                        ldr.loadOldKeepproxyConfig,
-                       ldr.loadOldGitHttpdConfig,
                        ldr.loadOldKeepBalanceConfig,
                )
        }
@@ -345,6 +340,7 @@ func (ldr *Loader) Load() (*arvados.Config, error) {
                        ldr.checkEnum("Containers.LocalKeepLogsToContainerLog", cc.Containers.LocalKeepLogsToContainerLog, "none", "all", "errors"),
                        ldr.checkEmptyKeepstores(cc),
                        ldr.checkUnlistedKeepstores(cc),
+                       ldr.checkLocalKeepBlobBuffers(cc),
                        ldr.checkStorageClasses(cc),
                        ldr.checkCUDAVersions(cc),
                        // TODO: check non-empty Rendezvous on
@@ -366,7 +362,7 @@ func (ldr *Loader) checkClusterID(label, clusterID string, emptyStringOk bool) e
        if emptyStringOk && clusterID == "" {
                return nil
        } else if !acceptableClusterIDRe.MatchString(clusterID) {
-               return fmt.Errorf("%s: cluster ID should be 5 alphanumeric characters", label)
+               return fmt.Errorf("%s: cluster ID should be 5 lowercase alphanumeric characters", label)
        }
        return nil
 }
@@ -447,6 +443,7 @@ func (ldr *Loader) setLoopbackInstanceType(cfg *arvados.Config) error {
                        RAM:             hostram,
                        Scratch:         scratch,
                        IncludedScratch: scratch,
+                       Price:           1.0,
                }}
                cfg.Clusters[id] = cc
        }
@@ -503,6 +500,24 @@ cluster:
        return nil
 }
 
+func (ldr *Loader) checkLocalKeepBlobBuffers(cc arvados.Cluster) error {
+       kbb := cc.Containers.LocalKeepBlobBuffersPerVCPU
+       if kbb == 0 {
+               return nil
+       }
+       for uuid, vol := range cc.Volumes {
+               if len(vol.AccessViaHosts) > 0 {
+                       ldr.Logger.Warnf("LocalKeepBlobBuffersPerVCPU is %d but will not be used because at least one volume (%s) uses AccessViaHosts -- suggest changing to 0", kbb, uuid)
+                       return nil
+               }
+               if !vol.ReadOnly && vol.Replication < cc.Collections.DefaultReplication {
+                       ldr.Logger.Warnf("LocalKeepBlobBuffersPerVCPU is %d but will not be used because at least one volume (%s) has lower replication than DefaultReplication (%d < %d) -- suggest changing to 0", kbb, uuid, vol.Replication, cc.Collections.DefaultReplication)
+                       return nil
+               }
+       }
+       return nil
+}
+
 func (ldr *Loader) checkStorageClasses(cc arvados.Cluster) error {
        classOnVolume := map[string]bool{}
        for volid, vol := range cc.Volumes {
@@ -670,3 +685,17 @@ func (ldr *Loader) RegisterMetrics(reg *prometheus.Registry) {
        vec.WithLabelValues(hash).Set(float64(ldr.loadTimestamp.UnixNano()) / 1e9)
        reg.MustRegister(vec)
 }
+
+// Load an SSH private key from the given confvalue, which is either
+// the literal key or an absolute path to a file containing the key.
+func LoadSSHKey(confvalue string) (ssh.Signer, error) {
+       if fnm := strings.TrimPrefix(confvalue, "file://"); fnm != confvalue && strings.HasPrefix(fnm, "/") {
+               keydata, err := os.ReadFile(fnm)
+               if err != nil {
+                       return nil, err
+               }
+               return ssh.ParsePrivateKey(keydata)
+       } else {
+               return ssh.ParsePrivateKey([]byte(confvalue))
+       }
+}