Fix test

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz@veritasgenetics.com>

diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index 7e5b47191e95d93d848df9fe56a6ca0762f6bae2..15bae9af895f33edfc3947c03dd54c1934eb4086 100644 (file)
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -449,6 +449,20 @@ Clusters:
       # stale locks from a previous dispatch process.
       StaleLockTimeout: 1m
 
+      # The crunch-run command to manage the container on a node
+      CrunchRunCommand: "crunch-run"
+
+      # Extra arguments to add to crunch-run invocation
+      # Example: ["--cgroup-parent-subsystem=memory"]
+      CrunchRunArgumentsList: []
+
+      # Extra RAM to reserve on the node, in addition to
+      # the amount specified in the container's RuntimeConstraints
+      ReserveExtraRAM: 256MiB
+
+      # Minimum time between two attempts to run the same container
+      MinRetryPeriod: 0s
+
       Logging:
         # When you run the db:delete_old_container_logs task, it will find
         # containers that have been finished for at least this many seconds,
@@ -492,6 +506,8 @@ Clusters:
         LogUpdateSize: 32MiB
 
       SLURM:
+        PrioritySpread: 0
+        SbatchArgumentsList: []
         Managed:
           # Path to dns server configuration directory
           # (e.g. /etc/unbound.d/conf.d). If false, do not write any config

diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go
index 0a9d7a5b6df6ce8ec00d775081c732d3d7b91a50..58a7690f488d6e57e330ccf5d8eefd157dd88da7 100644 (file)
--- a/lib/config/generated_config.go
+++ b/lib/config/generated_config.go
@@ -455,6 +455,20 @@ Clusters:
       # stale locks from a previous dispatch process.
       StaleLockTimeout: 1m
 
+      # The crunch-run command to manage the container on a node
+      CrunchRunCommand: "crunch-run"
+
+      # Extra arguments to add to crunch-run invocation
+      # Example: ["--cgroup-parent-subsystem=memory"]
+      CrunchRunArgumentsList: []
+
+      # Extra RAM to reserve on the node, in addition to
+      # the amount specified in the container's RuntimeConstraints
+      ReserveExtraRAM: 256MiB
+
+      # Minimum time between two attempts to run the same container
+      MinRetryPeriod: 0s
+
       Logging:
         # When you run the db:delete_old_container_logs task, it will find
         # containers that have been finished for at least this many seconds,
@@ -498,6 +512,8 @@ Clusters:
         LogUpdateSize: 32MiB
 
       SLURM:
+        PrioritySpread: 0
+        SbatchArgumentsList: []
         Managed:
           # Path to dns server configuration directory
           # (e.g. /etc/unbound.d/conf.d). If false, do not write any config

diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
index 889e41095e8c6fb859dfbb45219657381433478c..09e3d591a8b5107082556ef1e30227801fc610c8 100644 (file)
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
@@ -18,10 +18,10 @@ import (
        "strings"
        "time"
 
+       "git.curoverse.com/arvados.git/lib/config"
        "git.curoverse.com/arvados.git/lib/dispatchcloud"
        "git.curoverse.com/arvados.git/sdk/go/arvados"
        "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
-       "git.curoverse.com/arvados.git/sdk/go/config"
        "git.curoverse.com/arvados.git/sdk/go/dispatch"
        "github.com/coreos/go-systemd/daemon"
        "github.com/sirupsen/logrus"
@@ -47,26 +47,6 @@ type Dispatcher struct {
        slurm   Slurm
 
        Client arvados.Client
-
-       SbatchArguments []string
-       PollPeriod      arvados.Duration
-       PrioritySpread  int64
-
-       // crunch-run command to invoke. The container UUID will be
-       // appended. If nil, []string{"crunch-run"} will be used.
-       //
-       // Example: []string{"crunch-run", "--cgroup-parent-subsystem=memory"}
-       CrunchRunCommand []string
-
-       // Extra RAM to reserve (in Bytes) for SLURM job, in addition
-       // to the amount specified in the container's RuntimeConstraints
-       ReserveExtraRAM int64
-
-       // Minimum time between two attempts to run the same container
-       MinRetryPeriod arvados.Duration
-
-       // Batch size for container queries
-       BatchSize int64
 }
 
 func main() {
@@ -97,6 +77,9 @@ func (disp *Dispatcher) configure(prog string, args []string) error {
        flags := flag.NewFlagSet(prog, flag.ExitOnError)
        flags.Usage = func() { usage(flags) }
 
+       loader := config.NewLoader(stdin, log)
+       loader.SetupFlags(flags)
+
        configPath := flags.String(
                "config",
                defaultConfigPath,
@@ -109,9 +92,16 @@ func (disp *Dispatcher) configure(prog string, args []string) error {
                "version",
                false,
                "Print version information and exit.")
+
+       args = loader.MungeLegacyConfigArgs(logrus.StandardLogger(), args, "-crunch-dispatch-slurm-config")
+
        // Parse args; omit the first arg which is the command name
        flags.Parse(args)
 
+       if err == flag.ErrHelp {
+               return nil
+       }
+
        // Print version information if requested
        if *getVersion {
                fmt.Printf("crunch-dispatch-slurm %s\n", version)
@@ -120,18 +110,19 @@ func (disp *Dispatcher) configure(prog string, args []string) error {
 
        disp.logger.Printf("crunch-dispatch-slurm %s started", version)
 
-       err := disp.readConfig(*configPath)
+       cfg, err := loader.Load()
        if err != nil {
                return err
        }
 
-       if disp.CrunchRunCommand == nil {
-               disp.CrunchRunCommand = []string{"crunch-run"}
+       if disp.cluster, err = cfg.GetCluster(""); err != nil {
+               return fmt.Errorf("config error: %s", err)
        }
 
-       if disp.PollPeriod == 0 {
-               disp.PollPeriod = arvados.Duration(10 * time.Second)
-       }
+       disp.Client.APIHost = fmt.Sprintf("%s:%d", disp.cluster.Services.Controller.ExternalURL.Host,
+               disp.cluster.Services.Controller.ExternalURL.Port)
+       disp.Client.AuthToken = disp.cluster.SystemRootToken
+       disp.Client.Insecure = disp.cluster.TLS.Insecure
 
        if disp.Client.APIHost != "" || disp.Client.AuthToken != "" {
                // Copy real configs into env vars so [a]
@@ -150,16 +141,7 @@ func (disp *Dispatcher) configure(prog string, args []string) error {
        }
 
        if *dumpConfig {
-               return config.DumpAndExit(disp)
-       }
-
-       siteConfig, err := arvados.GetConfig(arvados.DefaultConfigFile)
-       if os.IsNotExist(err) {
-               disp.logger.Warnf("no cluster config (%s), proceeding with no node types defined", err)
-       } else if err != nil {
-               return fmt.Errorf("error loading config: %s", err)
-       } else if disp.cluster, err = siteConfig.GetCluster(""); err != nil {
-               return fmt.Errorf("config error: %s", err)
+               return config.DumpAndExit(cfg)
        }
 
        return nil

diff --git a/services/login-sync/Gemfile.lock b/services/login-sync/Gemfile.lock
index eff59d2f72104407615302109bd6967e3a9b3fff..f0283b6114a05fcf7c29eb72b2307671fc0b499c 100644 (file)
--- a/services/login-sync/Gemfile.lock
+++ b/services/login-sync/Gemfile.lock
@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    arvados-login-sync (1.4.0.20190701162225)
+    arvados-login-sync (1.4.0.20190709140013)
       arvados (~> 1.3.0, >= 1.3.0)
 
 GEM