Merge branch 'wtsi/14110-c-d-s-limit-slurm-concurrency' refs #14110

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz@veritasgenetics.com>

diff --combined services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
index ce0360261dab4aa3ab424d27c29c782e268b567f,16d9fd18db2765adb6aa05ab1d742660f51790d8..084700d39bfad76b109078f29e81ecf82c40c5be
--- 1/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
--- 2/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
@@@ -23,15 -23,9 +23,15 @@@ import 
        "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
        "git.curoverse.com/arvados.git/sdk/go/config"
        "git.curoverse.com/arvados.git/sdk/go/dispatch"
 +      "github.com/Sirupsen/logrus"
        "github.com/coreos/go-systemd/daemon"
  )
  
 +type logger interface {
 +      dispatch.Logger
 +      Fatalf(string, ...interface{})
 +}
 +
  const initialNiceValue int64 = 10000
  
  var (
@@@ -41,7 -35,6 +41,7 @@@
  
  type Dispatcher struct {
        *dispatch.Dispatcher
 +      logger  logrus.FieldLogger
        cluster *arvados.Cluster
        sqCheck *SqueueChecker
        slurm   Slurm
@@@ -70,17 -63,10 +70,17 @@@
  }
  
  func main() {
 -      disp := &Dispatcher{}
 +      logger := logrus.StandardLogger()
 +      if os.Getenv("DEBUG") != "" {
 +              logger.SetLevel(logrus.DebugLevel)
 +      }
 +      logger.Formatter = &logrus.JSONFormatter{
 +              TimestampFormat: "2006-01-02T15:04:05.000000000Z07:00",
 +      }
 +      disp := &Dispatcher{logger: logger}
        err := disp.Run(os.Args[0], os.Args[1:])
        if err != nil {
 -              log.Fatal(err)
 +              logrus.Fatalf("%s", err)
        }
  }
  
@@@ -118,7 -104,7 +118,7 @@@ func (disp *Dispatcher) configure(prog 
                return nil
        }
  
 -      log.Printf("crunch-dispatch-slurm %s started", version)
 +      disp.logger.Printf("crunch-dispatch-slurm %s started", version)
  
        err := disp.readConfig(*configPath)
        if err != nil {
@@@ -146,7 -132,7 +146,7 @@@
                os.Setenv("ARVADOS_KEEP_SERVICES", strings.Join(disp.Client.KeepServiceURIs, " "))
                os.Setenv("ARVADOS_EXTERNAL_CLIENT", "")
        } else {
 -              log.Printf("warning: Client credentials missing from config, so falling back on environment variables (deprecated).")
 +              disp.logger.Warnf("Client credentials missing from config, so falling back on environment variables (deprecated).")
        }
  
        if *dumpConfig {
@@@ -155,7 -141,7 +155,7 @@@
  
        siteConfig, err := arvados.GetConfig(arvados.DefaultConfigFile)
        if os.IsNotExist(err) {
 -              log.Printf("warning: no cluster config (%s), proceeding with no node types defined", err)
 +              disp.logger.Warnf("no cluster config (%s), proceeding with no node types defined", err)
        } else if err != nil {
                return fmt.Errorf("error loading config: %s", err)
        } else if disp.cluster, err = siteConfig.GetCluster(""); err != nil {
@@@ -167,25 -153,20 +167,25 @@@
  
  // setup() initializes private fields after configure().
  func (disp *Dispatcher) setup() {
 +      if disp.logger == nil {
 +              disp.logger = logrus.StandardLogger()
 +      }
        arv, err := arvadosclient.MakeArvadosClient()
        if err != nil {
 -              log.Fatalf("Error making Arvados client: %v", err)
 +              disp.logger.Fatalf("Error making Arvados client: %v", err)
        }
        arv.Retries = 25
  
-       disp.slurm = &slurmCLI{}
+       disp.slurm = NewSlurmCLI()
        disp.sqCheck = &SqueueChecker{
 +              Logger:         disp.logger,
                Period:         time.Duration(disp.PollPeriod),
                PrioritySpread: disp.PrioritySpread,
                Slurm:          disp.slurm,
        }
        disp.Dispatcher = &dispatch.Dispatcher{
                Arv:            arv,
 +              Logger:         disp.logger,
                BatchSize:      disp.BatchSize,
                RunContainer:   disp.runContainer,
                PollPeriod:     time.Duration(disp.PollPeriod),
@@@ -344,7 -325,7 +344,7 @@@ func (disp *Dispatcher) runContainer(_ 
                case <-ctx.Done():
                        // Disappeared from squeue
                        if err := disp.Arv.Get("containers", ctr.UUID, nil, &ctr); err != nil {
 -                              log.Printf("Error getting final container state for %s: %s", ctr.UUID, err)
 +                              log.Printf("error getting final container state for %s: %s", ctr.UUID, err)
                        }
                        switch ctr.State {
                        case dispatch.Running: