21254: Fix racy keep-balance test.

[arvados.git] / services / keep-balance / server.go

diff --git a/services/keep-balance/server.go b/services/keep-balance/server.go
index e485f5b2061f28134306d1d897b22cb62e4190e9..7a59c1e8c0edace3a8cb7637c6759a4962d44a99 100644 (file)
--- a/services/keep-balance/server.go
+++ b/services/keep-balance/server.go
@@ -5,12 +5,14 @@
 package keepbalance
 
 import (
+       "context"
        "net/http"
        "os"
        "os/signal"
        "syscall"
        "time"
 
+       "git.arvados.org/arvados.git/lib/controller/dblock"
        "git.arvados.org/arvados.git/sdk/go/arvados"
        "github.com/jmoiron/sqlx"
        "github.com/sirupsen/logrus"
@@ -25,9 +27,8 @@ import (
 // RunOptions fields are controlled by command line flags.
 type RunOptions struct {
        Once                  bool
-       CommitPulls           bool
-       CommitTrash           bool
        CommitConfirmedFields bool
+       ChunkPrefix           string
        Logger                logrus.FieldLogger
        Dumper                logrus.FieldLogger
 
@@ -62,12 +63,12 @@ func (srv *Server) Done() <-chan struct{} {
        return nil
 }
 
-func (srv *Server) run() {
+func (srv *Server) run(ctx context.Context) {
        var err error
        if srv.RunOptions.Once {
-               _, err = srv.runOnce()
+               _, err = srv.runOnce(ctx)
        } else {
-               err = srv.runForever(nil)
+               err = srv.runForever(ctx)
        }
        if err != nil {
                srv.Logger.Error(err)
@@ -77,40 +78,47 @@ func (srv *Server) run() {
        }
 }
 
-func (srv *Server) runOnce() (*Balancer, error) {
+func (srv *Server) runOnce(ctx context.Context) (*Balancer, error) {
        bal := &Balancer{
                DB:             srv.DB,
                Logger:         srv.Logger,
                Dumper:         srv.Dumper,
                Metrics:        srv.Metrics,
                LostBlocksFile: srv.Cluster.Collections.BlobMissingReport,
+               ChunkPrefix:    srv.RunOptions.ChunkPrefix,
        }
        var err error
-       srv.RunOptions, err = bal.Run(srv.ArvClient, srv.Cluster, srv.RunOptions)
+       srv.RunOptions, err = bal.Run(ctx, srv.ArvClient, srv.Cluster, srv.RunOptions)
        return bal, err
 }
 
-// RunForever runs forever, or (for testing purposes) until the given
-// stop channel is ready to receive.
-func (srv *Server) runForever(stop <-chan interface{}) error {
+// RunForever runs forever, or until ctx is cancelled.
+func (srv *Server) runForever(ctx context.Context) error {
        logger := srv.Logger
 
        ticker := time.NewTicker(time.Duration(srv.Cluster.Collections.BalancePeriod))
 
-       // The unbuffered channel here means we only hear SIGUSR1 if
-       // it arrives while we're waiting in select{}.
-       sigUSR1 := make(chan os.Signal)
+       sigUSR1 := make(chan os.Signal, 1)
        signal.Notify(sigUSR1, syscall.SIGUSR1)
+       defer signal.Stop(sigUSR1)
+
+       logger.Info("acquiring service lock")
+       dblock.KeepBalanceService.Lock(ctx, func(context.Context) (*sqlx.DB, error) { return srv.DB, nil })
+       defer dblock.KeepBalanceService.Unlock()
 
        logger.Printf("starting up: will scan every %v and on SIGUSR1", srv.Cluster.Collections.BalancePeriod)
 
        for {
-               if !srv.RunOptions.CommitPulls && !srv.RunOptions.CommitTrash {
+               if srv.Cluster.Collections.BalancePullLimit < 1 && srv.Cluster.Collections.BalanceTrashLimit < 1 {
                        logger.Print("WARNING: Will scan periodically, but no changes will be committed.")
-                       logger.Print("=======  Consider using -commit-pulls and -commit-trash flags.")
+                       logger.Print("=======  To commit changes, set BalancePullLimit and BalanceTrashLimit values greater than zero.")
                }
 
-               _, err := srv.runOnce()
+               if !dblock.KeepBalanceService.Check() {
+                       // context canceled
+                       return nil
+               }
+               _, err := srv.runOnce(ctx)
                if err != nil {
                        logger.Print("run failed: ", err)
                } else {
@@ -118,8 +126,7 @@ func (srv *Server) runForever(stop <-chan interface{}) error {
                }
 
                select {
-               case <-stop:
-                       signal.Stop(sigUSR1)
+               case <-ctx.Done():
                        return nil
                case <-ticker.C:
                        logger.Print("timer went off")
@@ -128,8 +135,7 @@ func (srv *Server) runForever(stop <-chan interface{}) error {
                        // Reset the timer so we don't start the N+1st
                        // run too soon after the Nth run is triggered
                        // by SIGUSR1.
-                       ticker.Stop()
-                       ticker = time.NewTicker(time.Duration(srv.Cluster.Collections.BalancePeriod))
+                       ticker.Reset(time.Duration(srv.Cluster.Collections.BalancePeriod))
                }
                logger.Print("starting next run")
        }