services/keep-balance/integration_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package keepbalance
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "io"
  11         "os"
  12         "strings"
  13         "testing"
  14         "time"
  15
  16         "git.arvados.org/arvados.git/lib/config"
  17         "git.arvados.org/arvados.git/sdk/go/arvados"
  18         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  19         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  20         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  21         "git.arvados.org/arvados.git/sdk/go/keepclient"
  22         "github.com/jmoiron/sqlx"
  23         "github.com/prometheus/client_golang/prometheus"
  24         "github.com/sirupsen/logrus"
  25         check "gopkg.in/check.v1"
  26 )
  27
  28 var _ = check.Suite(&integrationSuite{})
  29
  30 type integrationSuite struct {
  31         config     *arvados.Cluster
  32         db         *sqlx.DB
  33         client     *arvados.Client
  34         keepClient *keepclient.KeepClient
  35 }
  36
  37 func (s *integrationSuite) SetUpSuite(c *check.C) {
  38         if testing.Short() {
  39                 c.Skip("-short")
  40         }
  41         arvadostest.ResetEnv()
  42         arvadostest.StartKeep(4, true)
  43
  44         arv, err := arvadosclient.MakeArvadosClient()
  45         arv.ApiToken = arvadostest.SystemRootToken
  46         c.Assert(err, check.IsNil)
  47
  48         s.keepClient, err = keepclient.MakeKeepClient(arv)
  49         c.Assert(err, check.IsNil)
  50         s.keepClient.DiskCacheSize = keepclient.DiskCacheDisabled
  51         s.putReplicas(c, "foo", 4)
  52         s.putReplicas(c, "bar", 1)
  53 }
  54
  55 func (s *integrationSuite) putReplicas(c *check.C, data string, replicas int) {
  56         s.keepClient.Want_replicas = replicas
  57         _, _, err := s.keepClient.PutB([]byte(data))
  58         c.Assert(err, check.IsNil)
  59 }
  60
  61 func (s *integrationSuite) TearDownSuite(c *check.C) {
  62         if testing.Short() {
  63                 c.Skip("-short")
  64         }
  65         arvadostest.StopKeep(4)
  66 }
  67
  68 func (s *integrationSuite) SetUpTest(c *check.C) {
  69         cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
  70         c.Assert(err, check.Equals, nil)
  71         s.config, err = cfg.GetCluster("")
  72         c.Assert(err, check.Equals, nil)
  73         s.db, err = sqlx.Open("postgres", s.config.PostgreSQL.Connection.String())
  74         c.Assert(err, check.IsNil)
  75         s.config.Collections.BalancePeriod = arvados.Duration(time.Second)
  76
  77         s.client = &arvados.Client{
  78                 APIHost:   os.Getenv("ARVADOS_API_HOST"),
  79                 AuthToken: arvadostest.SystemRootToken,
  80                 Insecure:  true,
  81         }
  82 }
  83
  84 func (s *integrationSuite) TestBalanceAPIFixtures(c *check.C) {
  85         var logBuf bytes.Buffer
  86         for iter := 0; iter < 20; iter++ {
  87                 logBuf.Reset()
  88                 logger := logrus.New()
  89                 logger.Out = io.MultiWriter(&logBuf, ctxlog.LogWriter(c.Log))
  90                 opts := RunOptions{
  91                         CommitConfirmedFields: true,
  92                         Logger:                logger,
  93                 }
  94
  95                 bal := &Balancer{
  96                         DB:      s.db,
  97                         Logger:  logger,
  98                         Metrics: newMetrics(prometheus.NewRegistry()),
  99                 }
 100                 nextOpts, err := bal.Run(context.Background(), s.client, s.config, opts)
 101                 c.Check(err, check.IsNil)
 102                 c.Check(nextOpts.SafeRendezvousState, check.Not(check.Equals), "")
 103                 if iter == 0 {
 104                         c.Check(logBuf.String(), check.Matches, `(?ms).*ChangeSet{Pulls:1.*`)
 105                         c.Check(logBuf.String(), check.Not(check.Matches), `(?ms).*ChangeSet{.*Trashes:[^0]}*`)
 106                 } else if !strings.Contains(logBuf.String(), "ChangeSet{Pulls:1") {
 107                         break
 108                 }
 109                 time.Sleep(200 * time.Millisecond)
 110         }
 111         c.Check(logBuf.String(), check.Not(check.Matches), `(?ms).*0 replicas (0 blocks, 0 bytes) underreplicated.*`)
 112
 113         for _, trial := range []struct {
 114                 uuid    string
 115                 repl    int
 116                 classes []string
 117         }{
 118                 {arvadostest.EmptyCollectionUUID, 0, []string{}},
 119                 {arvadostest.FooCollection, 2, []string{"default"}},                                // "foo" blk
 120                 {arvadostest.StorageClassesDesiredDefaultConfirmedDefault, 2, []string{"default"}}, // "bar" blk
 121                 {arvadostest.StorageClassesDesiredArchiveConfirmedDefault, 0, []string{}},          // "bar" blk
 122         } {
 123                 c.Logf("%#v", trial)
 124                 var coll arvados.Collection
 125                 s.client.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+trial.uuid, nil, nil)
 126                 if c.Check(coll.ReplicationConfirmed, check.NotNil) {
 127                         c.Check(*coll.ReplicationConfirmed, check.Equals, trial.repl)
 128                 }
 129                 c.Check(coll.StorageClassesConfirmed, check.DeepEquals, trial.classes)
 130         }
 131 }