X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/1e03ea4765bee6db9331a1265c8abf2c3e45caff..cf1dcfb8715822f7ac5fe4fce067197dd84bad54:/services/keep-balance/balance_test.go diff --git a/services/keep-balance/balance_test.go b/services/keep-balance/balance_test.go index 8650de141c..423546c46a 100644 --- a/services/keep-balance/balance_test.go +++ b/services/keep-balance/balance_test.go @@ -92,6 +92,7 @@ func (bal *balancerSuite) SetUpTest(c *check.C) { } bal.MinMtime = time.Now().UnixNano() - bal.signatureTTL*1e9 + bal.cleanupMounts() } func (bal *balancerSuite) TestPerfect(c *check.C) { @@ -131,6 +132,75 @@ func (bal *balancerSuite) TestSkipReadonly(c *check.C) { shouldPull: slots{2, 4}}) } +func (bal *balancerSuite) TestMultipleViewsReadOnly(c *check.C) { + bal.testMultipleViews(c, true) +} + +func (bal *balancerSuite) TestMultipleViews(c *check.C) { + bal.testMultipleViews(c, false) +} + +func (bal *balancerSuite) testMultipleViews(c *check.C, readonly bool) { + for i, srv := range bal.srvs { + // Add a mount to each service + srv.mounts[0].KeepMount.DeviceID = fmt.Sprintf("writable-by-srv-%x", i) + srv.mounts = append(srv.mounts, &KeepMount{ + KeepMount: arvados.KeepMount{ + DeviceID: fmt.Sprintf("writable-by-srv-%x", (i+1)%len(bal.srvs)), + UUID: fmt.Sprintf("zzzzz-mount-%015x", i<<16), + ReadOnly: readonly, + Replication: 1, + }, + KeepService: srv, + }) + } + for i := 1; i < len(bal.srvs); i++ { + c.Logf("i=%d", i) + if i == 4 { + // Timestamps are all different, but one of + // the mounts on srv[4] has the same device ID + // where the non-deletable replica is stored + // on srv[3], so only one replica is safe to + // trash. + bal.try(c, tester{ + desired: map[string]int{"default": 1}, + current: slots{0, i, i}, + shouldTrash: slots{i}}) + } else if readonly { + // Timestamps are all different, and the third + // replica can't be trashed because it's on a + // read-only mount, so the first two replicas + // should be trashed. + bal.try(c, tester{ + desired: map[string]int{"default": 1}, + current: slots{0, i, i}, + shouldTrash: slots{0, i}}) + } else { + // Timestamps are all different, so both + // replicas on the non-optimal server should + // be trashed. + bal.try(c, tester{ + desired: map[string]int{"default": 1}, + current: slots{0, i, i}, + shouldTrash: slots{i, i}}) + } + // If the three replicas have identical timestamps, + // none of them can be trashed safely. + bal.try(c, tester{ + desired: map[string]int{"default": 1}, + current: slots{0, i, i}, + timestamps: []int64{12345678, 12345678, 12345678}}) + // If the first and third replicas have identical + // timestamps, only the second replica should be + // trashed. + bal.try(c, tester{ + desired: map[string]int{"default": 1}, + current: slots{0, i, i}, + timestamps: []int64{12345678, 12345679, 12345678}, + shouldTrash: slots{i}}) + } +} + func (bal *balancerSuite) TestFixUnbalanced(c *check.C) { bal.try(c, tester{ desired: map[string]int{"default": 2}, @@ -161,9 +231,10 @@ func (bal *balancerSuite) TestFixUnbalanced(c *check.C) { } func (bal *balancerSuite) TestMultipleReplicasPerService(c *check.C) { - for _, srv := range bal.srvs { + for s, srv := range bal.srvs { for i := 0; i < 3; i++ { m := *(srv.mounts[0]) + m.UUID = fmt.Sprintf("zzzzz-mount-%015x", (s<<10)+i) srv.mounts = append(srv.mounts, &m) } } @@ -237,7 +308,14 @@ func (bal *balancerSuite) TestDecreaseReplBlockTooNew(c *check.C) { bal.try(c, tester{ desired: map[string]int{"default": 2}, current: slots{0, 1, 2}, - timestamps: []int64{oldTime, newTime, newTime + 1}}) + timestamps: []int64{oldTime, newTime, newTime + 1}, + expectResult: balanceResult{ + have: 3, + want: 2, + classState: map[string]balancedBlockState{"default": { + desired: 2, + surplus: 1, + unachievable: false}}}}) // The best replicas are too new to delete, but the excess // replica is old enough. bal.try(c, tester{ @@ -247,12 +325,12 @@ func (bal *balancerSuite) TestDecreaseReplBlockTooNew(c *check.C) { shouldTrash: slots{2}}) } -func (bal *balancerSuite) TestDedupDevices(c *check.C) { +func (bal *balancerSuite) TestCleanupMounts(c *check.C) { bal.srvs[3].mounts[0].KeepMount.ReadOnly = true bal.srvs[3].mounts[0].KeepMount.DeviceID = "abcdef" bal.srvs[14].mounts[0].KeepMount.DeviceID = "abcdef" c.Check(len(bal.srvs[3].mounts), check.Equals, 1) - bal.dedupDevices() + bal.cleanupMounts() c.Check(len(bal.srvs[3].mounts), check.Equals, 0) bal.try(c, tester{ known: 0, @@ -261,6 +339,115 @@ func (bal *balancerSuite) TestDedupDevices(c *check.C) { shouldPull: slots{2}}) } +func (bal *balancerSuite) TestVolumeReplication(c *check.C) { + bal.srvs[0].mounts[0].KeepMount.Replication = 2 // srv 0 + bal.srvs[14].mounts[0].KeepMount.Replication = 2 // srv e + bal.cleanupMounts() + // block 0 rendezvous is 3,e,a -- so slot 1 has repl=2 + bal.try(c, tester{ + known: 0, + desired: map[string]int{"default": 2}, + current: slots{1}, + shouldPull: slots{0}}) + bal.try(c, tester{ + known: 0, + desired: map[string]int{"default": 2}, + current: slots{0, 1}, + shouldPull: nil}) + bal.try(c, tester{ + known: 0, + desired: map[string]int{"default": 2}, + current: slots{0, 1, 2}, + shouldTrash: slots{2}}) + bal.try(c, tester{ + known: 0, + desired: map[string]int{"default": 3}, + current: slots{0, 2, 3, 4}, + shouldPull: slots{1}, + shouldTrash: slots{4}, + expectResult: balanceResult{ + have: 4, + want: 3, + classState: map[string]balancedBlockState{"default": { + desired: 3, + surplus: 1, + unachievable: false}}}}) + bal.try(c, tester{ + known: 0, + desired: map[string]int{"default": 3}, + current: slots{0, 1, 2, 3, 4}, + shouldTrash: slots{2, 3, 4}}) + bal.try(c, tester{ + known: 0, + desired: map[string]int{"default": 4}, + current: slots{0, 1, 2, 3, 4}, + shouldTrash: slots{3, 4}, + expectResult: balanceResult{ + have: 6, + want: 4, + classState: map[string]balancedBlockState{"default": { + desired: 4, + surplus: 2, + unachievable: false}}}}) + // block 1 rendezvous is 0,9,7 -- so slot 0 has repl=2 + bal.try(c, tester{ + known: 1, + desired: map[string]int{"default": 2}, + current: slots{0}, + expectResult: balanceResult{ + have: 2, + want: 2, + classState: map[string]balancedBlockState{"default": { + desired: 2, + surplus: 0, + unachievable: false}}}}) + bal.try(c, tester{ + known: 1, + desired: map[string]int{"default": 3}, + current: slots{0}, + shouldPull: slots{1}}) + bal.try(c, tester{ + known: 1, + desired: map[string]int{"default": 4}, + current: slots{0}, + shouldPull: slots{1, 2}}) + bal.try(c, tester{ + known: 1, + desired: map[string]int{"default": 4}, + current: slots{2}, + shouldPull: slots{0, 1}}) + bal.try(c, tester{ + known: 1, + desired: map[string]int{"default": 4}, + current: slots{7}, + shouldPull: slots{0, 1, 2}, + expectResult: balanceResult{ + have: 1, + want: 4, + classState: map[string]balancedBlockState{"default": { + desired: 4, + surplus: -3, + unachievable: false}}}}) + bal.try(c, tester{ + known: 1, + desired: map[string]int{"default": 2}, + current: slots{1, 2, 3, 4}, + shouldPull: slots{0}, + shouldTrash: slots{3, 4}}) + bal.try(c, tester{ + known: 1, + desired: map[string]int{"default": 2}, + current: slots{0, 1, 2}, + shouldTrash: slots{1, 2}, + expectResult: balanceResult{ + have: 4, + want: 2, + classState: map[string]balancedBlockState{"default": { + desired: 2, + surplus: 2, + unachievable: false}}}}) +} + func (bal *balancerSuite) TestDeviceRWMountedByMultipleServers(c *check.C) { bal.srvs[0].mounts[0].KeepMount.DeviceID = "abcdef" bal.srvs[9].mounts[0].KeepMount.DeviceID = "abcdef" @@ -497,6 +684,9 @@ func (bal *balancerSuite) try(c *check.C, t tester) { if t.expectResult.have > 0 { c.Check(result.have, check.Equals, t.expectResult.have) } + if t.expectResult.want > 0 { + c.Check(result.want, check.Equals, t.expectResult.want) + } if t.expectResult.classState != nil { c.Check(result.classState, check.DeepEquals, t.expectResult.classState) }