//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepbalance
import (
"crypto/md5"
"testing"
"time"
- "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
check "gopkg.in/check.v1"
)
shouldPullMounts []string
shouldTrashMounts []string
- expectResult balanceResult
+ expectBlockState *balancedBlockState
+ expectClassState map[string]balancedBlockState
}
func (bal *balancerSuite) SetUpSuite(c *check.C) {
}
bal.signatureTTL = 3600
+ bal.Logger = ctxlog.TestLogger(c)
}
func (bal *balancerSuite) SetUpTest(c *check.C) {
}
srv.mounts = []*KeepMount{{
KeepMount: arvados.KeepMount{
- UUID: fmt.Sprintf("zzzzz-mount-%015x", i),
+ UUID: fmt.Sprintf("zzzzz-mount-%015x", i),
+ StorageClasses: map[string]bool{"default": true},
},
KeepService: srv,
}}
desired: map[string]int{"default": 2},
current: slots{0, 1},
shouldPull: nil,
- shouldTrash: nil})
+ shouldTrash: nil,
+ expectBlockState: &balancedBlockState{
+ needed: 2,
+ }})
}
func (bal *balancerSuite) TestDecreaseRepl(c *check.C) {
bal.try(c, tester{
desired: map[string]int{"default": 2},
current: slots{0, 2, 1},
- shouldTrash: slots{2}})
+ shouldTrash: slots{2},
+ expectBlockState: &balancedBlockState{
+ needed: 2,
+ unneeded: 1,
+ }})
}
func (bal *balancerSuite) TestDecreaseReplToZero(c *check.C) {
bal.try(c, tester{
desired: map[string]int{"default": 0},
current: slots{0, 1, 3},
- shouldTrash: slots{0, 1, 3}})
+ shouldTrash: slots{0, 1, 3},
+ expectBlockState: &balancedBlockState{
+ unneeded: 3,
+ }})
}
func (bal *balancerSuite) TestIncreaseRepl(c *check.C) {
bal.try(c, tester{
desired: map[string]int{"default": 4},
current: slots{0, 1},
- shouldPull: slots{2, 3}})
+ shouldPull: slots{2, 3},
+ expectBlockState: &balancedBlockState{
+ needed: 2,
+ pulling: 2,
+ }})
}
func (bal *balancerSuite) TestSkipReadonly(c *check.C) {
bal.try(c, tester{
desired: map[string]int{"default": 4},
current: slots{0, 1},
- shouldPull: slots{2, 4}})
+ shouldPull: slots{2, 4},
+ expectBlockState: &balancedBlockState{
+ needed: 2,
+ pulling: 2,
+ }})
}
func (bal *balancerSuite) TestMultipleViewsReadOnly(c *check.C) {
srv.mounts[0].KeepMount.DeviceID = fmt.Sprintf("writable-by-srv-%x", i)
srv.mounts = append(srv.mounts, &KeepMount{
KeepMount: arvados.KeepMount{
- DeviceID: fmt.Sprintf("writable-by-srv-%x", (i+1)%len(bal.srvs)),
- UUID: fmt.Sprintf("zzzzz-mount-%015x", i<<16),
- ReadOnly: readonly,
- Replication: 1,
+ DeviceID: bal.srvs[(i+1)%len(bal.srvs)].mounts[0].KeepMount.DeviceID,
+ UUID: bal.srvs[(i+1)%len(bal.srvs)].mounts[0].KeepMount.UUID,
+ ReadOnly: readonly,
+ Replication: 1,
+ StorageClasses: map[string]bool{"default": true},
},
KeepService: srv,
})
desired: map[string]int{"default": 2},
current: slots{0, 1, 2},
timestamps: []int64{12345678, 10000000, 10000000}})
+ bal.try(c, tester{
+ desired: map[string]int{"default": 0},
+ current: slots{0, 1, 2},
+ timestamps: []int64{12345678, 12345678, 12345678},
+ shouldTrash: slots{0},
+ shouldTrashMounts: []string{
+ bal.srvs[bal.knownRendezvous[0][0]].mounts[0].UUID}})
+ bal.try(c, tester{
+ desired: map[string]int{"default": 2},
+ current: slots{0, 1, 2, 5, 6},
+ timestamps: []int64{12345678, 12345679, 10000000, 10000000, 10000000},
+ shouldTrash: slots{2},
+ shouldTrashMounts: []string{
+ bal.srvs[bal.knownRendezvous[0][2]].mounts[0].UUID}})
+ bal.try(c, tester{
+ desired: map[string]int{"default": 2},
+ current: slots{0, 1, 2, 5, 6},
+ timestamps: []int64{12345678, 12345679, 12345671, 10000000, 10000000},
+ shouldTrash: slots{2, 5},
+ shouldTrashMounts: []string{
+ bal.srvs[bal.knownRendezvous[0][2]].mounts[0].UUID,
+ bal.srvs[bal.knownRendezvous[0][5]].mounts[0].UUID}})
+ bal.try(c, tester{
+ desired: map[string]int{"default": 2},
+ current: slots{0, 1, 2, 5, 6},
+ timestamps: []int64{12345678, 12345679, 12345679, 10000000, 10000000},
+ shouldTrash: slots{5},
+ shouldTrashMounts: []string{
+ bal.srvs[bal.knownRendezvous[0][5]].mounts[0].UUID}})
}
func (bal *balancerSuite) TestDecreaseReplBlockTooNew(c *check.C) {
desired: map[string]int{"default": 2},
current: slots{0, 1, 2},
timestamps: []int64{oldTime, newTime, newTime + 1},
- expectResult: balanceResult{
- have: 3,
- want: 2,
- classState: map[string]balancedBlockState{"default": {
- desired: 2,
- surplus: 1,
- unachievable: false}}}})
+ expectBlockState: &balancedBlockState{
+ needed: 2,
+ unneeded: 1,
+ }})
// The best replicas are too new to delete, but the excess
// replica is old enough.
bal.try(c, tester{
func (bal *balancerSuite) TestCleanupMounts(c *check.C) {
bal.srvs[3].mounts[0].KeepMount.ReadOnly = true
bal.srvs[3].mounts[0].KeepMount.DeviceID = "abcdef"
+ bal.srvs[14].mounts[0].KeepMount.UUID = bal.srvs[3].mounts[0].KeepMount.UUID
bal.srvs[14].mounts[0].KeepMount.DeviceID = "abcdef"
c.Check(len(bal.srvs[3].mounts), check.Equals, 1)
bal.cleanupMounts()
known: 0,
desired: map[string]int{"default": 2},
current: slots{1},
- shouldPull: slots{0}})
+ shouldPull: slots{0},
+ expectBlockState: &balancedBlockState{
+ needed: 1,
+ pulling: 1,
+ }})
bal.try(c, tester{
known: 0,
desired: map[string]int{"default": 2},
current: slots{0, 1},
- shouldPull: nil})
+ shouldPull: nil,
+ expectBlockState: &balancedBlockState{
+ needed: 2,
+ }})
bal.try(c, tester{
known: 0,
desired: map[string]int{"default": 2},
current: slots{0, 1, 2},
- shouldTrash: slots{2}})
+ shouldTrash: slots{2},
+ expectBlockState: &balancedBlockState{
+ needed: 2,
+ unneeded: 1,
+ }})
bal.try(c, tester{
known: 0,
desired: map[string]int{"default": 3},
current: slots{0, 2, 3, 4},
shouldPull: slots{1},
shouldTrash: slots{4},
- expectResult: balanceResult{
- have: 4,
- want: 3,
- classState: map[string]balancedBlockState{"default": {
- desired: 3,
- surplus: 1,
- unachievable: false}}}})
+ expectBlockState: &balancedBlockState{
+ needed: 3,
+ unneeded: 1,
+ pulling: 1,
+ }})
bal.try(c, tester{
known: 0,
desired: map[string]int{"default": 3},
current: slots{0, 1, 2, 3, 4},
- shouldTrash: slots{2, 3, 4}})
+ shouldTrash: slots{2, 3, 4},
+ expectBlockState: &balancedBlockState{
+ needed: 2,
+ unneeded: 3,
+ }})
bal.try(c, tester{
known: 0,
desired: map[string]int{"default": 4},
current: slots{0, 1, 2, 3, 4},
shouldTrash: slots{3, 4},
- expectResult: balanceResult{
- have: 6,
- want: 4,
- classState: map[string]balancedBlockState{"default": {
- desired: 4,
- surplus: 2,
- unachievable: false}}}})
+ expectBlockState: &balancedBlockState{
+ needed: 3,
+ unneeded: 2,
+ }})
// block 1 rendezvous is 0,9,7 -- so slot 0 has repl=2
bal.try(c, tester{
known: 1,
desired: map[string]int{"default": 2},
current: slots{0},
- expectResult: balanceResult{
- have: 2,
- want: 2,
- classState: map[string]balancedBlockState{"default": {
- desired: 2,
- surplus: 0,
- unachievable: false}}}})
+ expectBlockState: &balancedBlockState{
+ needed: 1,
+ }})
bal.try(c, tester{
known: 1,
desired: map[string]int{"default": 3},
current: slots{0},
- shouldPull: slots{1}})
+ shouldPull: slots{1},
+ expectBlockState: &balancedBlockState{
+ needed: 1,
+ pulling: 1,
+ }})
bal.try(c, tester{
known: 1,
desired: map[string]int{"default": 4},
current: slots{0},
- shouldPull: slots{1, 2}})
+ shouldPull: slots{1, 2},
+ expectBlockState: &balancedBlockState{
+ needed: 1,
+ pulling: 2,
+ }})
bal.try(c, tester{
known: 1,
desired: map[string]int{"default": 4},
current: slots{2},
- shouldPull: slots{0, 1}})
+ shouldPull: slots{0, 1},
+ expectBlockState: &balancedBlockState{
+ needed: 1,
+ pulling: 2,
+ }})
bal.try(c, tester{
known: 1,
desired: map[string]int{"default": 4},
current: slots{7},
shouldPull: slots{0, 1, 2},
- expectResult: balanceResult{
- have: 1,
- want: 4,
- classState: map[string]balancedBlockState{"default": {
- desired: 4,
- surplus: -3,
- unachievable: false}}}})
+ expectBlockState: &balancedBlockState{
+ needed: 1,
+ pulling: 3,
+ }})
bal.try(c, tester{
known: 1,
desired: map[string]int{"default": 2},
current: slots{1, 2, 3, 4},
shouldPull: slots{0},
- shouldTrash: slots{3, 4}})
+ shouldTrash: slots{3, 4},
+ expectBlockState: &balancedBlockState{
+ needed: 2,
+ unneeded: 2,
+ pulling: 1,
+ }})
bal.try(c, tester{
known: 1,
desired: map[string]int{"default": 2},
current: slots{0, 1, 2},
shouldTrash: slots{1, 2},
- expectResult: balanceResult{
- have: 4,
- want: 2,
- classState: map[string]balancedBlockState{"default": {
- desired: 2,
- surplus: 2,
- unachievable: false}}}})
+ expectBlockState: &balancedBlockState{
+ needed: 1,
+ unneeded: 2,
+ }})
}
func (bal *balancerSuite) TestDeviceRWMountedByMultipleServers(c *check.C) {
- bal.srvs[0].mounts[0].KeepMount.DeviceID = "abcdef"
- bal.srvs[9].mounts[0].KeepMount.DeviceID = "abcdef"
- bal.srvs[14].mounts[0].KeepMount.DeviceID = "abcdef"
+ dupUUID := bal.srvs[0].mounts[0].KeepMount.UUID
+ bal.srvs[9].mounts[0].KeepMount.UUID = dupUUID
+ bal.srvs[14].mounts[0].KeepMount.UUID = dupUUID
// block 0 belongs on servers 3 and e, which have different
- // device IDs.
+ // UUIDs.
bal.try(c, tester{
known: 0,
desired: map[string]int{"default": 2},
current: slots{1},
shouldPull: slots{0}})
// block 1 belongs on servers 0 and 9, which both report
- // having a replica, but the replicas are on the same device
- // ID -- so we should pull to the third position (7).
+ // having a replica, but the replicas are on the same volume
+ // -- so we should pull to the third position (7).
bal.try(c, tester{
known: 1,
desired: map[string]int{"default": 2},
current: slots{0, 1},
shouldPull: slots{2}})
- // block 1 can be pulled to the doubly-mounted device, but the
+ // block 1 can be pulled to the doubly-mounted volume, but the
// pull should only be done on the first of the two servers.
bal.try(c, tester{
known: 1,
desired: map[string]int{"default": 2},
current: slots{2},
shouldPull: slots{0}})
- // block 0 has one replica on a single device mounted on two
+ // block 0 has one replica on a single volume mounted on two
// servers (e,9 at positions 1,9). Trashing the replica on 9
// would lose the block.
bal.try(c, tester{
desired: map[string]int{"default": 2},
current: slots{1, 9},
shouldPull: slots{0},
- expectResult: balanceResult{
- have: 1,
- classState: map[string]balancedBlockState{"default": {
- desired: 2,
- surplus: -1,
- unachievable: false}}}})
+ expectBlockState: &balancedBlockState{
+ needed: 1,
+ pulling: 1,
+ }})
// block 0 is overreplicated, but the second and third
- // replicas are the same replica according to DeviceID
+ // replicas are the same replica according to volume UUID
// (despite different Mtimes). Don't trash the third replica.
bal.try(c, tester{
known: 0,
desired: map[string]int{"default": 2},
current: slots{0, 1, 9},
- expectResult: balanceResult{
- have: 2,
- classState: map[string]balancedBlockState{"default": {
- desired: 2,
- surplus: 0,
- unachievable: false}}}})
+ expectBlockState: &balancedBlockState{
+ needed: 2,
+ }})
// block 0 is overreplicated; the third and fifth replicas are
// extra, but the fourth is another view of the second and
// shouldn't be trashed.
desired: map[string]int{"default": 2},
current: slots{0, 1, 5, 9, 12},
shouldTrash: slots{5, 12},
- expectResult: balanceResult{
- have: 4,
- classState: map[string]balancedBlockState{"default": {
- desired: 2,
- surplus: 2,
- unachievable: false}}}})
+ expectBlockState: &balancedBlockState{
+ needed: 2,
+ unneeded: 2,
+ }})
}
func (bal *balancerSuite) TestChangeStorageClasses(c *check.C) {
bal.srvs[9].mounts = []*KeepMount{{
KeepMount: arvados.KeepMount{
Replication: 1,
- StorageClasses: []string{"special"},
+ StorageClasses: map[string]bool{"special": true},
UUID: "zzzzz-mount-special00000009",
DeviceID: "9-special",
},
}, {
KeepMount: arvados.KeepMount{
Replication: 1,
- StorageClasses: []string{"special", "special2"},
+ StorageClasses: map[string]bool{"special": true, "special2": true},
UUID: "zzzzz-mount-special20000009",
DeviceID: "9-special-and-special2",
},
bal.srvs[13].mounts = []*KeepMount{{
KeepMount: arvados.KeepMount{
Replication: 1,
- StorageClasses: []string{"special2"},
+ StorageClasses: map[string]bool{"special2": true},
UUID: "zzzzz-mount-special2000000d",
DeviceID: "13-special2",
},
}, {
KeepMount: arvados.KeepMount{
Replication: 1,
- StorageClasses: []string{"default"},
+ StorageClasses: map[string]bool{"default": true},
UUID: "zzzzz-mount-00000000000000d",
DeviceID: "13-default",
},
desired: map[string]int{"default": 2, "special": 1},
current: slots{0, 1},
shouldPull: slots{9},
- shouldPullMounts: []string{"zzzzz-mount-special00000009"}})
+ shouldPullMounts: []string{"zzzzz-mount-special20000009"}})
// If some storage classes are not satisfied, don't trash any
// excess replicas. (E.g., if someone desires repl=1 on
// class=durable, and we have two copies on class=volatile, we
desired: map[string]int{"special": 1},
current: slots{0, 1},
shouldPull: slots{9},
- shouldPullMounts: []string{"zzzzz-mount-special00000009"}})
+ shouldPullMounts: []string{"zzzzz-mount-special20000009"}})
// Once storage classes are satisfied, trash excess replicas
// that appear earlier in probe order but aren't needed to
// satisfy the desired classes.
sort.Strings(didTrashMounts)
c.Check(didTrashMounts, check.DeepEquals, t.shouldTrashMounts)
}
- if t.expectResult.have > 0 {
- c.Check(result.have, check.Equals, t.expectResult.have)
- }
- if t.expectResult.want > 0 {
- c.Check(result.want, check.Equals, t.expectResult.want)
+ if t.expectBlockState != nil {
+ c.Check(result.blockState, check.Equals, *t.expectBlockState)
}
- if t.expectResult.classState != nil {
- c.Check(result.classState, check.DeepEquals, t.expectResult.classState)
+ if t.expectClassState != nil {
+ c.Check(result.classState, check.DeepEquals, t.expectClassState)
}
}