+func (bal *balancerSuite) TestCleanupMounts(c *check.C) {
+ bal.srvs[3].mounts[0].KeepMount.ReadOnly = true
+ bal.srvs[3].mounts[0].KeepMount.DeviceID = "abcdef"
+ bal.srvs[14].mounts[0].KeepMount.DeviceID = "abcdef"
+ c.Check(len(bal.srvs[3].mounts), check.Equals, 1)
+ bal.cleanupMounts()
+ c.Check(len(bal.srvs[3].mounts), check.Equals, 0)
+ bal.try(c, tester{
+ known: 0,
+ desired: map[string]int{"default": 2},
+ current: slots{1},
+ shouldPull: slots{2}})
+}
+
+func (bal *balancerSuite) TestVolumeReplication(c *check.C) {
+ bal.srvs[0].mounts[0].KeepMount.Replication = 2 // srv 0
+ bal.srvs[14].mounts[0].KeepMount.Replication = 2 // srv e
+ bal.cleanupMounts()
+ // block 0 rendezvous is 3,e,a -- so slot 1 has repl=2
+ bal.try(c, tester{
+ known: 0,
+ desired: map[string]int{"default": 2},
+ current: slots{1},
+ shouldPull: slots{0}})
+ bal.try(c, tester{
+ known: 0,
+ desired: map[string]int{"default": 2},
+ current: slots{0, 1},
+ shouldPull: nil})
+ bal.try(c, tester{
+ known: 0,
+ desired: map[string]int{"default": 2},
+ current: slots{0, 1, 2},
+ shouldTrash: slots{2}})
+ bal.try(c, tester{
+ known: 0,
+ desired: map[string]int{"default": 3},
+ current: slots{0, 2, 3, 4},
+ shouldPull: slots{1},
+ shouldTrash: slots{4},
+ expectResult: balanceResult{
+ have: 4,
+ want: 3,
+ classState: map[string]balancedBlockState{"default": {
+ desired: 3,
+ surplus: 1,
+ unachievable: false}}}})
+ bal.try(c, tester{
+ known: 0,
+ desired: map[string]int{"default": 3},
+ current: slots{0, 1, 2, 3, 4},
+ shouldTrash: slots{2, 3, 4}})
+ bal.try(c, tester{
+ known: 0,
+ desired: map[string]int{"default": 4},
+ current: slots{0, 1, 2, 3, 4},
+ shouldTrash: slots{3, 4},
+ expectResult: balanceResult{
+ have: 6,
+ want: 4,
+ classState: map[string]balancedBlockState{"default": {
+ desired: 4,
+ surplus: 2,
+ unachievable: false}}}})
+ // block 1 rendezvous is 0,9,7 -- so slot 0 has repl=2
+ bal.try(c, tester{
+ known: 1,
+ desired: map[string]int{"default": 2},
+ current: slots{0},
+ expectResult: balanceResult{
+ have: 2,
+ want: 2,
+ classState: map[string]balancedBlockState{"default": {
+ desired: 2,
+ surplus: 0,
+ unachievable: false}}}})
+ bal.try(c, tester{
+ known: 1,
+ desired: map[string]int{"default": 3},
+ current: slots{0},
+ shouldPull: slots{1}})
+ bal.try(c, tester{
+ known: 1,
+ desired: map[string]int{"default": 4},
+ current: slots{0},
+ shouldPull: slots{1, 2}})
+ bal.try(c, tester{
+ known: 1,
+ desired: map[string]int{"default": 4},
+ current: slots{2},
+ shouldPull: slots{0, 1}})
+ bal.try(c, tester{
+ known: 1,
+ desired: map[string]int{"default": 4},
+ current: slots{7},
+ shouldPull: slots{0, 1, 2},
+ expectResult: balanceResult{
+ have: 1,
+ want: 4,
+ classState: map[string]balancedBlockState{"default": {
+ desired: 4,
+ surplus: -3,
+ unachievable: false}}}})
+ bal.try(c, tester{
+ known: 1,
+ desired: map[string]int{"default": 2},
+ current: slots{1, 2, 3, 4},
+ shouldPull: slots{0},
+ shouldTrash: slots{3, 4}})
+ bal.try(c, tester{
+ known: 1,
+ desired: map[string]int{"default": 2},
+ current: slots{0, 1, 2},
+ shouldTrash: slots{1, 2},
+ expectResult: balanceResult{
+ have: 4,
+ want: 2,
+ classState: map[string]balancedBlockState{"default": {
+ desired: 2,
+ surplus: 2,
+ unachievable: false}}}})
+}
+
+func (bal *balancerSuite) TestDeviceRWMountedByMultipleServers(c *check.C) {
+ bal.srvs[0].mounts[0].KeepMount.DeviceID = "abcdef"
+ bal.srvs[9].mounts[0].KeepMount.DeviceID = "abcdef"
+ bal.srvs[14].mounts[0].KeepMount.DeviceID = "abcdef"
+ // block 0 belongs on servers 3 and e, which have different
+ // device IDs.
+ bal.try(c, tester{
+ known: 0,
+ desired: map[string]int{"default": 2},
+ current: slots{1},
+ shouldPull: slots{0}})
+ // block 1 belongs on servers 0 and 9, which both report
+ // having a replica, but the replicas are on the same device
+ // ID -- so we should pull to the third position (7).
+ bal.try(c, tester{
+ known: 1,
+ desired: map[string]int{"default": 2},
+ current: slots{0, 1},
+ shouldPull: slots{2}})
+ // block 1 can be pulled to the doubly-mounted device, but the
+ // pull should only be done on the first of the two servers.
+ bal.try(c, tester{
+ known: 1,
+ desired: map[string]int{"default": 2},
+ current: slots{2},
+ shouldPull: slots{0}})
+ // block 0 has one replica on a single device mounted on two
+ // servers (e,9 at positions 1,9). Trashing the replica on 9
+ // would lose the block.
+ bal.try(c, tester{
+ known: 0,
+ desired: map[string]int{"default": 2},
+ current: slots{1, 9},
+ shouldPull: slots{0},
+ expectResult: balanceResult{
+ have: 1,
+ classState: map[string]balancedBlockState{"default": {
+ desired: 2,
+ surplus: -1,
+ unachievable: false}}}})
+ // block 0 is overreplicated, but the second and third
+ // replicas are the same replica according to DeviceID
+ // (despite different Mtimes). Don't trash the third replica.
+ bal.try(c, tester{
+ known: 0,
+ desired: map[string]int{"default": 2},
+ current: slots{0, 1, 9},
+ expectResult: balanceResult{
+ have: 2,
+ classState: map[string]balancedBlockState{"default": {
+ desired: 2,
+ surplus: 0,
+ unachievable: false}}}})
+ // block 0 is overreplicated; the third and fifth replicas are
+ // extra, but the fourth is another view of the second and
+ // shouldn't be trashed.
+ bal.try(c, tester{
+ known: 0,
+ desired: map[string]int{"default": 2},
+ current: slots{0, 1, 5, 9, 12},
+ shouldTrash: slots{5, 12},
+ expectResult: balanceResult{
+ have: 4,
+ classState: map[string]balancedBlockState{"default": {
+ desired: 2,
+ surplus: 2,
+ unachievable: false}}}})
+}
+
+func (bal *balancerSuite) TestChangeStorageClasses(c *check.C) {
+ // For known blocks 0/1/2/3, server 9 is slot 9/1/14/0 in
+ // probe order. For these tests we give it two mounts, one
+ // with classes=[special], one with
+ // classes=[special,special2].
+ bal.srvs[9].mounts = []*KeepMount{{
+ KeepMount: arvados.KeepMount{
+ Replication: 1,
+ StorageClasses: []string{"special"},
+ UUID: "zzzzz-mount-special00000009",
+ DeviceID: "9-special",
+ },
+ KeepService: bal.srvs[9],
+ }, {
+ KeepMount: arvados.KeepMount{
+ Replication: 1,
+ StorageClasses: []string{"special", "special2"},
+ UUID: "zzzzz-mount-special20000009",
+ DeviceID: "9-special-and-special2",
+ },
+ KeepService: bal.srvs[9],
+ }}
+ // For known blocks 0/1/2/3, server 13 (d) is slot 5/3/11/1 in
+ // probe order. We give it two mounts, one with
+ // classes=[special3], one with classes=[default].
+ bal.srvs[13].mounts = []*KeepMount{{
+ KeepMount: arvados.KeepMount{
+ Replication: 1,
+ StorageClasses: []string{"special2"},
+ UUID: "zzzzz-mount-special2000000d",
+ DeviceID: "13-special2",
+ },
+ KeepService: bal.srvs[13],
+ }, {
+ KeepMount: arvados.KeepMount{
+ Replication: 1,
+ StorageClasses: []string{"default"},
+ UUID: "zzzzz-mount-00000000000000d",
+ DeviceID: "13-default",
+ },
+ KeepService: bal.srvs[13],
+ }}
+ // Pull to slot 9 because that's the only server with the
+ // desired class "special".
+ bal.try(c, tester{
+ known: 0,
+ desired: map[string]int{"default": 2, "special": 1},
+ current: slots{0, 1},
+ shouldPull: slots{9},
+ shouldPullMounts: []string{"zzzzz-mount-special00000009"}})
+ // If some storage classes are not satisfied, don't trash any
+ // excess replicas. (E.g., if someone desires repl=1 on
+ // class=durable, and we have two copies on class=volatile, we
+ // should wait for pull to succeed before trashing anything).
+ bal.try(c, tester{
+ known: 0,
+ desired: map[string]int{"special": 1},
+ current: slots{0, 1},
+ shouldPull: slots{9},
+ shouldPullMounts: []string{"zzzzz-mount-special00000009"}})
+ // Once storage classes are satisfied, trash excess replicas
+ // that appear earlier in probe order but aren't needed to
+ // satisfy the desired classes.
+ bal.try(c, tester{
+ known: 0,
+ desired: map[string]int{"special": 1},
+ current: slots{0, 1, 9},
+ shouldTrash: slots{0, 1}})
+ // Pull to slot 5, the best server with class "special2".
+ bal.try(c, tester{
+ known: 0,
+ desired: map[string]int{"special2": 1},
+ current: slots{0, 1},
+ shouldPull: slots{5},
+ shouldPullMounts: []string{"zzzzz-mount-special2000000d"}})
+ // Pull to slot 5 and 9 to get replication 2 in desired class
+ // "special2".
+ bal.try(c, tester{
+ known: 0,
+ desired: map[string]int{"special2": 2},
+ current: slots{0, 1},
+ shouldPull: slots{5, 9},
+ shouldPullMounts: []string{"zzzzz-mount-special20000009", "zzzzz-mount-special2000000d"}})
+ // Slot 0 has a replica in "default", slot 1 has a replica
+ // in "special"; we need another replica in "default", i.e.,
+ // on slot 2.
+ bal.try(c, tester{
+ known: 1,
+ desired: map[string]int{"default": 2, "special": 1},
+ current: slots{0, 1},
+ shouldPull: slots{2}})
+ // Pull to best probe position 0 (despite wrong storage class)
+ // if it's impossible to achieve desired replication in the
+ // desired class (only slots 1 and 3 have special2).
+ bal.try(c, tester{
+ known: 1,
+ desired: map[string]int{"special2": 3},
+ current: slots{3},
+ shouldPull: slots{0, 1}})
+ // Trash excess replica.
+ bal.try(c, tester{
+ known: 3,
+ desired: map[string]int{"special": 1},
+ current: slots{0, 1},
+ shouldTrash: slots{1}})
+ // Leave one copy on slot 1 because slot 0 (server 9) only
+ // gives us repl=1.
+ bal.try(c, tester{
+ known: 3,
+ desired: map[string]int{"special": 2},
+ current: slots{0, 1}})
+}
+