Merge branch '21207-pytest'

[arvados.git] / services / keep-balance / block_state.go

diff --git a/services/keep-balance/block_state.go b/services/keep-balance/block_state.go
index 22e89c019ab9fa5a5fb833bf84bbc63df7a4e93b..63a994096bc8c4e1c6f974592c5ee4cf2b6f364f 100644 (file)
--- a/services/keep-balance/block_state.go
+++ b/services/keep-balance/block_state.go
@@ -2,12 +2,12 @@
 //
 // SPDX-License-Identifier: AGPL-3.0
 
-package main
+package keepbalance
 
 import (
        "sync"
 
-       "git.curoverse.com/arvados.git/sdk/go/arvados"
+       "git.arvados.org/arvados.git/sdk/go/arvados"
 )
 
 // Replica is a file on disk (or object in an S3 bucket, or blob in an
@@ -23,6 +23,8 @@ type Replica struct {
 // replicas actually stored (according to the keepstore indexes we
 // know about).
 type BlockState struct {
+       Refs     map[string]bool // pdh => true (only tracked when len(Replicas)==0)
+       RefCount int
        Replicas []Replica
        Desired  map[string]int
        // TODO: Support combinations of classes ("private + durable")
@@ -39,9 +41,22 @@ var defaultClasses = []string{"default"}
 
 func (bs *BlockState) addReplica(r Replica) {
        bs.Replicas = append(bs.Replicas, r)
+       // Free up memory wasted by tracking PDHs that will never be
+       // reported (see comment in increaseDesired)
+       bs.Refs = nil
 }
 
-func (bs *BlockState) increaseDesired(classes []string, n int) {
+func (bs *BlockState) increaseDesired(pdh string, classes []string, n int) {
+       if pdh != "" && len(bs.Replicas) == 0 {
+               // Note we only track PDHs if there's a possibility
+               // that we will report the list of referring PDHs,
+               // i.e., if we haven't yet seen a replica.
+               if bs.Refs == nil {
+                       bs.Refs = map[string]bool{}
+               }
+               bs.Refs[pdh] = true
+       }
+       bs.RefCount++
        if len(classes) == 0 {
                classes = defaultClasses
        }
@@ -107,11 +122,68 @@ func (bsm *BlockStateMap) AddReplicas(mnt *KeepMount, idx []arvados.KeepServiceI
 
 // IncreaseDesired updates the map to indicate the desired replication
 // for the given blocks in the given storage class is at least n.
-func (bsm *BlockStateMap) IncreaseDesired(classes []string, n int, blocks []arvados.SizedDigest) {
+//
+// If pdh is non-empty, it will be tracked and reported in the "lost
+// blocks" report.
+func (bsm *BlockStateMap) IncreaseDesired(pdh string, classes []string, n int, blocks []arvados.SizedDigest) {
        bsm.mutex.Lock()
        defer bsm.mutex.Unlock()
 
        for _, blkid := range blocks {
-               bsm.get(blkid).increaseDesired(classes, n)
+               bsm.get(blkid).increaseDesired(pdh, classes, n)
+       }
+}
+
+// GetConfirmedReplication returns the replication level of the given
+// blocks, considering only the specified storage classes.
+//
+// If len(classes)==0, returns the replication level without regard to
+// storage classes.
+//
+// Safe to call concurrently with other calls to GetCurrent, but not
+// with different BlockStateMap methods.
+func (bsm *BlockStateMap) GetConfirmedReplication(blkids []arvados.SizedDigest, classes []string) int {
+       defaultClasses := map[string]bool{"default": true}
+       min := 0
+       for _, blkid := range blkids {
+               total := 0
+               perclass := make(map[string]int, len(classes))
+               for _, c := range classes {
+                       perclass[c] = 0
+               }
+               bs, ok := bsm.entries[blkid]
+               if !ok {
+                       return 0
+               }
+               for _, r := range bs.Replicas {
+                       total += r.KeepMount.Replication
+                       mntclasses := r.KeepMount.StorageClasses
+                       if len(mntclasses) == 0 {
+                               mntclasses = defaultClasses
+                       }
+                       for c := range mntclasses {
+                               n, ok := perclass[c]
+                               if !ok {
+                                       // Don't care about this storage class
+                                       continue
+                               }
+                               perclass[c] = n + r.KeepMount.Replication
+                       }
+               }
+               if total == 0 {
+                       return 0
+               }
+               for _, n := range perclass {
+                       if n == 0 {
+                               return 0
+                       }
+                       if n < min || min == 0 {
+                               min = n
+                       }
+               }
+               if len(perclass) == 0 && (min == 0 || min > total) {
+                       min = total
+               }
        }
+       return min
 }