services/datamanager/datamanager.go

   1 /* Keep Datamanager. Responsible for checking on and reporting on Keep Storage */
   2
   3 package main
   4
   5 import (
   6         "flag"
   7         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
   8         "git.curoverse.com/arvados.git/sdk/go/util"
   9         "git.curoverse.com/arvados.git/services/datamanager/collection"
  10         "git.curoverse.com/arvados.git/services/datamanager/keep"
  11         "log"
  12 )
  13
  14 func main() {
  15         flag.Parse()
  16
  17         arv, err := arvadosclient.MakeArvadosClient()
  18         if err != nil {
  19                 log.Fatalf("Error setting up arvados client %s", err.Error())
  20         }
  21
  22         if is_admin, err := util.UserIsAdmin(arv); err != nil {
  23                 log.Fatalf("Error querying current arvados user %s", err.Error())
  24         } else if !is_admin {
  25                 log.Fatalf("Current user is not an admin. Datamanager can only be run by admins.")
  26         }
  27
  28         // TODO(misha): Read Collections and Keep Contents concurrently as goroutines.
  29
  30         // readCollections := collection.GetCollections(
  31         //      collection.GetCollectionsParams{
  32         //              Client: arv, BatchSize: 500})
  33
  34         // UserUsage := ComputeSizeOfOwnedCollections(readCollections)
  35         // log.Printf("Uuid to Size used: %v", UserUsage)
  36
  37         // // TODO(misha): Add a "readonly" flag. If we're in readonly mode,
  38         // // lots of behaviors can become warnings (and obviously we can't
  39         // // write anything).
  40         // // if !readCollections.ReadAllCollections {
  41         // //   log.Fatalf("Did not read all collections")
  42         // // }
  43
  44         // log.Printf("Read and processed %d collections",
  45         //      len(readCollections.UuidToCollection))
  46
  47         readServers := keep.GetKeepServers(
  48                 keep.GetKeepServersParams{Client: arv, Limit: 1000})
  49
  50         log.Printf("Returned %d keep disks", len(readServers.ServerToContents))
  51
  52         blockReplicationCounts := make(map[int]int)
  53         for _, infos := range readServers.BlockToServers {
  54                 replication := len(infos)
  55                 blockReplicationCounts[replication] += 1
  56         }
  57
  58         log.Printf("Replication level distribution: %v", blockReplicationCounts)
  59 }
  60
  61 func ComputeSizeOfOwnedCollections(readCollections collection.ReadCollections) (
  62         results map[string]int) {
  63         results = make(map[string]int)
  64         for _, coll := range readCollections.UuidToCollection {
  65                 results[coll.OwnerUuid] = results[coll.OwnerUuid] + coll.TotalSize
  66         }
  67         return
  68 }