services/datamanager/collection/collection.go

   1 /* Deals with parsing Collection responses from API Server. */
   2
   3 package collection
   4
   5 import (
   6         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
   7         "git.curoverse.com/arvados.git/sdk/go/manifest"
   8         "log"
   9 )
  10
  11 type Collection struct {
  12         BlockDigestToSize map[string]int
  13         ReplicationLevel int
  14         Uuid string
  15         OwnerUuid string
  16 }
  17
  18 type readCollections struct {
  19         ReadAllCollections bool
  20         UuidToCollection map[string]Collection
  21 }
  22
  23 func GetCollections(arv arvadosclient.ArvadosClient) (results readCollections) {
  24         fieldsWanted := []string{"manifest_text",
  25                 "owner_uuid",
  26                 "uuid",
  27                 // TODO(misha): Start using the redundancy field.
  28                 "redundancy"}
  29
  30         // TODO(misha): Set the limit param with a flag.
  31         params := arvadosclient.Dict{"limit": 1, "select": fieldsWanted}
  32
  33         var collections map[string]interface{}
  34         err := arv.List("collections", params, &collections)
  35         if err != nil {
  36                 log.Fatalf("error querying collections: %v", err)
  37         }
  38
  39         results.ReadAllCollections = false
  40
  41         if value, ok := collections["items"]; ok {
  42                 items := value.([]interface{})
  43
  44                 {
  45                         var itemsAvailable interface{}
  46                         if itemsAvailable, ok = collections["items_available"]; !ok {
  47                                 log.Fatalf("API server did not return the number of items available")
  48                         }
  49                         numReceived := len(items)
  50                         numAvailable := int(itemsAvailable.(float64))
  51                         results.ReadAllCollections = numReceived == numAvailable
  52
  53                         if (!results.ReadAllCollections) {
  54                                 log.Printf(
  55                                         "ERROR: Did not receive all collections. " +
  56                                                 "Received %d of %d available collections.",
  57                                         numReceived,
  58                                         numAvailable)
  59                         }
  60                 }
  61
  62                 results.UuidToCollection = make(map[string]Collection)
  63                 for _, item := range items {
  64                         item_map := item.(map[string]interface{})
  65                         collection := Collection{Uuid: item_map["uuid"].(string),
  66                                 OwnerUuid: item_map["owner_uuid"].(string),
  67                                 BlockDigestToSize: make(map[string]int)}
  68                         manifest := manifest.Manifest{item_map["manifest_text"].(string)}
  69                         blockChannel := manifest.BlockIterWithDuplicates()
  70                         for block := range blockChannel {
  71                                 if stored_size, stored := collection.BlockDigestToSize[block.Digest];
  72                                 stored && stored_size != block.Size {
  73                                         log.Fatalf(
  74                                                 "Collection %s contains multiple sizes (%d and %d) for block %s",
  75                                                 collection.Uuid,
  76                                                 stored_size,
  77                                                 block.Size,
  78                                                 block.Digest)
  79                                 }
  80                                 collection.BlockDigestToSize[block.Digest] = block.Size
  81                         }
  82                         results.UuidToCollection[collection.Uuid] = collection
  83                 }
  84         }
  85         return
  86 }