1 /* Deals with parsing Collection responses from API Server. */
6 "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
7 "git.curoverse.com/arvados.git/sdk/go/manifest"
8 "git.curoverse.com/arvados.git/sdk/go/util"
12 type Collection struct {
13 BlockDigestToSize map[string]int
19 type ReadCollections struct {
20 ReadAllCollections bool
21 UuidToCollection map[string]Collection
24 type GetCollectionsParams struct {
25 Client arvadosclient.ArvadosClient
27 LogEveryNthCollectionProcessed int // 0 means don't report any
33 func GetCollections(params GetCollectionsParams) (results ReadCollections) {
34 if ¶ms.Client == nil {
35 log.Fatalf("Received params.Client passed to GetCollections() should " +
36 "contain a valid ArvadosClient, but instead it is nil.")
39 fieldsWanted := []string{"manifest_text",
42 // TODO(misha): Start using the redundancy field.
45 sdkParams := arvadosclient.Dict{"select": fieldsWanted}
47 sdkParams["limit"] = params.Limit
50 var collections map[string]interface{}
51 err := params.Client.List("collections", sdkParams, &collections)
53 log.Fatalf("error querying collections: %v", err)
57 var numReceived, numAvailable int
58 results.ReadAllCollections, numReceived, numAvailable =
59 util.SdkListResponseContainsAllAvailableItems(collections)
61 if (!results.ReadAllCollections) {
62 log.Printf("ERROR: Did not receive all collections.")
64 log.Printf("Received %d of %d available collections.",
69 if collectionChannel, err := util.IterateSdkListItems(collections); err != nil {
70 log.Fatalf("Error trying to iterate collections returned by SDK: %v", err)
73 results.UuidToCollection = make(map[string]Collection)
74 for item_map := range collectionChannel {
76 if m := params.LogEveryNthCollectionProcessed; m >0 && (index % m) == 0 {
77 log.Printf("Processing collection #%d", index)
79 collection := Collection{Uuid: item_map["uuid"].(string),
80 OwnerUuid: item_map["owner_uuid"].(string),
81 BlockDigestToSize: make(map[string]int)}
82 manifest := manifest.Manifest{item_map["manifest_text"].(string)}
83 blockChannel := manifest.BlockIterWithDuplicates()
84 for block := range blockChannel {
85 if stored_size, stored := collection.BlockDigestToSize[block.Digest];
86 stored && stored_size != block.Size {
88 "Collection %s contains multiple sizes (%d and %d) for block %s",
94 collection.BlockDigestToSize[block.Digest] = block.Size
96 results.UuidToCollection[collection.Uuid] = collection