services/datamanager/collection/collection_test.go

   1 package collection
   2
   3 import (
   4         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
   5         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
   6         "git.curoverse.com/arvados.git/sdk/go/blockdigest"
   7         . "gopkg.in/check.v1"
   8         "net/http"
   9         "net/http/httptest"
  10         "testing"
  11 )
  12
  13 // Gocheck boilerplate
  14 func Test(t *testing.T) {
  15         TestingT(t)
  16 }
  17
  18 type MySuite struct{}
  19
  20 var _ = Suite(&MySuite{})
  21
  22 // This captures the result we expect from
  23 // ReadCollections.Summarize().  Because CollectionUUIDToIndex is
  24 // indeterminate, we replace BlockToCollectionIndices with
  25 // BlockToCollectionUuids.
  26 type ExpectedSummary struct {
  27         OwnerToCollectionSize     map[string]int
  28         BlockToDesiredReplication map[blockdigest.DigestWithSize]int
  29         BlockToCollectionUuids    map[blockdigest.DigestWithSize][]string
  30 }
  31
  32 func CompareSummarizedReadCollections(c *C,
  33         summarized ReadCollections,
  34         expected ExpectedSummary) {
  35
  36         c.Assert(summarized.OwnerToCollectionSize, DeepEquals,
  37                 expected.OwnerToCollectionSize)
  38
  39         c.Assert(summarized.BlockToDesiredReplication, DeepEquals,
  40                 expected.BlockToDesiredReplication)
  41
  42         summarizedBlockToCollectionUuids :=
  43                 make(map[blockdigest.DigestWithSize]map[string]struct{})
  44         for digest, indices := range summarized.BlockToCollectionIndices {
  45                 uuidSet := make(map[string]struct{})
  46                 summarizedBlockToCollectionUuids[digest] = uuidSet
  47                 for _, index := range indices {
  48                         uuidSet[summarized.CollectionIndexToUUID[index]] = struct{}{}
  49                 }
  50         }
  51
  52         expectedBlockToCollectionUuids :=
  53                 make(map[blockdigest.DigestWithSize]map[string]struct{})
  54         for digest, uuidSlice := range expected.BlockToCollectionUuids {
  55                 uuidSet := make(map[string]struct{})
  56                 expectedBlockToCollectionUuids[digest] = uuidSet
  57                 for _, uuid := range uuidSlice {
  58                         uuidSet[uuid] = struct{}{}
  59                 }
  60         }
  61
  62         c.Assert(summarizedBlockToCollectionUuids, DeepEquals,
  63                 expectedBlockToCollectionUuids)
  64 }
  65
  66 func (s *MySuite) TestSummarizeSimple(checker *C) {
  67         rc := MakeTestReadCollections([]TestCollectionSpec{TestCollectionSpec{
  68                 ReplicationLevel: 5,
  69                 Blocks:           []int{1, 2},
  70         }})
  71
  72         rc.Summarize(nil)
  73
  74         c := rc.UUIDToCollection["col0"]
  75
  76         blockDigest1 := blockdigest.MakeTestDigestWithSize(1)
  77         blockDigest2 := blockdigest.MakeTestDigestWithSize(2)
  78
  79         expected := ExpectedSummary{
  80                 OwnerToCollectionSize:     map[string]int{c.OwnerUUID: c.TotalSize},
  81                 BlockToDesiredReplication: map[blockdigest.DigestWithSize]int{blockDigest1: 5, blockDigest2: 5},
  82                 BlockToCollectionUuids:    map[blockdigest.DigestWithSize][]string{blockDigest1: []string{c.UUID}, blockDigest2: []string{c.UUID}},
  83         }
  84
  85         CompareSummarizedReadCollections(checker, rc, expected)
  86 }
  87
  88 func (s *MySuite) TestSummarizeOverlapping(checker *C) {
  89         rc := MakeTestReadCollections([]TestCollectionSpec{
  90                 TestCollectionSpec{
  91                         ReplicationLevel: 5,
  92                         Blocks:           []int{1, 2},
  93                 },
  94                 TestCollectionSpec{
  95                         ReplicationLevel: 8,
  96                         Blocks:           []int{2, 3},
  97                 },
  98         })
  99
 100         rc.Summarize(nil)
 101
 102         c0 := rc.UUIDToCollection["col0"]
 103         c1 := rc.UUIDToCollection["col1"]
 104
 105         blockDigest1 := blockdigest.MakeTestDigestWithSize(1)
 106         blockDigest2 := blockdigest.MakeTestDigestWithSize(2)
 107         blockDigest3 := blockdigest.MakeTestDigestWithSize(3)
 108
 109         expected := ExpectedSummary{
 110                 OwnerToCollectionSize: map[string]int{
 111                         c0.OwnerUUID: c0.TotalSize,
 112                         c1.OwnerUUID: c1.TotalSize,
 113                 },
 114                 BlockToDesiredReplication: map[blockdigest.DigestWithSize]int{
 115                         blockDigest1: 5,
 116                         blockDigest2: 8,
 117                         blockDigest3: 8,
 118                 },
 119                 BlockToCollectionUuids: map[blockdigest.DigestWithSize][]string{
 120                         blockDigest1: []string{c0.UUID},
 121                         blockDigest2: []string{c0.UUID, c1.UUID},
 122                         blockDigest3: []string{c1.UUID},
 123                 },
 124         }
 125
 126         CompareSummarizedReadCollections(checker, rc, expected)
 127 }
 128
 129 type APITestData struct {
 130         // path and response map
 131         data map[string]arvadostest.StatusAndBody
 132
 133         // expected error, if any
 134         expectedError string
 135 }
 136
 137 func (s *MySuite) TestGetCollectionsAndSummarize_DiscoveryError(c *C) {
 138         testData := APITestData{}
 139         testData.expectedError = "arvados API server error: 500.*"
 140         testGetCollectionsAndSummarize(c, testData)
 141 }
 142
 143 func (s *MySuite) TestGetCollectionsAndSummarize_ApiErrorGetCollections(c *C) {
 144         dataMap := make(map[string]arvadostest.StatusAndBody)
 145         dataMap["/discovery/v1/apis/arvados/v1/rest"] = arvadostest.StatusAndBody{200, `{"defaultCollectionReplication":2}`}
 146         dataMap["/arvados/v1/collections"] = arvadostest.StatusAndBody{-1, ``}
 147
 148         testData := APITestData{}
 149         testData.data = dataMap
 150         testData.expectedError = "arvados API server error: 302.*"
 151
 152         testGetCollectionsAndSummarize(c, testData)
 153 }
 154
 155 func (s *MySuite) TestGetCollectionsAndSummarize_GetCollectionsBadStreamName(c *C) {
 156         dataMap := make(map[string]arvadostest.StatusAndBody)
 157         dataMap["/discovery/v1/apis/arvados/v1/rest"] = arvadostest.StatusAndBody{200, `{"defaultCollectionReplication":2}`}
 158         dataMap["/arvados/v1/collections"] = arvadostest.StatusAndBody{200, `{"items_available":1,"items":[{"modified_at":"2015-11-24T15:04:05Z","manifest_text":"badstreamname"}]}`}
 159
 160         testData := APITestData{}
 161         testData.data = dataMap
 162         testData.expectedError = "Invalid stream name: badstreamname"
 163
 164         testGetCollectionsAndSummarize(c, testData)
 165 }
 166
 167 func (s *MySuite) TestGetCollectionsAndSummarize_GetCollectionsBadFileToken(c *C) {
 168         dataMap := make(map[string]arvadostest.StatusAndBody)
 169         dataMap["/discovery/v1/apis/arvados/v1/rest"] = arvadostest.StatusAndBody{200, `{"defaultCollectionReplication":2}`}
 170         dataMap["/arvados/v1/collections"] = arvadostest.StatusAndBody{200, `{"items_available":1,"items":[{"modified_at":"2015-11-24T15:04:05Z","manifest_text":"./goodstream acbd18db4cc2f85cedef654fccc4a4d8+3 0:1:file1.txt file2.txt"}]}`}
 171
 172         testData := APITestData{}
 173         testData.data = dataMap
 174         testData.expectedError = "Invalid file token: file2.txt"
 175
 176         testGetCollectionsAndSummarize(c, testData)
 177 }
 178
 179 func testGetCollectionsAndSummarize(c *C, testData APITestData) {
 180         apiStub := arvadostest.APIStub{testData.data}
 181
 182         api := httptest.NewServer(&apiStub)
 183         defer api.Close()
 184
 185         arv := arvadosclient.ArvadosClient{
 186                 Scheme:    "http",
 187                 ApiServer: api.URL[7:],
 188                 ApiToken:  "abc123",
 189                 Client:    &http.Client{Transport: &http.Transport{}},
 190         }
 191
 192         // GetCollectionsAndSummarize
 193         results := GetCollectionsAndSummarize(GetCollectionsParams{arv, nil, 10})
 194
 195         if testData.expectedError == "" {
 196                 c.Assert(results.Err, IsNil)
 197                 c.Assert(results, NotNil)
 198         } else {
 199                 c.Assert(results.Err, ErrorMatches, testData.expectedError)
 200         }
 201 }