services/datamanager/collection/collection_test.go

   1 package collection
   2
   3 import (
   4         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
   5         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
   6         "git.curoverse.com/arvados.git/sdk/go/blockdigest"
   7         . "gopkg.in/check.v1"
   8         "net/http"
   9         "net/http/httptest"
  10         "testing"
  11 )
  12
  13 // Gocheck boilerplate
  14 func Test(t *testing.T) {
  15         TestingT(t)
  16 }
  17
  18 type MySuite struct{}
  19
  20 var _ = Suite(&MySuite{})
  21
  22 // This captures the result we expect from
  23 // ReadCollections.Summarize().  Because CollectionUUIDToIndex is
  24 // indeterminate, we replace BlockToCollectionIndices with
  25 // BlockToCollectionUuids.
  26 type ExpectedSummary struct {
  27         OwnerToCollectionSize     map[string]int
  28         BlockToDesiredReplication map[blockdigest.DigestWithSize]int
  29         BlockToCollectionUuids    map[blockdigest.DigestWithSize][]string
  30 }
  31
  32 func CompareSummarizedReadCollections(c *C,
  33         summarized ReadCollections,
  34         expected ExpectedSummary) {
  35
  36         c.Assert(summarized.OwnerToCollectionSize, DeepEquals,
  37                 expected.OwnerToCollectionSize)
  38
  39         c.Assert(summarized.BlockToDesiredReplication, DeepEquals,
  40                 expected.BlockToDesiredReplication)
  41
  42         summarizedBlockToCollectionUuids :=
  43                 make(map[blockdigest.DigestWithSize]map[string]struct{})
  44         for digest, indices := range summarized.BlockToCollectionIndices {
  45                 uuidSet := make(map[string]struct{})
  46                 summarizedBlockToCollectionUuids[digest] = uuidSet
  47                 for _, index := range indices {
  48                         uuidSet[summarized.CollectionIndexToUUID[index]] = struct{}{}
  49                 }
  50         }
  51
  52         expectedBlockToCollectionUuids :=
  53                 make(map[blockdigest.DigestWithSize]map[string]struct{})
  54         for digest, uuidSlice := range expected.BlockToCollectionUuids {
  55                 uuidSet := make(map[string]struct{})
  56                 expectedBlockToCollectionUuids[digest] = uuidSet
  57                 for _, uuid := range uuidSlice {
  58                         uuidSet[uuid] = struct{}{}
  59                 }
  60         }
  61
  62         c.Assert(summarizedBlockToCollectionUuids, DeepEquals,
  63                 expectedBlockToCollectionUuids)
  64 }
  65
  66 func (s *MySuite) TestSummarizeSimple(checker *C) {
  67         rc := MakeTestReadCollections([]TestCollectionSpec{TestCollectionSpec{
  68                 ReplicationLevel: 5,
  69                 Blocks:           []int{1, 2},
  70         }})
  71
  72         rc.Summarize(nil)
  73
  74         c := rc.UUIDToCollection["col0"]
  75
  76         blockDigest1 := blockdigest.MakeTestDigestWithSize(1)
  77         blockDigest2 := blockdigest.MakeTestDigestWithSize(2)
  78
  79         expected := ExpectedSummary{
  80                 OwnerToCollectionSize:     map[string]int{c.OwnerUUID: c.TotalSize},
  81                 BlockToDesiredReplication: map[blockdigest.DigestWithSize]int{blockDigest1: 5, blockDigest2: 5},
  82                 BlockToCollectionUuids:    map[blockdigest.DigestWithSize][]string{blockDigest1: []string{c.UUID}, blockDigest2: []string{c.UUID}},
  83         }
  84
  85         CompareSummarizedReadCollections(checker, rc, expected)
  86 }
  87
  88 func (s *MySuite) TestSummarizeOverlapping(checker *C) {
  89         rc := MakeTestReadCollections([]TestCollectionSpec{
  90                 TestCollectionSpec{
  91                         ReplicationLevel: 5,
  92                         Blocks:           []int{1, 2},
  93                 },
  94                 TestCollectionSpec{
  95                         ReplicationLevel: 8,
  96                         Blocks:           []int{2, 3},
  97                 },
  98         })
  99
 100         rc.Summarize(nil)
 101
 102         c0 := rc.UUIDToCollection["col0"]
 103         c1 := rc.UUIDToCollection["col1"]
 104
 105         blockDigest1 := blockdigest.MakeTestDigestWithSize(1)
 106         blockDigest2 := blockdigest.MakeTestDigestWithSize(2)
 107         blockDigest3 := blockdigest.MakeTestDigestWithSize(3)
 108
 109         expected := ExpectedSummary{
 110                 OwnerToCollectionSize: map[string]int{
 111                         c0.OwnerUUID: c0.TotalSize,
 112                         c1.OwnerUUID: c1.TotalSize,
 113                 },
 114                 BlockToDesiredReplication: map[blockdigest.DigestWithSize]int{
 115                         blockDigest1: 5,
 116                         blockDigest2: 8,
 117                         blockDigest3: 8,
 118                 },
 119                 BlockToCollectionUuids: map[blockdigest.DigestWithSize][]string{
 120                         blockDigest1: []string{c0.UUID},
 121                         blockDigest2: []string{c0.UUID, c1.UUID},
 122                         blockDigest3: []string{c1.UUID},
 123                 },
 124         }
 125
 126         CompareSummarizedReadCollections(checker, rc, expected)
 127 }
 128
 129 type APITestData struct {
 130         // path and response map
 131         responses map[string]arvadostest.StubResponse
 132
 133         // expected error, if any
 134         expectedError string
 135 }
 136
 137 func (s *MySuite) TestGetCollectionsAndSummarize_DiscoveryError(c *C) {
 138         testGetCollectionsAndSummarize(c,
 139                 APITestData{
 140                         responses:     make(map[string]arvadostest.StubResponse),
 141                         expectedError: "arvados API server error: 500.*",
 142                 })
 143 }
 144
 145 func (s *MySuite) TestGetCollectionsAndSummarize_ApiErrorGetCollections(c *C) {
 146         respMap := make(map[string]arvadostest.StubResponse)
 147         respMap["/discovery/v1/apis/arvados/v1/rest"] = arvadostest.StubResponse{200, `{"defaultCollectionReplication":2}`}
 148         respMap["/arvados/v1/collections"] = arvadostest.StubResponse{-1, ``}
 149
 150         testGetCollectionsAndSummarize(c,
 151                 APITestData{
 152                         responses:     respMap,
 153                         expectedError: "arvados API server error: 302.*",
 154                 })
 155 }
 156
 157 func (s *MySuite) TestGetCollectionsAndSummarize_GetCollectionsBadStreamName(c *C) {
 158         respMap := make(map[string]arvadostest.StubResponse)
 159         respMap["/discovery/v1/apis/arvados/v1/rest"] = arvadostest.StubResponse{200, `{"defaultCollectionReplication":2}`}
 160         respMap["/arvados/v1/collections"] = arvadostest.StubResponse{200, `{"items_available":1,"items":[{"modified_at":"2015-11-24T15:04:05Z","manifest_text":"badstreamname"}]}`}
 161
 162         testGetCollectionsAndSummarize(c,
 163                 APITestData{
 164                         responses:     respMap,
 165                         expectedError: "Invalid stream name: badstreamname",
 166                 })
 167 }
 168
 169 func (s *MySuite) TestGetCollectionsAndSummarize_GetCollectionsBadFileToken(c *C) {
 170         respMap := make(map[string]arvadostest.StubResponse)
 171         respMap["/discovery/v1/apis/arvados/v1/rest"] = arvadostest.StubResponse{200, `{"defaultCollectionReplication":2}`}
 172         respMap["/arvados/v1/collections"] = arvadostest.StubResponse{200, `{"items_available":1,"items":[{"modified_at":"2015-11-24T15:04:05Z","manifest_text":"./goodstream acbd18db4cc2f85cedef654fccc4a4d8+3 0:1:file1.txt file2.txt"}]}`}
 173
 174         testGetCollectionsAndSummarize(c,
 175                 APITestData{
 176                         responses:     respMap,
 177                         expectedError: "Invalid file token: file2.txt",
 178                 })
 179 }
 180
 181 func testGetCollectionsAndSummarize(c *C, testData APITestData) {
 182         apiStub := arvadostest.ServerStub{testData.responses}
 183
 184         api := httptest.NewServer(&apiStub)
 185         defer api.Close()
 186
 187         arv := arvadosclient.ArvadosClient{
 188                 Scheme:    "http",
 189                 ApiServer: api.URL[7:],
 190                 ApiToken:  "abc123",
 191                 Client:    &http.Client{Transport: &http.Transport{}},
 192         }
 193
 194         // GetCollectionsAndSummarize
 195         _, err := GetCollectionsAndSummarize(GetCollectionsParams{arv, nil, 10})
 196
 197         if testData.expectedError == "" {
 198                 c.Assert(err, IsNil)
 199         } else {
 200                 c.Assert(err, ErrorMatches, testData.expectedError)
 201         }
 202 }