services/datamanager/keep/keep_test.go

   1 package keep
   2
   3 import (
   4         "encoding/json"
   5         "fmt"
   6         "net"
   7         "net/http"
   8         "net/http/httptest"
   9         "net/url"
  10         "strconv"
  11         "strings"
  12         "testing"
  13
  14         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
  15         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  16         "git.curoverse.com/arvados.git/sdk/go/blockdigest"
  17         "git.curoverse.com/arvados.git/sdk/go/keepclient"
  18
  19         . "gopkg.in/check.v1"
  20 )
  21
  22 // Gocheck boilerplate
  23 func Test(t *testing.T) {
  24         TestingT(t)
  25 }
  26
  27 type KeepSuite struct{}
  28
  29 var _ = Suite(&KeepSuite{})
  30
  31 type TestHandler struct {
  32         request TrashList
  33 }
  34
  35 func (ts *TestHandler) ServeHTTP(writer http.ResponseWriter, req *http.Request) {
  36         r := json.NewDecoder(req.Body)
  37         r.Decode(&ts.request)
  38 }
  39
  40 func (s *KeepSuite) TestSendTrashLists(c *C) {
  41         th := TestHandler{}
  42         server := httptest.NewServer(&th)
  43         defer server.Close()
  44
  45         tl := map[string]TrashList{
  46                 server.URL: TrashList{TrashRequest{"000000000000000000000000deadbeef", 99}}}
  47
  48         arv := arvadosclient.ArvadosClient{ApiToken: "abc123"}
  49         kc := keepclient.KeepClient{Arvados: &arv, Client: &http.Client{}}
  50         kc.SetServiceRoots(map[string]string{"xxxx": server.URL},
  51                 map[string]string{"xxxx": server.URL},
  52                 map[string]string{})
  53
  54         err := SendTrashLists(nil, &kc, tl, false)
  55
  56         c.Check(err, IsNil)
  57
  58         c.Check(th.request,
  59                 DeepEquals,
  60                 tl[server.URL])
  61
  62 }
  63
  64 type TestHandlerError struct {
  65 }
  66
  67 func (tse *TestHandlerError) ServeHTTP(writer http.ResponseWriter, req *http.Request) {
  68         http.Error(writer, "I'm a teapot", 418)
  69 }
  70
  71 func sendTrashListError(c *C, server *httptest.Server) {
  72         tl := map[string]TrashList{
  73                 server.URL: TrashList{TrashRequest{"000000000000000000000000deadbeef", 99}}}
  74
  75         arv := arvadosclient.ArvadosClient{ApiToken: "abc123"}
  76         kc := keepclient.KeepClient{Arvados: &arv, Client: &http.Client{}}
  77         kc.SetServiceRoots(map[string]string{"xxxx": server.URL},
  78                 map[string]string{"xxxx": server.URL},
  79                 map[string]string{})
  80
  81         err := SendTrashLists(nil, &kc, tl, false)
  82
  83         c.Check(err, NotNil)
  84         c.Check(err[0], NotNil)
  85 }
  86
  87 func (s *KeepSuite) TestSendTrashListErrorResponse(c *C) {
  88         server := httptest.NewServer(&TestHandlerError{})
  89         sendTrashListError(c, server)
  90         defer server.Close()
  91 }
  92
  93 func (s *KeepSuite) TestSendTrashListUnreachable(c *C) {
  94         sendTrashListError(c, httptest.NewUnstartedServer(&TestHandler{}))
  95 }
  96
  97 type APITestData struct {
  98         numServers int
  99         serverType string
 100         statusCode int
 101 }
 102
 103 func (s *KeepSuite) TestGetKeepServers_UnsupportedServiceType(c *C) {
 104         testGetKeepServersFromAPI(c, APITestData{1, "notadisk", 200}, "Found no keepservices with the service type disk")
 105 }
 106
 107 func (s *KeepSuite) TestGetKeepServers_ReceivedTooFewServers(c *C) {
 108         testGetKeepServersFromAPI(c, APITestData{2, "disk", 200}, "Did not receive all available keep servers")
 109 }
 110
 111 func (s *KeepSuite) TestGetKeepServers_ServerError(c *C) {
 112         testGetKeepServersFromAPI(c, APITestData{-1, "disk", -1}, "arvados API server error")
 113 }
 114
 115 func testGetKeepServersFromAPI(c *C, testData APITestData, expectedError string) {
 116         keepServers := ServiceList{
 117                 ItemsAvailable: testData.numServers,
 118                 KeepServers: []ServerAddress{{
 119                         SSL:         false,
 120                         Host:        "example.com",
 121                         Port:        12345,
 122                         UUID:        "abcdefg",
 123                         ServiceType: testData.serverType,
 124                 }},
 125         }
 126
 127         ksJSON, _ := json.Marshal(keepServers)
 128         apiStubResponses := make(map[string]arvadostest.StubResponse)
 129         apiStubResponses["/arvados/v1/keep_services"] = arvadostest.StubResponse{testData.statusCode, string(ksJSON)}
 130         apiStub := arvadostest.ServerStub{apiStubResponses}
 131
 132         api := httptest.NewServer(&apiStub)
 133         defer api.Close()
 134
 135         arv := arvadosclient.ArvadosClient{
 136                 Scheme:    "http",
 137                 ApiServer: api.URL[7:],
 138                 ApiToken:  "abc123",
 139                 Client:    &http.Client{Transport: &http.Transport{}},
 140         }
 141
 142         kc := keepclient.KeepClient{Arvados: &arv, Client: &http.Client{}}
 143         kc.SetServiceRoots(map[string]string{"xxxx": "http://example.com:23456"},
 144                 map[string]string{"xxxx": "http://example.com:23456"},
 145                 map[string]string{})
 146
 147         params := GetKeepServersParams{
 148                 Client: arv,
 149                 Logger: nil,
 150                 Limit:  10,
 151         }
 152
 153         _, err := GetKeepServersAndSummarize(params)
 154         c.Assert(err, NotNil)
 155         c.Assert(err, ErrorMatches, fmt.Sprintf(".*%s.*", expectedError))
 156 }
 157
 158 type KeepServerTestData struct {
 159         // handle /status.json
 160         statusStatusCode int
 161
 162         // handle /index
 163         indexStatusCode   int
 164         indexResponseBody string
 165
 166         // expected error, if any
 167         expectedError string
 168 }
 169
 170 func (s *KeepSuite) TestGetKeepServers_ErrorGettingKeepServerStatus(c *C) {
 171         testGetKeepServersAndSummarize(c, KeepServerTestData{500, 200, "ok",
 172                 ".*http://.* 500 Internal Server Error"})
 173 }
 174
 175 func (s *KeepSuite) TestGetKeepServers_GettingIndex(c *C) {
 176         testGetKeepServersAndSummarize(c, KeepServerTestData{200, -1, "notok",
 177                 ".*redirect-loop.*"})
 178 }
 179
 180 func (s *KeepSuite) TestGetKeepServers_ErrorReadServerResponse(c *C) {
 181         testGetKeepServersAndSummarize(c, KeepServerTestData{200, 500, "notok",
 182                 ".*http://.* 500 Internal Server Error"})
 183 }
 184
 185 func (s *KeepSuite) TestGetKeepServers_ReadServerResponseTuncatedAtLineOne(c *C) {
 186         testGetKeepServersAndSummarize(c, KeepServerTestData{200, 200,
 187                 "notterminatedwithnewline", "Index from http://.* truncated at line 1"})
 188 }
 189
 190 func (s *KeepSuite) TestGetKeepServers_InvalidBlockLocatorPattern(c *C) {
 191         testGetKeepServersAndSummarize(c, KeepServerTestData{200, 200, "testing\n",
 192                 "Error parsing BlockInfo from index line.*"})
 193 }
 194
 195 func (s *KeepSuite) TestGetKeepServers_ReadServerResponseEmpty(c *C) {
 196         testGetKeepServersAndSummarize(c, KeepServerTestData{200, 200, "\n", ""})
 197 }
 198
 199 func (s *KeepSuite) TestGetKeepServers_ReadServerResponseWithTwoBlocks(c *C) {
 200         testGetKeepServersAndSummarize(c, KeepServerTestData{200, 200,
 201                 "51752ba076e461ec9ec1d27400a08548+20 1447526361\na048cc05c02ba1ee43ad071274b9e547+52 1447526362\n\n", ""})
 202 }
 203
 204 func testGetKeepServersAndSummarize(c *C, testData KeepServerTestData) {
 205         ksStubResponses := make(map[string]arvadostest.StubResponse)
 206         ksStubResponses["/status.json"] = arvadostest.StubResponse{testData.statusStatusCode, string(`{}`)}
 207         ksStubResponses["/index"] = arvadostest.StubResponse{testData.indexStatusCode, testData.indexResponseBody}
 208         ksStub := arvadostest.ServerStub{ksStubResponses}
 209         ks := httptest.NewServer(&ksStub)
 210         defer ks.Close()
 211
 212         ksURL, err := url.Parse(ks.URL)
 213         c.Check(err, IsNil)
 214         ksHost, port, err := net.SplitHostPort(ksURL.Host)
 215         ksPort, err := strconv.Atoi(port)
 216         c.Check(err, IsNil)
 217
 218         servers_list := ServiceList{
 219                 ItemsAvailable: 1,
 220                 KeepServers: []ServerAddress{{
 221                         SSL:         false,
 222                         Host:        ksHost,
 223                         Port:        ksPort,
 224                         UUID:        "abcdefg",
 225                         ServiceType: "disk",
 226                 }},
 227         }
 228         ksJSON, _ := json.Marshal(servers_list)
 229         apiStubResponses := make(map[string]arvadostest.StubResponse)
 230         apiStubResponses["/arvados/v1/keep_services"] = arvadostest.StubResponse{200, string(ksJSON)}
 231         apiStub := arvadostest.ServerStub{apiStubResponses}
 232
 233         api := httptest.NewServer(&apiStub)
 234         defer api.Close()
 235
 236         arv := arvadosclient.ArvadosClient{
 237                 Scheme:    "http",
 238                 ApiServer: api.URL[7:],
 239                 ApiToken:  "abc123",
 240                 Client:    &http.Client{Transport: &http.Transport{}},
 241         }
 242
 243         kc := keepclient.KeepClient{Arvados: &arv, Client: &http.Client{}}
 244         kc.SetServiceRoots(map[string]string{"xxxx": ks.URL},
 245                 map[string]string{"xxxx": ks.URL},
 246                 map[string]string{})
 247
 248         params := GetKeepServersParams{
 249                 Client: arv,
 250                 Logger: nil,
 251                 Limit:  10,
 252         }
 253
 254         // GetKeepServersAndSummarize
 255         results, err := GetKeepServersAndSummarize(params)
 256
 257         if testData.expectedError == "" {
 258                 c.Assert(err, IsNil)
 259                 c.Assert(results, NotNil)
 260
 261                 blockToServers := results.BlockToServers
 262
 263                 blockLocators := strings.Split(testData.indexResponseBody, "\n")
 264                 for _, loc := range blockLocators {
 265                         locator := strings.Split(loc, " ")[0]
 266                         if locator != "" {
 267                                 blockLocator, err := blockdigest.ParseBlockLocator(locator)
 268                                 c.Assert(err, IsNil)
 269
 270                                 blockDigestWithSize := blockdigest.DigestWithSize{blockLocator.Digest, uint32(blockLocator.Size)}
 271                                 blockServerInfo := blockToServers[blockDigestWithSize]
 272                                 c.Assert(blockServerInfo[0].Mtime, NotNil)
 273                         }
 274                 }
 275         } else {
 276                 c.Assert(err, ErrorMatches, testData.expectedError)
 277         }
 278 }