X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/2255281ed7406e7c10027ed778f53ee28e6869fa..f9e3d32c92920a806d50548dbb9b6c0aab7d8c54:/services/datamanager/keep/keep_test.go diff --git a/services/datamanager/keep/keep_test.go b/services/datamanager/keep/keep_test.go index 272fad296d..6698849848 100644 --- a/services/datamanager/keep/keep_test.go +++ b/services/datamanager/keep/keep_test.go @@ -2,11 +2,18 @@ package keep import ( "encoding/json" + "fmt" + "net" "net/http" "net/http/httptest" + "net/url" + "strconv" + "strings" "testing" "git.curoverse.com/arvados.git/sdk/go/arvadosclient" + "git.curoverse.com/arvados.git/sdk/go/arvadostest" + "git.curoverse.com/arvados.git/sdk/go/blockdigest" "git.curoverse.com/arvados.git/sdk/go/keepclient" . "gopkg.in/check.v1" @@ -36,7 +43,7 @@ func (s *KeepSuite) TestSendTrashLists(c *C) { defer server.Close() tl := map[string]TrashList{ - server.URL: TrashList{TrashRequest{"000000000000000000000000deadbeef", 99}}} + server.URL: {TrashRequest{"000000000000000000000000deadbeef", 99}}} arv := arvadosclient.ArvadosClient{ApiToken: "abc123"} kc := keepclient.KeepClient{Arvados: &arv, Client: &http.Client{}} @@ -44,8 +51,7 @@ func (s *KeepSuite) TestSendTrashLists(c *C) { map[string]string{"xxxx": server.URL}, map[string]string{}) - err := SendTrashLists(&kc, tl) - server.Close() + err := SendTrashLists(nil, &kc, tl, false) c.Check(err, IsNil) @@ -64,7 +70,7 @@ func (tse *TestHandlerError) ServeHTTP(writer http.ResponseWriter, req *http.Req func sendTrashListError(c *C, server *httptest.Server) { tl := map[string]TrashList{ - server.URL: TrashList{TrashRequest{"000000000000000000000000deadbeef", 99}}} + server.URL: {TrashRequest{"000000000000000000000000deadbeef", 99}}} arv := arvadosclient.ArvadosClient{ApiToken: "abc123"} kc := keepclient.KeepClient{Arvados: &arv, Client: &http.Client{}} @@ -72,7 +78,7 @@ func sendTrashListError(c *C, server *httptest.Server) { map[string]string{"xxxx": server.URL}, map[string]string{}) - err := SendTrashLists(&kc, tl) + err := SendTrashLists(nil, &kc, tl, false) c.Check(err, NotNil) c.Check(err[0], NotNil) @@ -87,3 +93,186 @@ func (s *KeepSuite) TestSendTrashListErrorResponse(c *C) { func (s *KeepSuite) TestSendTrashListUnreachable(c *C) { sendTrashListError(c, httptest.NewUnstartedServer(&TestHandler{})) } + +type APITestData struct { + numServers int + serverType string + statusCode int +} + +func (s *KeepSuite) TestGetKeepServers_UnsupportedServiceType(c *C) { + testGetKeepServersFromAPI(c, APITestData{1, "notadisk", 200}, "Found no keepservices with the service type disk") +} + +func (s *KeepSuite) TestGetKeepServers_ReceivedTooFewServers(c *C) { + testGetKeepServersFromAPI(c, APITestData{2, "disk", 200}, "Did not receive all available keep servers") +} + +func (s *KeepSuite) TestGetKeepServers_ServerError(c *C) { + testGetKeepServersFromAPI(c, APITestData{-1, "disk", -1}, "arvados API server error") +} + +func testGetKeepServersFromAPI(c *C, testData APITestData, expectedError string) { + keepServers := ServiceList{ + ItemsAvailable: testData.numServers, + KeepServers: []ServerAddress{{ + SSL: false, + Host: "example.com", + Port: 12345, + UUID: "abcdefg", + ServiceType: testData.serverType, + }}, + } + + ksJSON, _ := json.Marshal(keepServers) + apiStubResponses := make(map[string]arvadostest.StubResponse) + apiStubResponses["/arvados/v1/keep_services"] = arvadostest.StubResponse{testData.statusCode, string(ksJSON)} + apiStub := arvadostest.ServerStub{apiStubResponses} + + api := httptest.NewServer(&apiStub) + defer api.Close() + + arv := arvadosclient.ArvadosClient{ + Scheme: "http", + ApiServer: api.URL[7:], + ApiToken: "abc123", + Client: &http.Client{Transport: &http.Transport{}}, + } + + kc := keepclient.KeepClient{Arvados: &arv, Client: &http.Client{}} + kc.SetServiceRoots(map[string]string{"xxxx": "http://example.com:23456"}, + map[string]string{"xxxx": "http://example.com:23456"}, + map[string]string{}) + + params := GetKeepServersParams{ + Client: arv, + Logger: nil, + Limit: 10, + } + + _, err := GetKeepServersAndSummarize(params) + c.Assert(err, NotNil) + c.Assert(err, ErrorMatches, fmt.Sprintf(".*%s.*", expectedError)) +} + +type KeepServerTestData struct { + // handle /status.json + statusStatusCode int + + // handle /index + indexStatusCode int + indexResponseBody string + + // expected error, if any + expectedError string +} + +func (s *KeepSuite) TestGetKeepServers_ErrorGettingKeepServerStatus(c *C) { + testGetKeepServersAndSummarize(c, KeepServerTestData{500, 200, "ok", + ".*http://.* 500 Internal Server Error"}) +} + +func (s *KeepSuite) TestGetKeepServers_GettingIndex(c *C) { + testGetKeepServersAndSummarize(c, KeepServerTestData{200, -1, "notok", + ".*redirect-loop.*"}) +} + +func (s *KeepSuite) TestGetKeepServers_ErrorReadServerResponse(c *C) { + testGetKeepServersAndSummarize(c, KeepServerTestData{200, 500, "notok", + ".*http://.* 500 Internal Server Error"}) +} + +func (s *KeepSuite) TestGetKeepServers_ReadServerResponseTuncatedAtLineOne(c *C) { + testGetKeepServersAndSummarize(c, KeepServerTestData{200, 200, + "notterminatedwithnewline", "Index from http://.* truncated at line 1"}) +} + +func (s *KeepSuite) TestGetKeepServers_InvalidBlockLocatorPattern(c *C) { + testGetKeepServersAndSummarize(c, KeepServerTestData{200, 200, "testing\n", + "Error parsing BlockInfo from index line.*"}) +} + +func (s *KeepSuite) TestGetKeepServers_ReadServerResponseEmpty(c *C) { + testGetKeepServersAndSummarize(c, KeepServerTestData{200, 200, "\n", ""}) +} + +func (s *KeepSuite) TestGetKeepServers_ReadServerResponseWithTwoBlocks(c *C) { + testGetKeepServersAndSummarize(c, KeepServerTestData{200, 200, + "51752ba076e461ec9ec1d27400a08548+20 1447526361\na048cc05c02ba1ee43ad071274b9e547+52 1447526362\n\n", ""}) +} + +func testGetKeepServersAndSummarize(c *C, testData KeepServerTestData) { + ksStubResponses := make(map[string]arvadostest.StubResponse) + ksStubResponses["/status.json"] = arvadostest.StubResponse{testData.statusStatusCode, string(`{}`)} + ksStubResponses["/index"] = arvadostest.StubResponse{testData.indexStatusCode, testData.indexResponseBody} + ksStub := arvadostest.ServerStub{ksStubResponses} + ks := httptest.NewServer(&ksStub) + defer ks.Close() + + ksURL, err := url.Parse(ks.URL) + c.Check(err, IsNil) + ksHost, port, err := net.SplitHostPort(ksURL.Host) + ksPort, err := strconv.Atoi(port) + c.Check(err, IsNil) + + servers_list := ServiceList{ + ItemsAvailable: 1, + KeepServers: []ServerAddress{{ + SSL: false, + Host: ksHost, + Port: ksPort, + UUID: "abcdefg", + ServiceType: "disk", + }}, + } + ksJSON, _ := json.Marshal(servers_list) + apiStubResponses := make(map[string]arvadostest.StubResponse) + apiStubResponses["/arvados/v1/keep_services"] = arvadostest.StubResponse{200, string(ksJSON)} + apiStub := arvadostest.ServerStub{apiStubResponses} + + api := httptest.NewServer(&apiStub) + defer api.Close() + + arv := arvadosclient.ArvadosClient{ + Scheme: "http", + ApiServer: api.URL[7:], + ApiToken: "abc123", + Client: &http.Client{Transport: &http.Transport{}}, + } + + kc := keepclient.KeepClient{Arvados: &arv, Client: &http.Client{}} + kc.SetServiceRoots(map[string]string{"xxxx": ks.URL}, + map[string]string{"xxxx": ks.URL}, + map[string]string{}) + + params := GetKeepServersParams{ + Client: arv, + Logger: nil, + Limit: 10, + } + + // GetKeepServersAndSummarize + results, err := GetKeepServersAndSummarize(params) + + if testData.expectedError == "" { + c.Assert(err, IsNil) + c.Assert(results, NotNil) + + blockToServers := results.BlockToServers + + blockLocators := strings.Split(testData.indexResponseBody, "\n") + for _, loc := range blockLocators { + locator := strings.Split(loc, " ")[0] + if locator != "" { + blockLocator, err := blockdigest.ParseBlockLocator(locator) + c.Assert(err, IsNil) + + blockDigestWithSize := blockdigest.DigestWithSize{blockLocator.Digest, uint32(blockLocator.Size)} + blockServerInfo := blockToServers[blockDigestWithSize] + c.Assert(blockServerInfo[0].Mtime, NotNil) + } + } + } else { + c.Assert(err, ErrorMatches, testData.expectedError) + } +}