X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/2ae40cbbb77f0843662628732b9ca7679a369749..59a972c831bc8f7cd4e896ed8e1c71277b97f04e:/services/keepstore/pull_worker_integration_test.go diff --git a/services/keepstore/pull_worker_integration_test.go b/services/keepstore/pull_worker_integration_test.go index b25d0ab04c..ad7ec15e72 100644 --- a/services/keepstore/pull_worker_integration_test.go +++ b/services/keepstore/pull_worker_integration_test.go @@ -1,17 +1,22 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: AGPL-3.0 + package main import ( - "crypto/tls" - "git.curoverse.com/arvados.git/sdk/go/arvadosclient" - "git.curoverse.com/arvados.git/sdk/go/arvadostest" - "git.curoverse.com/arvados.git/sdk/go/keepclient" - "net/http" - "os" + "bytes" + "context" + "errors" + "io" + "io/ioutil" "strings" - "testing" -) -var keepClient keepclient.KeepClient + "git.arvados.org/arvados.git/sdk/go/arvadostest" + "git.arvados.org/arvados.git/sdk/go/keepclient" + "github.com/prometheus/client_golang/prometheus" + check "gopkg.in/check.v1" +) type PullWorkIntegrationTestData struct { Name string @@ -20,57 +25,31 @@ type PullWorkIntegrationTestData struct { GetError string } -func SetupPullWorkerIntegrationTest(t *testing.T, testData PullWorkIntegrationTestData, wantData bool) PullRequest { - arvadostest.StartAPI() - arvadostest.StartKeep() - - os.Setenv("ARVADOS_API_HOST_INSECURE", "true") - - arv, err := arvadosclient.MakeArvadosClient() - if err != nil { - t.Error("Error creating arv") - } - - client := &http.Client{Transport: &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}} - - keepClient = keepclient.KeepClient{ - Arvados: &arv, - Want_replicas: 1, - Using_proxy: true, - Client: client, - } - - random_token := GenerateRandomApiToken() - keepClient.Arvados.ApiToken = random_token - - if err != nil { - t.Error("Error creating keepclient") +func (s *HandlerSuite) setupPullWorkerIntegrationTest(c *check.C, testData PullWorkIntegrationTestData, wantData bool) PullRequest { + arvadostest.StartKeep(2, false) + c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil) + // Put content if the test needs it + if wantData { + locator, _, err := s.handler.keepClient.PutB([]byte(testData.Content)) + if err != nil { + c.Errorf("Error putting test data in setup for %s %s %v", testData.Content, locator, err) + } + if locator == "" { + c.Errorf("No locator found after putting test data") + } } - servers := make([]string, 1) - servers[0] = "https://" + os.Getenv("ARVADOS_API_HOST") + // Create pullRequest for the test pullRequest := PullRequest{ Locator: testData.Locator, - Servers: servers, - } - - service_roots := make(map[string]string) - for _, addr := range pullRequest.Servers { - service_roots[addr] = addr - } - keepClient.SetServiceRoots(service_roots) - - if wantData { - locator, _, err := keepClient.PutB([]byte(testData.Content)) - if err != nil { - t.Errorf("Error putting test data in setup for %s %s", testData.Content, locator) - } } return pullRequest } -func TestPullWorkerIntegration_GetNonExistingLocator(t *testing.T) { +// Do a get on a block that is not existing in any of the keep servers. +// Expect "block not found" error. +func (s *HandlerSuite) TestPullWorkerIntegration_GetNonExistingLocator(c *check.C) { + c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil) testData := PullWorkIntegrationTestData{ Name: "TestPullWorkerIntegration_GetLocator", Locator: "5d41402abc4b2a76b9719d911017c592", @@ -78,12 +57,17 @@ func TestPullWorkerIntegration_GetNonExistingLocator(t *testing.T) { GetError: "Block not found", } - pullRequest := SetupPullWorkerIntegrationTest(t, testData, false) + pullRequest := s.setupPullWorkerIntegrationTest(c, testData, false) + defer arvadostest.StopAPI() + defer arvadostest.StopKeep(2) - performPullWorkerIntegrationTest(testData, pullRequest, t) + s.performPullWorkerIntegrationTest(testData, pullRequest, c) } -func TestPullWorkerIntegration_GetExistingLocator(t *testing.T) { +// Do a get on a block that exists on one of the keep servers. +// The setup method will create this block before doing the get. +func (s *HandlerSuite) TestPullWorkerIntegration_GetExistingLocator(c *check.C) { + c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil) testData := PullWorkIntegrationTestData{ Name: "TestPullWorkerIntegration_GetLocator", Locator: "5d41402abc4b2a76b9719d911017c592", @@ -91,25 +75,46 @@ func TestPullWorkerIntegration_GetExistingLocator(t *testing.T) { GetError: "", } - pullRequest := SetupPullWorkerIntegrationTest(t, testData, true) + pullRequest := s.setupPullWorkerIntegrationTest(c, testData, true) + defer arvadostest.StopAPI() + defer arvadostest.StopKeep(2) - performPullWorkerIntegrationTest(testData, pullRequest, t) + s.performPullWorkerIntegrationTest(testData, pullRequest, c) } -func performPullWorkerIntegrationTest(testData PullWorkIntegrationTestData, pullRequest PullRequest, t *testing.T) { - err := PullItemAndProcess(pullRequest, keepClient.Arvados.ApiToken, keepClient) +// Perform the test. +// The test directly invokes the "PullItemAndProcess" rather than +// putting an item on the pullq so that the errors can be verified. +func (s *HandlerSuite) performPullWorkerIntegrationTest(testData PullWorkIntegrationTestData, pullRequest PullRequest, c *check.C) { + + // Override writePulledBlock to mock PutBlock functionality + defer func(orig func(*RRVolumeManager, Volume, []byte, string) error) { writePulledBlock = orig }(writePulledBlock) + writePulledBlock = func(_ *RRVolumeManager, _ Volume, content []byte, _ string) error { + c.Check(string(content), check.Equals, testData.Content) + return nil + } + + // Override GetContent to mock keepclient Get functionality + defer func(orig func(string, *keepclient.KeepClient) (io.ReadCloser, int64, string, error)) { + GetContent = orig + }(GetContent) + GetContent = func(signedLocator string, keepClient *keepclient.KeepClient) (reader io.ReadCloser, contentLength int64, url string, err error) { + if testData.GetError != "" { + return nil, 0, "", errors.New(testData.GetError) + } + rdr := ioutil.NopCloser(bytes.NewBufferString(testData.Content)) + return rdr, int64(len(testData.Content)), "", nil + } + + err := s.handler.pullItemAndProcess(pullRequest) if len(testData.GetError) > 0 { if (err == nil) || (!strings.Contains(err.Error(), testData.GetError)) { - t.Fail() + c.Errorf("Got error %v, expected %v", err, testData.GetError) } } else { - t.Fail() - } - - // Override PutContent to mock PutBlock functionality - PutContent = func(content []byte, locator string) (err error) { - // do nothing - return + if err != nil { + c.Errorf("Got error %v, expected nil", err) + } } }