X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/2581a27602bd98fa532926efc02d9401ee095108..bb99d90e25a47dff9405fb6f68be0779dd2d8989:/tools/keep-block-check/keep-block-check_test.go diff --git a/tools/keep-block-check/keep-block-check_test.go b/tools/keep-block-check/keep-block-check_test.go index 65d3e3434b..d973e06027 100644 --- a/tools/keep-block-check/keep-block-check_test.go +++ b/tools/keep-block-check/keep-block-check_test.go @@ -1,3 +1,7 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: AGPL-3.0 + package main import ( @@ -10,9 +14,11 @@ import ( "regexp" "strings" "testing" + "time" - "git.curoverse.com/arvados.git/sdk/go/arvadostest" - "git.curoverse.com/arvados.git/sdk/go/keepclient" + "git.arvados.org/arvados.git/sdk/go/arvadosclient" + "git.arvados.org/arvados.git/sdk/go/arvadostest" + "git.arvados.org/arvados.git/sdk/go/keepclient" . "gopkg.in/check.v1" ) @@ -35,12 +41,9 @@ var logBuffer bytes.Buffer var TestHash = "aaaa09c290d0fb1ca068ffaddf22cbd0" var TestHash2 = "aaaac516f788aec4f30932ffb6395c39" -func (s *ServerRequiredSuite) SetUpSuite(c *C) { - arvadostest.StartAPI() -} +var blobSignatureTTL = time.Duration(2*7*24) * time.Hour func (s *ServerRequiredSuite) TearDownSuite(c *C) { - arvadostest.StopAPI() arvadostest.ResetEnv() } @@ -61,6 +64,7 @@ func (s *DoMainTestSuite) SetUpSuite(c *C) { func (s *DoMainTestSuite) SetUpTest(c *C) { logOutput := io.MultiWriter(&logBuffer) log.SetOutput(logOutput) + keepclient.RefreshServiceDiscovery() } func (s *DoMainTestSuite) TearDownTest(c *C) { @@ -69,18 +73,25 @@ func (s *DoMainTestSuite) TearDownTest(c *C) { } func setupKeepBlockCheck(c *C, enforcePermissions bool, keepServicesJSON string) { + setupKeepBlockCheckWithTTL(c, enforcePermissions, keepServicesJSON, blobSignatureTTL) +} + +func setupKeepBlockCheckWithTTL(c *C, enforcePermissions bool, keepServicesJSON string, ttl time.Duration) { var config apiConfig config.APIHost = os.Getenv("ARVADOS_API_HOST") config.APIToken = arvadostest.DataManagerToken - config.APIHostInsecure = matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE")) + config.APIHostInsecure = arvadosclient.StringBool(os.Getenv("ARVADOS_API_HOST_INSECURE")) // Start Keep servers arvadostest.StartKeep(2, enforcePermissions) // setup keepclients var err error - kc, err = setupKeepClient(config, keepServicesJSON) + kc, ttl, err = setupKeepClient(config, keepServicesJSON, ttl) + c.Assert(ttl, Equals, blobSignatureTTL) c.Check(err, IsNil) + + keepclient.RefreshServiceDiscovery() } // Setup test data @@ -136,9 +147,8 @@ func setupBlockHashFile(c *C, name string, blocks []string) string { func checkErrorLog(c *C, blocks []string, prefix, suffix string) { for _, hash := range blocks { - expected := prefix + `.*` + hash + `.*` + suffix - match, _ := regexp.MatchString(expected, logBuffer.String()) - c.Assert(match, Equals, true) + expected := `(?ms).*` + prefix + `.*` + hash + `.*` + suffix + `.*` + c.Check(logBuffer.String(), Matches, expected) } } @@ -151,7 +161,7 @@ func checkNoErrorsLogged(c *C, prefix, suffix string) { func (s *ServerRequiredSuite) TestBlockCheck(c *C) { setupKeepBlockCheck(c, false, "") allLocators := setupTestData(c) - err := performKeepBlockCheck(kc, "", allLocators, true) + err := performKeepBlockCheck(kc, blobSignatureTTL, "", allLocators, true) c.Check(err, IsNil) checkNoErrorsLogged(c, "Error verifying block", "Block not found") } @@ -159,7 +169,15 @@ func (s *ServerRequiredSuite) TestBlockCheck(c *C) { func (s *ServerRequiredSuite) TestBlockCheckWithBlobSigning(c *C) { setupKeepBlockCheck(c, true, "") allLocators := setupTestData(c) - err := performKeepBlockCheck(kc, arvadostest.BlobSigningKey, allLocators, true) + err := performKeepBlockCheck(kc, blobSignatureTTL, arvadostest.BlobSigningKey, allLocators, true) + c.Check(err, IsNil) + checkNoErrorsLogged(c, "Error verifying block", "Block not found") +} + +func (s *ServerRequiredSuite) TestBlockCheckWithBlobSigningAndTTLFromDiscovery(c *C) { + setupKeepBlockCheckWithTTL(c, true, "", 0) + allLocators := setupTestData(c) + err := performKeepBlockCheck(kc, blobSignatureTTL, arvadostest.BlobSigningKey, allLocators, true) c.Check(err, IsNil) checkNoErrorsLogged(c, "Error verifying block", "Block not found") } @@ -169,9 +187,9 @@ func (s *ServerRequiredSuite) TestBlockCheck_NoSuchBlock(c *C) { allLocators := setupTestData(c) allLocators = append(allLocators, TestHash) allLocators = append(allLocators, TestHash2) - err := performKeepBlockCheck(kc, "", allLocators, true) + err := performKeepBlockCheck(kc, blobSignatureTTL, "", allLocators, true) c.Check(err, NotNil) - c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 7 blocks with matching prefix.") + c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 7 blocks with matching prefix") checkErrorLog(c, []string{TestHash, TestHash2}, "Error verifying block", "Block not found") } @@ -184,10 +202,10 @@ func (s *ServerRequiredSuite) TestBlockCheck_NoSuchBlock_WithMatchingPrefix(c *C defer os.Remove(locatorFile) locators, err := getBlockLocators(locatorFile, "aaa") c.Check(err, IsNil) - err = performKeepBlockCheck(kc, "", locators, true) + err = performKeepBlockCheck(kc, blobSignatureTTL, "", locators, true) c.Check(err, NotNil) // Of the 7 blocks in allLocators, only two match the prefix and hence only those are checked - c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix.") + c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix") checkErrorLog(c, []string{TestHash, TestHash2}, "Error verifying block", "Block not found") } @@ -200,15 +218,15 @@ func (s *ServerRequiredSuite) TestBlockCheck_NoSuchBlock_WithPrefixMismatch(c *C defer os.Remove(locatorFile) locators, err := getBlockLocators(locatorFile, "999") c.Check(err, IsNil) - err = performKeepBlockCheck(kc, "", locators, true) + err = performKeepBlockCheck(kc, blobSignatureTTL, "", locators, true) c.Check(err, IsNil) // there were no matching locators in file and hence nothing was checked } func (s *ServerRequiredSuite) TestBlockCheck_BadSignature(c *C) { setupKeepBlockCheck(c, true, "") setupTestData(c) - err := performKeepBlockCheck(kc, "badblobsigningkey", []string{TestHash, TestHash2}, false) - c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix.") + err := performKeepBlockCheck(kc, blobSignatureTTL, "badblobsigningkey", []string{TestHash, TestHash2}, false) + c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix") checkErrorLog(c, []string{TestHash, TestHash2}, "Error verifying block", "HTTP 403") // verbose logging not requested c.Assert(strings.Contains(logBuffer.String(), "Verifying block 1 of 2"), Equals, false) @@ -243,8 +261,8 @@ var testKeepServicesJSON = `{ // Expect error during performKeepBlockCheck due to unreachable keepservers. func (s *ServerRequiredSuite) TestErrorDuringKeepBlockCheck_FakeKeepservers(c *C) { setupKeepBlockCheck(c, false, testKeepServicesJSON) - err := performKeepBlockCheck(kc, "", []string{TestHash, TestHash2}, true) - c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix.") + err := performKeepBlockCheck(kc, blobSignatureTTL, "", []string{TestHash, TestHash2}, true) + c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix") checkErrorLog(c, []string{TestHash, TestHash2}, "Error verifying block", "") } @@ -272,23 +290,25 @@ func (s *ServerRequiredSuite) TestLoadConfig(c *C) { c.Assert(config.APIHost, Equals, os.Getenv("ARVADOS_API_HOST")) c.Assert(config.APIToken, Equals, arvadostest.DataManagerToken) - c.Assert(config.APIHostInsecure, Equals, matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))) + c.Assert(config.APIHostInsecure, Equals, arvadosclient.StringBool(os.Getenv("ARVADOS_API_HOST_INSECURE"))) c.Assert(config.ExternalClient, Equals, false) c.Assert(blobSigningKey, Equals, "abcdefg") } func (s *DoMainTestSuite) Test_doMain_WithNoConfig(c *C) { args := []string{"-prefix", "a"} - err := doMain(args) - c.Check(err, NotNil) - c.Assert(strings.Contains(err.Error(), "config file not specified"), Equals, true) + var stderr bytes.Buffer + code := doMain(args, &stderr) + c.Check(code, Equals, 1) + c.Check(stderr.String(), Matches, ".*config file not specified\n") } func (s *DoMainTestSuite) Test_doMain_WithNoSuchConfigFile(c *C) { args := []string{"-config", "no-such-file"} - err := doMain(args) - c.Check(err, NotNil) - c.Assert(strings.Contains(err.Error(), "no such file or directory"), Equals, true) + var stderr bytes.Buffer + code := doMain(args, &stderr) + c.Check(code, Equals, 1) + c.Check(stderr.String(), Matches, ".*no such file or directory\n") } func (s *DoMainTestSuite) Test_doMain_WithNoBlockHashFile(c *C) { @@ -300,8 +320,10 @@ func (s *DoMainTestSuite) Test_doMain_WithNoBlockHashFile(c *C) { defer arvadostest.StopKeep(2) args := []string{"-config", config} - err := doMain(args) - c.Assert(strings.Contains(err.Error(), "block-hash-file not specified"), Equals, true) + var stderr bytes.Buffer + code := doMain(args, &stderr) + c.Check(code, Equals, 1) + c.Check(stderr.String(), Matches, ".*block-hash-file not specified\n") } func (s *DoMainTestSuite) Test_doMain_WithNoSuchBlockHashFile(c *C) { @@ -312,8 +334,10 @@ func (s *DoMainTestSuite) Test_doMain_WithNoSuchBlockHashFile(c *C) { defer arvadostest.StopKeep(2) args := []string{"-config", config, "-block-hash-file", "no-such-file"} - err := doMain(args) - c.Assert(strings.Contains(err.Error(), "no such file or directory"), Equals, true) + var stderr bytes.Buffer + code := doMain(args, &stderr) + c.Check(code, Equals, 1) + c.Check(stderr.String(), Matches, ".*no such file or directory\n") } func (s *DoMainTestSuite) Test_doMain(c *C) { @@ -328,9 +352,10 @@ func (s *DoMainTestSuite) Test_doMain(c *C) { defer os.Remove(locatorFile) args := []string{"-config", config, "-block-hash-file", locatorFile, "-v"} - err := doMain(args) - c.Check(err, NotNil) - c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix.") + var stderr bytes.Buffer + code := doMain(args, &stderr) + c.Check(code, Equals, 1) + c.Assert(stderr.String(), Matches, "Block verification failed for 2 out of 2 blocks with matching prefix\n") checkErrorLog(c, []string{TestHash, TestHash2}, "Error verifying block", "Block not found") c.Assert(strings.Contains(logBuffer.String(), "Verifying block 1 of 2"), Equals, true) }