X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/bbb0760e909b3ceca850b3aa319011fba2c98ed0..3e7ab60f94a6076383bf4aec41b6271310ab8823:/tools/keep-block-check/keep-block-check_test.go diff --git a/tools/keep-block-check/keep-block-check_test.go b/tools/keep-block-check/keep-block-check_test.go index 2ea1972a57..9f409e6af0 100644 --- a/tools/keep-block-check/keep-block-check_test.go +++ b/tools/keep-block-check/keep-block-check_test.go @@ -1,16 +1,24 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: AGPL-3.0 + package main import ( + "bytes" "fmt" + "io" "io/ioutil" "log" "os" "regexp" "strings" "testing" + "time" - "git.curoverse.com/arvados.git/sdk/go/arvadostest" - "git.curoverse.com/arvados.git/sdk/go/keepclient" + "git.arvados.org/arvados.git/sdk/go/arvadosclient" + "git.arvados.org/arvados.git/sdk/go/arvadostest" + "git.arvados.org/arvados.git/sdk/go/keepclient" . "gopkg.in/check.v1" ) @@ -27,6 +35,14 @@ var _ = Suite(&DoMainTestSuite{}) type ServerRequiredSuite struct{} type DoMainTestSuite struct{} +var kc *keepclient.KeepClient +var logBuffer bytes.Buffer + +var TestHash = "aaaa09c290d0fb1ca068ffaddf22cbd0" +var TestHash2 = "aaaac516f788aec4f30932ffb6395c39" + +var blobSignatureTTL = time.Duration(2*7*24) * time.Hour + func (s *ServerRequiredSuite) SetUpSuite(c *C) { arvadostest.StartAPI() } @@ -37,75 +53,55 @@ func (s *ServerRequiredSuite) TearDownSuite(c *C) { } func (s *ServerRequiredSuite) SetUpTest(c *C) { - blobSigningKey = "" - keepServicesJSON = "" - - tempfile, err := ioutil.TempFile(os.TempDir(), "temp-log-file") - c.Check(err, IsNil) - log.SetOutput(tempfile) - tempLogFileName = tempfile.Name() + logOutput := io.MultiWriter(&logBuffer) + log.SetOutput(logOutput) } func (s *ServerRequiredSuite) TearDownTest(c *C) { arvadostest.StopKeep(2) - os.Remove(tempLogFileName) + log.SetOutput(os.Stdout) + log.Printf("%v", logBuffer.String()) } -var tempLogFileName = "" -var initialArgs []string -var kc *keepclient.KeepClient -var keepServicesJSON, blobSigningKey string - func (s *DoMainTestSuite) SetUpSuite(c *C) { - initialArgs = os.Args } func (s *DoMainTestSuite) SetUpTest(c *C) { - blobSigningKey = "" - keepServicesJSON = "" - - args := []string{"keep-block-check"} - os.Args = args - - tempfile, err := ioutil.TempFile(os.TempDir(), "temp-log-file") - c.Check(err, IsNil) - log.SetOutput(tempfile) - tempLogFileName = tempfile.Name() + logOutput := io.MultiWriter(&logBuffer) + log.SetOutput(logOutput) + keepclient.RefreshServiceDiscovery() } func (s *DoMainTestSuite) TearDownTest(c *C) { - os.Remove(tempLogFileName) - os.Args = initialArgs + log.SetOutput(os.Stdout) + log.Printf("%v", logBuffer.String()) } -var testKeepServicesJSON = "{ \"kind\":\"arvados#keepServiceList\", \"etag\":\"\", \"self_link\":\"\", \"offset\":null, \"limit\":null, \"items\":[ { \"href\":\"/keep_services/zzzzz-bi6l4-123456789012340\", \"kind\":\"arvados#keepService\", \"etag\":\"641234567890enhj7hzx432e5\", \"uuid\":\"zzzzz-bi6l4-123456789012340\", \"owner_uuid\":\"zzzzz-tpzed-123456789012345\", \"service_host\":\"keep0.zzzzz.arvadosapi.com\", \"service_port\":25107, \"service_ssl_flag\":false, \"service_type\":\"disk\", \"read_only\":false }, { \"href\":\"/keep_services/zzzzz-bi6l4-123456789012341\", \"kind\":\"arvados#keepService\", \"etag\":\"641234567890enhj7hzx432e5\", \"uuid\":\"zzzzz-bi6l4-123456789012341\", \"owner_uuid\":\"zzzzz-tpzed-123456789012345\", \"service_host\":\"keep0.zzzzz.arvadosapi.com\", \"service_port\":25108, \"service_ssl_flag\":false, \"service_type\":\"disk\", \"read_only\":false } ], \"items_available\":2 }" - -var TestHash = "aaaa09c290d0fb1ca068ffaddf22cbd0" -var TestHash2 = "aaaac516f788aec4f30932ffb6395c39" +func setupKeepBlockCheck(c *C, enforcePermissions bool, keepServicesJSON string) { + setupKeepBlockCheckWithTTL(c, enforcePermissions, keepServicesJSON, blobSignatureTTL) +} -func setupKeepBlockCheck(c *C, enforcePermissions bool) { +func setupKeepBlockCheckWithTTL(c *C, enforcePermissions bool, keepServicesJSON string, ttl time.Duration) { var config apiConfig config.APIHost = os.Getenv("ARVADOS_API_HOST") config.APIToken = arvadostest.DataManagerToken - config.APIHostInsecure = matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE")) - if enforcePermissions { - blobSigningKey = "zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc" - } + config.APIHostInsecure = arvadosclient.StringBool(os.Getenv("ARVADOS_API_HOST_INSECURE")) // Start Keep servers arvadostest.StartKeep(2, enforcePermissions) // setup keepclients var err error - kc, err = setupKeepClient(config, keepServicesJSON) + kc, ttl, err = setupKeepClient(config, keepServicesJSON, ttl) + c.Assert(ttl, Equals, blobSignatureTTL) c.Check(err, IsNil) + + keepclient.RefreshServiceDiscovery() } // Setup test data -var allLocators []string - -func setupTestData(c *C) { - allLocators = []string{} +func setupTestData(c *C) []string { + allLocators := []string{} // Put a few blocks for i := 0; i < 5; i++ { @@ -113,6 +109,8 @@ func setupTestData(c *C) { c.Check(err, IsNil) allLocators = append(allLocators, strings.Split(hash, "+A")[0]) } + + return allLocators } func setupConfigFile(c *C, fileName string) string { @@ -126,6 +124,7 @@ func setupConfigFile(c *C, fileName string) string { fileContent += "\n" fileContent += "ARVADOS_API_HOST_INSECURE=" + os.Getenv("ARVADOS_API_HOST_INSECURE") + "\n" fileContent += " ARVADOS_EXTERNAL_CLIENT = false \n" + fileContent += " NotANameValuePairAndShouldGetIgnored \n" fileContent += "ARVADOS_BLOB_SIGNING_KEY=abcdefg\n" _, err = file.Write([]byte(fileContent)) @@ -152,91 +151,129 @@ func setupBlockHashFile(c *C, name string, blocks []string) string { } func checkErrorLog(c *C, blocks []string, prefix, suffix string) { - buf, _ := ioutil.ReadFile(tempLogFileName) - if len(blocks) == 0 { - expected := prefix + `.*` + suffix - match, _ := regexp.MatchString(expected, string(buf)) - c.Assert(match, Equals, false) - return - } for _, hash := range blocks { - expected := prefix + `.*` + hash + `.*` + suffix - match, _ := regexp.MatchString(expected, string(buf)) - c.Assert(match, Equals, true) + expected := `(?ms).*` + prefix + `.*` + hash + `.*` + suffix + `.*` + c.Check(logBuffer.String(), Matches, expected) } } +func checkNoErrorsLogged(c *C, prefix, suffix string) { + expected := prefix + `.*` + suffix + match, _ := regexp.MatchString(expected, logBuffer.String()) + c.Assert(match, Equals, false) +} + func (s *ServerRequiredSuite) TestBlockCheck(c *C) { - setupKeepBlockCheck(c, false) - setupTestData(c) - err := performKeepBlockCheck(kc, blobSigningKey, "", allLocators) + setupKeepBlockCheck(c, false, "") + allLocators := setupTestData(c) + err := performKeepBlockCheck(kc, blobSignatureTTL, "", allLocators, true) c.Check(err, IsNil) - checkErrorLog(c, []string{}, "head", "Block not found") // no errors + checkNoErrorsLogged(c, "Error verifying block", "Block not found") } func (s *ServerRequiredSuite) TestBlockCheckWithBlobSigning(c *C) { - setupKeepBlockCheck(c, true) - setupTestData(c) - err := performKeepBlockCheck(kc, blobSigningKey, "", allLocators) + setupKeepBlockCheck(c, true, "") + allLocators := setupTestData(c) + err := performKeepBlockCheck(kc, blobSignatureTTL, arvadostest.BlobSigningKey, allLocators, true) c.Check(err, IsNil) - checkErrorLog(c, []string{}, "head", "Block not found") // no errors + checkNoErrorsLogged(c, "Error verifying block", "Block not found") +} + +func (s *ServerRequiredSuite) TestBlockCheckWithBlobSigningAndTTLFromDiscovery(c *C) { + setupKeepBlockCheckWithTTL(c, true, "", 0) + allLocators := setupTestData(c) + err := performKeepBlockCheck(kc, blobSignatureTTL, arvadostest.BlobSigningKey, allLocators, true) + c.Check(err, IsNil) + checkNoErrorsLogged(c, "Error verifying block", "Block not found") } func (s *ServerRequiredSuite) TestBlockCheck_NoSuchBlock(c *C) { - setupKeepBlockCheck(c, false) - setupTestData(c) + setupKeepBlockCheck(c, false, "") + allLocators := setupTestData(c) allLocators = append(allLocators, TestHash) allLocators = append(allLocators, TestHash2) - err := performKeepBlockCheck(kc, blobSigningKey, "", allLocators) + err := performKeepBlockCheck(kc, blobSignatureTTL, "", allLocators, true) c.Check(err, NotNil) - c.Assert(err.Error(), Equals, "Head information not found for 2 out of 7 blocks with matching prefix.") - checkErrorLog(c, []string{TestHash, TestHash2}, "head", "Block not found") + c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 7 blocks with matching prefix") + checkErrorLog(c, []string{TestHash, TestHash2}, "Error verifying block", "Block not found") } func (s *ServerRequiredSuite) TestBlockCheck_NoSuchBlock_WithMatchingPrefix(c *C) { - setupKeepBlockCheck(c, false) - setupTestData(c) + setupKeepBlockCheck(c, false, "") + allLocators := setupTestData(c) allLocators = append(allLocators, TestHash) allLocators = append(allLocators, TestHash2) - err := performKeepBlockCheck(kc, blobSigningKey, "aaa", allLocators) + locatorFile := setupBlockHashFile(c, "block-hash", allLocators) + defer os.Remove(locatorFile) + locators, err := getBlockLocators(locatorFile, "aaa") + c.Check(err, IsNil) + err = performKeepBlockCheck(kc, blobSignatureTTL, "", locators, true) c.Check(err, NotNil) - // Of the 7 blocks given, only two match the prefix and hence only those are checked - c.Assert(err.Error(), Equals, "Head information not found for 2 out of 2 blocks with matching prefix.") - checkErrorLog(c, []string{TestHash, TestHash2}, "head", "Block not found") + // Of the 7 blocks in allLocators, only two match the prefix and hence only those are checked + c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix") + checkErrorLog(c, []string{TestHash, TestHash2}, "Error verifying block", "Block not found") } func (s *ServerRequiredSuite) TestBlockCheck_NoSuchBlock_WithPrefixMismatch(c *C) { - setupKeepBlockCheck(c, false) - setupTestData(c) + setupKeepBlockCheck(c, false, "") + allLocators := setupTestData(c) allLocators = append(allLocators, TestHash) allLocators = append(allLocators, TestHash2) - err := performKeepBlockCheck(kc, blobSigningKey, "999", allLocators) + locatorFile := setupBlockHashFile(c, "block-hash", allLocators) + defer os.Remove(locatorFile) + locators, err := getBlockLocators(locatorFile, "999") c.Check(err, IsNil) - checkErrorLog(c, []string{}, "head", "Block not found") // no errors + err = performKeepBlockCheck(kc, blobSignatureTTL, "", locators, true) + c.Check(err, IsNil) // there were no matching locators in file and hence nothing was checked +} + +func (s *ServerRequiredSuite) TestBlockCheck_BadSignature(c *C) { + setupKeepBlockCheck(c, true, "") + setupTestData(c) + err := performKeepBlockCheck(kc, blobSignatureTTL, "badblobsigningkey", []string{TestHash, TestHash2}, false) + c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix") + checkErrorLog(c, []string{TestHash, TestHash2}, "Error verifying block", "HTTP 403") + // verbose logging not requested + c.Assert(strings.Contains(logBuffer.String(), "Verifying block 1 of 2"), Equals, false) } +var testKeepServicesJSON = `{ + "kind":"arvados#keepServiceList", + "etag":"", + "self_link":"", + "offset":null, "limit":null, + "items":[ + {"href":"/keep_services/zzzzz-bi6l4-123456789012340", + "kind":"arvados#keepService", + "uuid":"zzzzz-bi6l4-123456789012340", + "service_host":"keep0.zzzzz.arvadosapi.com", + "service_port":25107, + "service_ssl_flag":false, + "service_type":"disk", + "read_only":false }, + {"href":"/keep_services/zzzzz-bi6l4-123456789012341", + "kind":"arvados#keepService", + "uuid":"zzzzz-bi6l4-123456789012341", + "service_host":"keep0.zzzzz.arvadosapi.com", + "service_port":25108, + "service_ssl_flag":false, + "service_type":"disk", + "read_only":false } + ], + "items_available":2 }` + // Setup block-check using keepServicesJSON with fake keepservers. // Expect error during performKeepBlockCheck due to unreachable keepservers. func (s *ServerRequiredSuite) TestErrorDuringKeepBlockCheck_FakeKeepservers(c *C) { - keepServicesJSON = testKeepServicesJSON - setupKeepBlockCheck(c, false) - err := performKeepBlockCheck(kc, blobSigningKey, "", []string{TestHash, TestHash2}) - c.Assert(err.Error(), Equals, "Head information not found for 2 out of 2 blocks with matching prefix.") - checkErrorLog(c, []string{TestHash, TestHash2}, "head", "no such host") -} - -func (s *ServerRequiredSuite) TestBlockCheck_BadSignature(c *C) { - setupKeepBlockCheck(c, true) - setupTestData(c) - err := performKeepBlockCheck(kc, "badblobsigningkey", "", []string{TestHash, TestHash2}) - c.Assert(err.Error(), Equals, "Head information not found for 2 out of 2 blocks with matching prefix.") - checkErrorLog(c, []string{TestHash, TestHash2}, "head", "HTTP 403") + setupKeepBlockCheck(c, false, testKeepServicesJSON) + err := performKeepBlockCheck(kc, blobSignatureTTL, "", []string{TestHash, TestHash2}, true) + c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix") + checkErrorLog(c, []string{TestHash, TestHash2}, "Error verifying block", "") } // Test keep-block-check initialization with keepServicesJSON func (s *ServerRequiredSuite) TestKeepBlockCheck_InitializeWithKeepServicesJSON(c *C) { - keepServicesJSON = testKeepServicesJSON - setupKeepBlockCheck(c, false) + setupKeepBlockCheck(c, false, testKeepServicesJSON) found := 0 for k := range kc.LocalRoots() { if k == "zzzzz-bi6l4-123456789012340" || k == "zzzzz-bi6l4-123456789012341" { @@ -258,23 +295,21 @@ func (s *ServerRequiredSuite) TestLoadConfig(c *C) { c.Assert(config.APIHost, Equals, os.Getenv("ARVADOS_API_HOST")) c.Assert(config.APIToken, Equals, arvadostest.DataManagerToken) - c.Assert(config.APIHostInsecure, Equals, matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))) + c.Assert(config.APIHostInsecure, Equals, arvadosclient.StringBool(os.Getenv("ARVADOS_API_HOST_INSECURE"))) c.Assert(config.ExternalClient, Equals, false) c.Assert(blobSigningKey, Equals, "abcdefg") } func (s *DoMainTestSuite) Test_doMain_WithNoConfig(c *C) { args := []string{"-prefix", "a"} - os.Args = append(os.Args, args...) - err := doMain() + err := doMain(args) c.Check(err, NotNil) c.Assert(strings.Contains(err.Error(), "config file not specified"), Equals, true) } func (s *DoMainTestSuite) Test_doMain_WithNoSuchConfigFile(c *C) { args := []string{"-config", "no-such-file"} - os.Args = append(os.Args, args...) - err := doMain() + err := doMain(args) c.Check(err, NotNil) c.Assert(strings.Contains(err.Error(), "no such file or directory"), Equals, true) } @@ -283,14 +318,12 @@ func (s *DoMainTestSuite) Test_doMain_WithNoBlockHashFile(c *C) { config := setupConfigFile(c, "config") defer os.Remove(config) - args := []string{"-config", config} - os.Args = append(os.Args, args...) - // Start keepservers. arvadostest.StartKeep(2, false) defer arvadostest.StopKeep(2) - err := doMain() + args := []string{"-config", config} + err := doMain(args) c.Assert(strings.Contains(err.Error(), "block-hash-file not specified"), Equals, true) } @@ -298,14 +331,11 @@ func (s *DoMainTestSuite) Test_doMain_WithNoSuchBlockHashFile(c *C) { config := setupConfigFile(c, "config") defer os.Remove(config) - args := []string{"-config", config, "-block-hash-file", "no-such-file"} - os.Args = append(os.Args, args...) - - // Start keepservers. arvadostest.StartKeep(2, false) defer arvadostest.StopKeep(2) - err := doMain() + args := []string{"-config", config, "-block-hash-file", "no-such-file"} + err := doMain(args) c.Assert(strings.Contains(err.Error(), "no such file or directory"), Equals, true) } @@ -320,11 +350,10 @@ func (s *DoMainTestSuite) Test_doMain(c *C) { locatorFile := setupBlockHashFile(c, "block-hash", []string{TestHash, TestHash2}) defer os.Remove(locatorFile) - args := []string{"-config", config, "-block-hash-file", locatorFile} - os.Args = append(os.Args, args...) - - err := doMain() + args := []string{"-config", config, "-block-hash-file", locatorFile, "-v"} + err := doMain(args) c.Check(err, NotNil) - c.Assert(err.Error(), Equals, "Head information not found for 2 out of 2 blocks with matching prefix.") - checkErrorLog(c, []string{TestHash, TestHash2}, "head", "Block not found") + c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix") + checkErrorLog(c, []string{TestHash, TestHash2}, "Error verifying block", "Block not found") + c.Assert(strings.Contains(logBuffer.String(), "Verifying block 1 of 2"), Equals, true) }