From 65efe2535e49058ccf8250f93e958ea653f55cd8 Mon Sep 17 00:00:00 2001 From: radhika Date: Mon, 11 Apr 2016 23:36:11 -0400 Subject: [PATCH] 8724: test updates --- sdk/go/arvadostest/fixtures.go | 2 + tools/keep-block-check/keep-block-check.go | 85 +++++---- .../keep-block-check/keep-block-check_test.go | 175 ++++++++++-------- tools/keep-rsync/keep-rsync_test.go | 2 +- 4 files changed, 150 insertions(+), 114 deletions(-) diff --git a/sdk/go/arvadostest/fixtures.go b/sdk/go/arvadostest/fixtures.go index 47b75b3845..bebef79074 100644 --- a/sdk/go/arvadostest/fixtures.go +++ b/sdk/go/arvadostest/fixtures.go @@ -33,3 +33,5 @@ var ( } MD5CollisionMD5 = "cee9a457e790cf20d4bdaa6d69f01e41" ) + +const BlobSigningKey = "zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc" diff --git a/tools/keep-block-check/keep-block-check.go b/tools/keep-block-check/keep-block-check.go index caed9d0469..646d417d88 100644 --- a/tools/keep-block-check/keep-block-check.go +++ b/tools/keep-block-check/keep-block-check.go @@ -17,13 +17,13 @@ import ( ) func main() { - err := doMain() + err := doMain(os.Args[1:]) if err != nil { log.Fatalf("%v", err) } } -func doMain() error { +func doMain(args []string) error { flags := flag.NewFlagSet("keep-block-check", flag.ExitOnError) configFile := flags.String( @@ -48,8 +48,13 @@ func doMain() error { "", "Block hash prefix. When a prefix is specified, only hashes listed in the file with this prefix will be checked.") + verbose := flags.Bool( + "v", + false, + "Log progress of each block verification") + // Parse args; omit the first arg which is the command name - flags.Parse(os.Args[1:]) + flags.Parse(args) config, blobSigningKey, err := loadConfig(*configFile) if err != nil { @@ -57,7 +62,7 @@ func doMain() error { } // get list of block locators to be checked - blockLocators, err := getBlockLocators(*locatorFile) + blockLocators, err := getBlockLocators(*locatorFile, *prefix) if err != nil { return fmt.Errorf("Error reading block hashes to be checked from file: %s", err.Error()) } @@ -68,7 +73,7 @@ func doMain() error { return fmt.Errorf("Error configuring keepclient: %s", err.Error()) } - return performKeepBlockCheck(kc, blobSigningKey, *prefix, blockLocators) + return performKeepBlockCheck(kc, blobSigningKey, blockLocators, *verbose) } type apiConfig struct { @@ -81,7 +86,7 @@ type apiConfig struct { // Load config from given file func loadConfig(configFile string) (config apiConfig, blobSigningKey string, err error) { if configFile == "" { - err = errors.New("API config file not specified") + err = errors.New("Client config file not specified") return } @@ -110,20 +115,22 @@ func readConfigFromFile(filename string) (config apiConfig, blobSigningKey strin } kv := strings.SplitN(line, "=", 2) - key := strings.TrimSpace(kv[0]) - value := strings.TrimSpace(kv[1]) - - switch key { - case "ARVADOS_API_TOKEN": - config.APIToken = value - case "ARVADOS_API_HOST": - config.APIHost = value - case "ARVADOS_API_HOST_INSECURE": - config.APIHostInsecure = matchTrue.MatchString(value) - case "ARVADOS_EXTERNAL_CLIENT": - config.ExternalClient = matchTrue.MatchString(value) - case "ARVADOS_BLOB_SIGNING_KEY": - blobSigningKey = value + if len(kv) == 2 { + key := strings.TrimSpace(kv[0]) + value := strings.TrimSpace(kv[1]) + + switch key { + case "ARVADOS_API_TOKEN": + config.APIToken = value + case "ARVADOS_API_HOST": + config.APIHost = value + case "ARVADOS_API_HOST_INSECURE": + config.APIHostInsecure = matchTrue.MatchString(value) + case "ARVADOS_EXTERNAL_CLIENT": + config.ExternalClient = matchTrue.MatchString(value) + case "ARVADOS_BLOB_SIGNING_KEY": + blobSigningKey = value + } } } @@ -159,45 +166,40 @@ func setupKeepClient(config apiConfig, keepServicesJSON string) (kc *keepclient. } // Get list of unique block locators from the given file -func getBlockLocators(locatorFile string) (locators []string, err error) { +func getBlockLocators(locatorFile, prefix string) (locators []string, err error) { if locatorFile == "" { err = errors.New("block-hash-file not specified") return } content, err := ioutil.ReadFile(locatorFile) - if err != nil { return } - locatorMap := make(map[string]string) - lines := strings.Split(string(content), "\n") - for _, line := range lines { - if line == "" { + locatorMap := make(map[string]bool) + for _, line := range strings.Split(string(content), "\n") { + line = strings.TrimSpace(line) + if line == "" || !strings.HasPrefix(line, prefix) || locatorMap[line] { continue } - trimmedLine := strings.TrimSpace(line) - locatorMap[trimmedLine] = trimmedLine - } - - for _, locator := range locatorMap { - locators = append(locators, locator) + locators = append(locators, line) + locatorMap[line] = true } return } // Get block headers from keep. Log any errors. -func performKeepBlockCheck(kc *keepclient.KeepClient, blobSigningKey, prefix string, blockLocators []string) error { - totalBlocks := 0 +func performKeepBlockCheck(kc *keepclient.KeepClient, blobSigningKey string, blockLocators []string, verbose bool) error { + totalBlocks := len(blockLocators) notFoundBlocks := 0 + current := 0 for _, locator := range blockLocators { - if !strings.HasPrefix(locator, prefix) { - continue + current++ + if verbose { + log.Printf("Checking block %d of %d: %v", current, totalBlocks, locator) } - - totalBlocks++ getLocator := locator if blobSigningKey != "" { expiresAt := time.Now().AddDate(0, 0, 1) @@ -207,11 +209,14 @@ func performKeepBlockCheck(kc *keepclient.KeepClient, blobSigningKey, prefix str _, _, err := kc.Ask(getLocator) if err != nil { notFoundBlocks++ - log.Printf("Error getting head info for block: %v %v", locator, err) + log.Printf("Error verifying block %v: %v", locator, err) } } + + log.Printf("Verify block totals: %d attempts, %d successes, %d errors", totalBlocks, totalBlocks-notFoundBlocks, notFoundBlocks) + if notFoundBlocks > 0 { - return fmt.Errorf("Head information not found for %d out of %d blocks with matching prefix.", notFoundBlocks, totalBlocks) + return fmt.Errorf("Block verification failed for %d out of %d blocks with matching prefix.", notFoundBlocks, totalBlocks) } return nil diff --git a/tools/keep-block-check/keep-block-check_test.go b/tools/keep-block-check/keep-block-check_test.go index 2ea1972a57..5bd606266c 100644 --- a/tools/keep-block-check/keep-block-check_test.go +++ b/tools/keep-block-check/keep-block-check_test.go @@ -1,7 +1,9 @@ package main import ( + "bytes" "fmt" + "io" "io/ioutil" "log" "os" @@ -27,6 +29,12 @@ var _ = Suite(&DoMainTestSuite{}) type ServerRequiredSuite struct{} type DoMainTestSuite struct{} +var kc *keepclient.KeepClient +var keepServicesJSON, blobSigningKey string +var TestHash = "aaaa09c290d0fb1ca068ffaddf22cbd0" +var TestHash2 = "aaaac516f788aec4f30932ffb6395c39" +var allLocators []string + func (s *ServerRequiredSuite) SetUpSuite(c *C) { arvadostest.StartAPI() } @@ -36,60 +44,48 @@ func (s *ServerRequiredSuite) TearDownSuite(c *C) { arvadostest.ResetEnv() } +var logBuffer bytes.Buffer + func (s *ServerRequiredSuite) SetUpTest(c *C) { blobSigningKey = "" keepServicesJSON = "" - tempfile, err := ioutil.TempFile(os.TempDir(), "temp-log-file") - c.Check(err, IsNil) - log.SetOutput(tempfile) - tempLogFileName = tempfile.Name() + logOutput := io.MultiWriter(&logBuffer) + log.SetOutput(logOutput) } func (s *ServerRequiredSuite) TearDownTest(c *C) { arvadostest.StopKeep(2) - os.Remove(tempLogFileName) + log.SetOutput(os.Stdout) + log.Printf("%v", logBuffer.String()) } -var tempLogFileName = "" -var initialArgs []string -var kc *keepclient.KeepClient -var keepServicesJSON, blobSigningKey string - func (s *DoMainTestSuite) SetUpSuite(c *C) { - initialArgs = os.Args } +var testArgs = []string{} + func (s *DoMainTestSuite) SetUpTest(c *C) { blobSigningKey = "" keepServicesJSON = "" - args := []string{"keep-block-check"} - os.Args = args - - tempfile, err := ioutil.TempFile(os.TempDir(), "temp-log-file") - c.Check(err, IsNil) - log.SetOutput(tempfile) - tempLogFileName = tempfile.Name() + logOutput := io.MultiWriter(&logBuffer) + log.SetOutput(logOutput) } func (s *DoMainTestSuite) TearDownTest(c *C) { - os.Remove(tempLogFileName) - os.Args = initialArgs + log.SetOutput(os.Stdout) + log.Printf("%v", logBuffer.String()) + testArgs = []string{} } -var testKeepServicesJSON = "{ \"kind\":\"arvados#keepServiceList\", \"etag\":\"\", \"self_link\":\"\", \"offset\":null, \"limit\":null, \"items\":[ { \"href\":\"/keep_services/zzzzz-bi6l4-123456789012340\", \"kind\":\"arvados#keepService\", \"etag\":\"641234567890enhj7hzx432e5\", \"uuid\":\"zzzzz-bi6l4-123456789012340\", \"owner_uuid\":\"zzzzz-tpzed-123456789012345\", \"service_host\":\"keep0.zzzzz.arvadosapi.com\", \"service_port\":25107, \"service_ssl_flag\":false, \"service_type\":\"disk\", \"read_only\":false }, { \"href\":\"/keep_services/zzzzz-bi6l4-123456789012341\", \"kind\":\"arvados#keepService\", \"etag\":\"641234567890enhj7hzx432e5\", \"uuid\":\"zzzzz-bi6l4-123456789012341\", \"owner_uuid\":\"zzzzz-tpzed-123456789012345\", \"service_host\":\"keep0.zzzzz.arvadosapi.com\", \"service_port\":25108, \"service_ssl_flag\":false, \"service_type\":\"disk\", \"read_only\":false } ], \"items_available\":2 }" - -var TestHash = "aaaa09c290d0fb1ca068ffaddf22cbd0" -var TestHash2 = "aaaac516f788aec4f30932ffb6395c39" - func setupKeepBlockCheck(c *C, enforcePermissions bool) { var config apiConfig config.APIHost = os.Getenv("ARVADOS_API_HOST") config.APIToken = arvadostest.DataManagerToken config.APIHostInsecure = matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE")) if enforcePermissions { - blobSigningKey = "zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc" + blobSigningKey = arvadostest.BlobSigningKey } // Start Keep servers @@ -102,8 +98,6 @@ func setupKeepBlockCheck(c *C, enforcePermissions bool) { } // Setup test data -var allLocators []string - func setupTestData(c *C) { allLocators = []string{} @@ -126,6 +120,7 @@ func setupConfigFile(c *C, fileName string) string { fileContent += "\n" fileContent += "ARVADOS_API_HOST_INSECURE=" + os.Getenv("ARVADOS_API_HOST_INSECURE") + "\n" fileContent += " ARVADOS_EXTERNAL_CLIENT = false \n" + fileContent += " NotANameValuePairAndShouldGetIgnored \n" fileContent += "ARVADOS_BLOB_SIGNING_KEY=abcdefg\n" _, err = file.Write([]byte(fileContent)) @@ -152,34 +147,33 @@ func setupBlockHashFile(c *C, name string, blocks []string) string { } func checkErrorLog(c *C, blocks []string, prefix, suffix string) { - buf, _ := ioutil.ReadFile(tempLogFileName) - if len(blocks) == 0 { - expected := prefix + `.*` + suffix - match, _ := regexp.MatchString(expected, string(buf)) - c.Assert(match, Equals, false) - return - } for _, hash := range blocks { expected := prefix + `.*` + hash + `.*` + suffix - match, _ := regexp.MatchString(expected, string(buf)) + match, _ := regexp.MatchString(expected, logBuffer.String()) c.Assert(match, Equals, true) } } +func checkNoErrorsLogged(c *C, prefix, suffix string) { + expected := prefix + `.*` + suffix + match, _ := regexp.MatchString(expected, logBuffer.String()) + c.Assert(match, Equals, false) +} + func (s *ServerRequiredSuite) TestBlockCheck(c *C) { setupKeepBlockCheck(c, false) setupTestData(c) - err := performKeepBlockCheck(kc, blobSigningKey, "", allLocators) + err := performKeepBlockCheck(kc, blobSigningKey, allLocators, true) c.Check(err, IsNil) - checkErrorLog(c, []string{}, "head", "Block not found") // no errors + checkNoErrorsLogged(c, "Error verifying block", "Block not found") } func (s *ServerRequiredSuite) TestBlockCheckWithBlobSigning(c *C) { setupKeepBlockCheck(c, true) setupTestData(c) - err := performKeepBlockCheck(kc, blobSigningKey, "", allLocators) + err := performKeepBlockCheck(kc, blobSigningKey, allLocators, true) c.Check(err, IsNil) - checkErrorLog(c, []string{}, "head", "Block not found") // no errors + checkNoErrorsLogged(c, "Error verifying block", "Block not found") } func (s *ServerRequiredSuite) TestBlockCheck_NoSuchBlock(c *C) { @@ -187,10 +181,10 @@ func (s *ServerRequiredSuite) TestBlockCheck_NoSuchBlock(c *C) { setupTestData(c) allLocators = append(allLocators, TestHash) allLocators = append(allLocators, TestHash2) - err := performKeepBlockCheck(kc, blobSigningKey, "", allLocators) + err := performKeepBlockCheck(kc, blobSigningKey, allLocators, true) c.Check(err, NotNil) - c.Assert(err.Error(), Equals, "Head information not found for 2 out of 7 blocks with matching prefix.") - checkErrorLog(c, []string{TestHash, TestHash2}, "head", "Block not found") + c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 7 blocks with matching prefix.") + checkErrorLog(c, []string{TestHash, TestHash2}, "Error verifying block", "Block not found") } func (s *ServerRequiredSuite) TestBlockCheck_NoSuchBlock_WithMatchingPrefix(c *C) { @@ -198,11 +192,15 @@ func (s *ServerRequiredSuite) TestBlockCheck_NoSuchBlock_WithMatchingPrefix(c *C setupTestData(c) allLocators = append(allLocators, TestHash) allLocators = append(allLocators, TestHash2) - err := performKeepBlockCheck(kc, blobSigningKey, "aaa", allLocators) + locatorFile := setupBlockHashFile(c, "block-hash", allLocators) + defer os.Remove(locatorFile) + locators, err := getBlockLocators(locatorFile, "aaa") + c.Check(err, IsNil) + err = performKeepBlockCheck(kc, blobSigningKey, locators, true) c.Check(err, NotNil) - // Of the 7 blocks given, only two match the prefix and hence only those are checked - c.Assert(err.Error(), Equals, "Head information not found for 2 out of 2 blocks with matching prefix.") - checkErrorLog(c, []string{TestHash, TestHash2}, "head", "Block not found") + // Of the 7 blocks in allLocators, only two match the prefix and hence only those are checked + c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix.") + checkErrorLog(c, []string{TestHash, TestHash2}, "Error verifying block", "Block not found") } func (s *ServerRequiredSuite) TestBlockCheck_NoSuchBlock_WithPrefixMismatch(c *C) { @@ -210,27 +208,57 @@ func (s *ServerRequiredSuite) TestBlockCheck_NoSuchBlock_WithPrefixMismatch(c *C setupTestData(c) allLocators = append(allLocators, TestHash) allLocators = append(allLocators, TestHash2) - err := performKeepBlockCheck(kc, blobSigningKey, "999", allLocators) + locatorFile := setupBlockHashFile(c, "block-hash", allLocators) + defer os.Remove(locatorFile) + locators, err := getBlockLocators(locatorFile, "999") c.Check(err, IsNil) - checkErrorLog(c, []string{}, "head", "Block not found") // no errors + err = performKeepBlockCheck(kc, blobSigningKey, locators, true) + c.Check(err, IsNil) // there were no matching locators and hence no errors +} + +func (s *ServerRequiredSuite) TestBlockCheck_BadSignature(c *C) { + setupKeepBlockCheck(c, true) + setupTestData(c) + err := performKeepBlockCheck(kc, "badblobsigningkey", []string{TestHash, TestHash2}, false) + c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix.") + checkErrorLog(c, []string{TestHash, TestHash2}, "Error verifying block", "HTTP 403") + // verbose logging not requested + c.Assert(strings.Contains(logBuffer.String(), "Checking block 1 of 2"), Equals, false) } +var testKeepServicesJSON = `{ + "kind":"arvados#keepServiceList", + "etag":"", + "self_link":"", + "offset":null, "limit":null, + "items":[ + {"href":"/keep_services/zzzzz-bi6l4-123456789012340", + "kind":"arvados#keepService", + "uuid":"zzzzz-bi6l4-123456789012340", + "service_host":"keep0.zzzzz.arvadosapi.com", + "service_port":25107, + "service_ssl_flag":false, + "service_type":"disk", + "read_only":false }, + {"href":"/keep_services/zzzzz-bi6l4-123456789012341", + "kind":"arvados#keepService", + "uuid":"zzzzz-bi6l4-123456789012341", + "service_host":"keep0.zzzzz.arvadosapi.com", + "service_port":25108, + "service_ssl_flag":false, + "service_type":"disk", + "read_only":false } + ], + "items_available":2 }` + // Setup block-check using keepServicesJSON with fake keepservers. // Expect error during performKeepBlockCheck due to unreachable keepservers. func (s *ServerRequiredSuite) TestErrorDuringKeepBlockCheck_FakeKeepservers(c *C) { keepServicesJSON = testKeepServicesJSON setupKeepBlockCheck(c, false) - err := performKeepBlockCheck(kc, blobSigningKey, "", []string{TestHash, TestHash2}) - c.Assert(err.Error(), Equals, "Head information not found for 2 out of 2 blocks with matching prefix.") - checkErrorLog(c, []string{TestHash, TestHash2}, "head", "no such host") -} - -func (s *ServerRequiredSuite) TestBlockCheck_BadSignature(c *C) { - setupKeepBlockCheck(c, true) - setupTestData(c) - err := performKeepBlockCheck(kc, "badblobsigningkey", "", []string{TestHash, TestHash2}) - c.Assert(err.Error(), Equals, "Head information not found for 2 out of 2 blocks with matching prefix.") - checkErrorLog(c, []string{TestHash, TestHash2}, "head", "HTTP 403") + err := performKeepBlockCheck(kc, blobSigningKey, []string{TestHash, TestHash2}, true) + c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix.") + checkErrorLog(c, []string{TestHash, TestHash2}, "Error verifying block", "") } // Test keep-block-check initialization with keepServicesJSON @@ -265,16 +293,16 @@ func (s *ServerRequiredSuite) TestLoadConfig(c *C) { func (s *DoMainTestSuite) Test_doMain_WithNoConfig(c *C) { args := []string{"-prefix", "a"} - os.Args = append(os.Args, args...) - err := doMain() + testArgs = append(testArgs, args...) + err := doMain(testArgs) c.Check(err, NotNil) c.Assert(strings.Contains(err.Error(), "config file not specified"), Equals, true) } func (s *DoMainTestSuite) Test_doMain_WithNoSuchConfigFile(c *C) { args := []string{"-config", "no-such-file"} - os.Args = append(os.Args, args...) - err := doMain() + testArgs = append(testArgs, args...) + err := doMain(testArgs) c.Check(err, NotNil) c.Assert(strings.Contains(err.Error(), "no such file or directory"), Equals, true) } @@ -284,13 +312,13 @@ func (s *DoMainTestSuite) Test_doMain_WithNoBlockHashFile(c *C) { defer os.Remove(config) args := []string{"-config", config} - os.Args = append(os.Args, args...) + testArgs = append(testArgs, args...) // Start keepservers. arvadostest.StartKeep(2, false) defer arvadostest.StopKeep(2) - err := doMain() + err := doMain(testArgs) c.Assert(strings.Contains(err.Error(), "block-hash-file not specified"), Equals, true) } @@ -299,13 +327,13 @@ func (s *DoMainTestSuite) Test_doMain_WithNoSuchBlockHashFile(c *C) { defer os.Remove(config) args := []string{"-config", config, "-block-hash-file", "no-such-file"} - os.Args = append(os.Args, args...) + testArgs = append(testArgs, args...) // Start keepservers. arvadostest.StartKeep(2, false) defer arvadostest.StopKeep(2) - err := doMain() + err := doMain(testArgs) c.Assert(strings.Contains(err.Error(), "no such file or directory"), Equals, true) } @@ -320,11 +348,12 @@ func (s *DoMainTestSuite) Test_doMain(c *C) { locatorFile := setupBlockHashFile(c, "block-hash", []string{TestHash, TestHash2}) defer os.Remove(locatorFile) - args := []string{"-config", config, "-block-hash-file", locatorFile} - os.Args = append(os.Args, args...) + args := []string{"-config", config, "-block-hash-file", locatorFile, "-v"} + testArgs = append(testArgs, args...) - err := doMain() + err := doMain(testArgs) c.Check(err, NotNil) - c.Assert(err.Error(), Equals, "Head information not found for 2 out of 2 blocks with matching prefix.") - checkErrorLog(c, []string{TestHash, TestHash2}, "head", "Block not found") + c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix.") + checkErrorLog(c, []string{TestHash, TestHash2}, "Error verifying block", "Block not found") + c.Assert(strings.Contains(logBuffer.String(), "Checking block 1 of 2"), Equals, true) } diff --git a/tools/keep-rsync/keep-rsync_test.go b/tools/keep-rsync/keep-rsync_test.go index 94281fa8bc..afc27f513a 100644 --- a/tools/keep-rsync/keep-rsync_test.go +++ b/tools/keep-rsync/keep-rsync_test.go @@ -91,7 +91,7 @@ func setupRsync(c *C, enforcePermissions bool, replications int) { dstConfig.APIHostInsecure = matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE")) if enforcePermissions { - blobSigningKey = "zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc" + blobSigningKey = arvadostest.BlobSigningKey } // Start Keep servers -- 2.30.2