From f1e315aa701757ee8bf497805033b039b21515ba Mon Sep 17 00:00:00 2001 From: radhika Date: Wed, 6 Apr 2016 18:30:35 -0400 Subject: [PATCH] 8724: add keep-block-check script --- build/run-tests.sh | 2 + tools/keep-block-check/.gitignore | 1 + tools/keep-block-check/keep-block-check.go | 203 ++++++++++++ .../keep-block-check/keep-block-check_test.go | 309 ++++++++++++++++++ 4 files changed, 515 insertions(+) create mode 100644 tools/keep-block-check/.gitignore create mode 100644 tools/keep-block-check/keep-block-check.go create mode 100644 tools/keep-block-check/keep-block-check_test.go diff --git a/build/run-tests.sh b/build/run-tests.sh index 041c7c2f90..98c8ac55a8 100755 --- a/build/run-tests.sh +++ b/build/run-tests.sh @@ -87,6 +87,7 @@ sdk/go/crunchrunner sdk/cwl tools/crunchstat-summary tools/keep-rsync +tools/keep-block-check EOF @@ -709,6 +710,7 @@ gostuff=( services/crunch-dispatch-slurm services/crunch-run tools/keep-rsync + tools/keep-block-check ) for g in "${gostuff[@]}" do diff --git a/tools/keep-block-check/.gitignore b/tools/keep-block-check/.gitignore new file mode 100644 index 0000000000..97eb5da3bc --- /dev/null +++ b/tools/keep-block-check/.gitignore @@ -0,0 +1 @@ +keep-block-check diff --git a/tools/keep-block-check/keep-block-check.go b/tools/keep-block-check/keep-block-check.go new file mode 100644 index 0000000000..431703545a --- /dev/null +++ b/tools/keep-block-check/keep-block-check.go @@ -0,0 +1,203 @@ +package main + +import ( + "crypto/tls" + "errors" + "flag" + "fmt" + "git.curoverse.com/arvados.git/sdk/go/arvadosclient" + "git.curoverse.com/arvados.git/sdk/go/keepclient" + "io/ioutil" + "log" + "net/http" + "os" + "regexp" + "strings" + "time" +) + +func main() { + err := doMain() + if err != nil { + log.Fatalf("%v", err) + } +} + +func doMain() error { + flags := flag.NewFlagSet("keep-block-check", flag.ExitOnError) + + configFile := flags.String( + "config", + "", + "Configuration filename. May be either a pathname to a config file, or (for example) 'foo' as shorthand for $HOME/.config/arvados/foo.conf file. This file is expected to specify the values for ARVADOS_API_TOKEN, ARVADOS_API_HOST, ARVADOS_API_HOST_INSECURE, and ARVADOS_BLOB_SIGNING_KEY for the source.") + + keepServicesJSON := flags.String( + "keep-services-json", + "", + "An optional list of available keepservices. "+ + "If not provided, this list is obtained from api server configured in config-file.") + + locatorFile := flags.String( + "block-hash-file", + "", + "Filename containing the block hashes to be checked. This is required. "+ + "This file contains the block hashes one per line.") + + prefix := flags.String( + "prefix", + "", + "Block hash prefix. When a prefix is specified, only hashes listed in the file with this prefix will be checked.") + + // Parse args; omit the first arg which is the command name + flags.Parse(os.Args[1:]) + + config, blobSigningKey, err := loadConfig(*configFile) + if err != nil { + return fmt.Errorf("Error loading configuration from file: %s", err.Error()) + } + + // get list of block locators to be checked + blockLocators, err := getBlockLocators(*locatorFile) + if err != nil { + return fmt.Errorf("Error reading block hashes to be checked from file: %s", err.Error()) + } + + // setup keepclient + kc, err := setupKeepClient(config, *keepServicesJSON) + if err != nil { + return fmt.Errorf("Error configuring keepclient: %s", err.Error()) + } + + performKeepBlockCheck(kc, blobSigningKey, *prefix, blockLocators) + return nil +} + +type apiConfig struct { + APIToken string + APIHost string + APIHostInsecure bool + ExternalClient bool +} + +// Load config from given file +func loadConfig(configFile string) (config apiConfig, blobSigningKey string, err error) { + if configFile == "" { + err = errors.New("API config file not specified") + return + } + + config, blobSigningKey, err = readConfigFromFile(configFile) + return +} + +var matchTrue = regexp.MustCompile("^(?i:1|yes|true)$") + +// Read config from file +func readConfigFromFile(filename string) (config apiConfig, blobSigningKey string, err error) { + if !strings.Contains(filename, "/") { + filename = os.Getenv("HOME") + "/.config/arvados/" + filename + ".conf" + } + + content, err := ioutil.ReadFile(filename) + + if err != nil { + return + } + + lines := strings.Split(string(content), "\n") + for _, line := range lines { + if line == "" { + continue + } + + kv := strings.SplitN(line, "=", 2) + key := strings.TrimSpace(kv[0]) + value := strings.TrimSpace(kv[1]) + + switch key { + case "ARVADOS_API_TOKEN": + config.APIToken = value + case "ARVADOS_API_HOST": + config.APIHost = value + case "ARVADOS_API_HOST_INSECURE": + config.APIHostInsecure = matchTrue.MatchString(value) + case "ARVADOS_EXTERNAL_CLIENT": + config.ExternalClient = matchTrue.MatchString(value) + case "ARVADOS_BLOB_SIGNING_KEY": + blobSigningKey = value + } + } + + return +} + +// setup keepclient using the config provided +func setupKeepClient(config apiConfig, keepServicesJSON string) (kc *keepclient.KeepClient, err error) { + arv := arvadosclient.ArvadosClient{ + ApiToken: config.APIToken, + ApiServer: config.APIHost, + ApiInsecure: config.APIHostInsecure, + Client: &http.Client{Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: config.APIHostInsecure}}}, + External: config.ExternalClient, + } + + // if keepServicesJSON is provided, use it to load services; else, use DiscoverKeepServers + if keepServicesJSON == "" { + kc, err = keepclient.MakeKeepClient(&arv) + if err != nil { + return + } + } else { + kc = keepclient.New(&arv) + err = kc.LoadKeepServicesFromJSON(keepServicesJSON) + if err != nil { + return + } + } + + return +} + +// Get list of block locators from the given file +func getBlockLocators(locatorFile string) (locators []string, err error) { + if locatorFile == "" { + err = errors.New("block-hash-file not specified") + return + } + + content, err := ioutil.ReadFile(locatorFile) + + if err != nil { + return + } + + lines := strings.Split(string(content), "\n") + for _, line := range lines { + if line == "" { + continue + } + locators = append(locators, strings.TrimSpace(line)) + } + + return +} + +// Get block headers from keep. Log any errors. +func performKeepBlockCheck(kc *keepclient.KeepClient, blobSigningKey, prefix string, blockLocators []string) { + for _, locator := range blockLocators { + if !strings.HasPrefix(locator, prefix) { + continue + } + getLocator := locator + if blobSigningKey != "" { + expiresAt := time.Now().AddDate(0, 0, 1) + getLocator = keepclient.SignLocator(locator, kc.Arvados.ApiToken, expiresAt, []byte(blobSigningKey)) + } + + _, _, err := kc.Ask(getLocator) + if err != nil { + log.Printf("Error getting head info for block: %v %v", locator, err) + } + } +} diff --git a/tools/keep-block-check/keep-block-check_test.go b/tools/keep-block-check/keep-block-check_test.go new file mode 100644 index 0000000000..0c56e55d07 --- /dev/null +++ b/tools/keep-block-check/keep-block-check_test.go @@ -0,0 +1,309 @@ +package main + +import ( + "fmt" + "io/ioutil" + "log" + "os" + "regexp" + "strings" + "testing" + + "git.curoverse.com/arvados.git/sdk/go/arvadostest" + "git.curoverse.com/arvados.git/sdk/go/keepclient" + + . "gopkg.in/check.v1" +) + +// Gocheck boilerplate +func Test(t *testing.T) { + TestingT(t) +} + +// Gocheck boilerplate +var _ = Suite(&ServerRequiredSuite{}) +var _ = Suite(&DoMainTestSuite{}) + +type ServerRequiredSuite struct{} +type DoMainTestSuite struct{} + +func (s *ServerRequiredSuite) SetUpSuite(c *C) { + arvadostest.StartAPI() +} + +func (s *ServerRequiredSuite) TearDownSuite(c *C) { + arvadostest.StopAPI() + arvadostest.ResetEnv() +} + +func (s *ServerRequiredSuite) SetUpTest(c *C) { + blobSigningKey = "" + keepServicesJSON = "" + + tempfile, err := ioutil.TempFile(os.TempDir(), "temp-log-file") + c.Check(err, IsNil) + log.SetOutput(tempfile) + tempLogFileName = tempfile.Name() +} + +func (s *ServerRequiredSuite) TearDownTest(c *C) { + arvadostest.StopKeep(2) + os.Remove(tempLogFileName) +} + +var tempLogFileName = "" +var initialArgs []string +var kc *keepclient.KeepClient +var keepServicesJSON, blobSigningKey string + +func (s *DoMainTestSuite) SetUpSuite(c *C) { + initialArgs = os.Args +} + +func (s *DoMainTestSuite) SetUpTest(c *C) { + blobSigningKey = "" + keepServicesJSON = "" + + args := []string{"keep-block-check"} + os.Args = args + + tempfile, err := ioutil.TempFile(os.TempDir(), "temp-log-file") + c.Check(err, IsNil) + log.SetOutput(tempfile) + tempLogFileName = tempfile.Name() +} + +func (s *DoMainTestSuite) TearDownTest(c *C) { + os.Remove(tempLogFileName) + os.Args = initialArgs +} + +var testKeepServicesJSON = "{ \"kind\":\"arvados#keepServiceList\", \"etag\":\"\", \"self_link\":\"\", \"offset\":null, \"limit\":null, \"items\":[ { \"href\":\"/keep_services/zzzzz-bi6l4-123456789012340\", \"kind\":\"arvados#keepService\", \"etag\":\"641234567890enhj7hzx432e5\", \"uuid\":\"zzzzz-bi6l4-123456789012340\", \"owner_uuid\":\"zzzzz-tpzed-123456789012345\", \"service_host\":\"keep0.zzzzz.arvadosapi.com\", \"service_port\":25107, \"service_ssl_flag\":false, \"service_type\":\"disk\", \"read_only\":false }, { \"href\":\"/keep_services/zzzzz-bi6l4-123456789012341\", \"kind\":\"arvados#keepService\", \"etag\":\"641234567890enhj7hzx432e5\", \"uuid\":\"zzzzz-bi6l4-123456789012341\", \"owner_uuid\":\"zzzzz-tpzed-123456789012345\", \"service_host\":\"keep0.zzzzz.arvadosapi.com\", \"service_port\":25108, \"service_ssl_flag\":false, \"service_type\":\"disk\", \"read_only\":false } ], \"items_available\":2 }" + +var TestHash = "aaaa09c290d0fb1ca068ffaddf22cbd0" +var TestHash2 = "aaaac516f788aec4f30932ffb6395c39" + +func setupKeepBlockCheck(c *C, enforcePermissions bool) { + var config apiConfig + config.APIHost = os.Getenv("ARVADOS_API_HOST") + config.APIToken = arvadostest.DataManagerToken + config.APIHostInsecure = matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE")) + if enforcePermissions { + blobSigningKey = "zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc" + } + + // Start Keep servers + arvadostest.StartKeep(2, enforcePermissions) + + // setup keepclients + var err error + kc, err = setupKeepClient(config, keepServicesJSON) + c.Check(err, IsNil) +} + +// Setup test data +var allLocators []string + +func setupTestData(c *C) { + allLocators = []string{} + + // Put a few blocks + for i := 0; i < 5; i++ { + hash, _, err := kc.PutB([]byte(fmt.Sprintf("keep-block-check-test-data-%d", i))) + c.Check(err, IsNil) + allLocators = append(allLocators, strings.Split(hash, "+A")[0]) + } +} + +func setupConfigFile(c *C, fileName string) string { + // Setup a config file + file, err := ioutil.TempFile(os.TempDir(), fileName) + c.Check(err, IsNil) + + fileContent := "ARVADOS_API_HOST=" + os.Getenv("ARVADOS_API_HOST") + "\n" + fileContent += "ARVADOS_API_TOKEN=" + arvadostest.DataManagerToken + "\n" + fileContent += "ARVADOS_API_HOST_INSECURE=" + os.Getenv("ARVADOS_API_HOST_INSECURE") + "\n" + fileContent += "ARVADOS_EXTERNAL_CLIENT=false\n" + fileContent += "ARVADOS_BLOB_SIGNING_KEY=abcdefg" + + _, err = file.Write([]byte(fileContent)) + c.Check(err, IsNil) + + return file.Name() +} + +func setupBlockHashFile(c *C, name string, blocks []string) string { + // Setup a block hash file + file, err := ioutil.TempFile(os.TempDir(), name) + c.Check(err, IsNil) + + fileContent := "" + for _, hash := range blocks { + fileContent += fmt.Sprintf("%s\n", hash) + } + _, err = file.Write([]byte(fileContent)) + c.Check(err, IsNil) + + return file.Name() +} + +func checkErrorLog(c *C, blocks []string, msg string) { + buf, _ := ioutil.ReadFile(tempLogFileName) + if len(blocks) == 0 { + expected := `HEAD .*` + msg + match, _ := regexp.MatchString(expected, string(buf)) + c.Assert(match, Equals, false) + return + } + for _, hash := range blocks { + expected := hash + `.*` + msg + match, _ := regexp.MatchString(expected, string(buf)) + c.Assert(match, Equals, true) + } +} + +func (s *ServerRequiredSuite) TestBlockCheck(c *C) { + setupKeepBlockCheck(c, false) + setupTestData(c) + performKeepBlockCheck(kc, blobSigningKey, "", allLocators) + checkErrorLog(c, []string{}, "Block not found") // no errors +} + +func (s *ServerRequiredSuite) TestBlockCheckWithBlobSigning(c *C) { + setupKeepBlockCheck(c, true) + setupTestData(c) + performKeepBlockCheck(kc, blobSigningKey, "", allLocators) + checkErrorLog(c, []string{}, "Block not found") // no errors +} + +func (s *ServerRequiredSuite) TestBlockCheck_NoSuchBlock(c *C) { + setupKeepBlockCheck(c, false) + setupTestData(c) + performKeepBlockCheck(kc, blobSigningKey, "", []string{TestHash, TestHash2}) + checkErrorLog(c, []string{TestHash, TestHash2}, "Block not found") +} + +func (s *ServerRequiredSuite) TestBlockCheck_NoSuchBlock_WithMatchingPrefix(c *C) { + setupKeepBlockCheck(c, false) + setupTestData(c) + performKeepBlockCheck(kc, blobSigningKey, "aaa", []string{TestHash, TestHash2}) + checkErrorLog(c, []string{TestHash, TestHash2}, "Block not found") +} + +func (s *ServerRequiredSuite) TestBlockCheck_NoSuchBlock_WithPrefixMismatch(c *C) { + setupKeepBlockCheck(c, false) + setupTestData(c) + performKeepBlockCheck(kc, blobSigningKey, "999", []string{TestHash, TestHash2}) + checkErrorLog(c, []string{}, "Block not found") // no errors +} + +// Setup block-check using keepServicesJSON with fake keepservers. +// Expect error during performKeepBlockCheck due to unreachable keepservers. +func (s *ServerRequiredSuite) TestErrorDuringKeepBlockCheck_FakeKeepservers(c *C) { + keepServicesJSON = testKeepServicesJSON + setupKeepBlockCheck(c, false) + performKeepBlockCheck(kc, blobSigningKey, "", []string{TestHash, TestHash2}) + checkErrorLog(c, []string{TestHash, TestHash2}, "timeout") +} + +func (s *ServerRequiredSuite) TestBlockCheck_BadSignature(c *C) { + setupKeepBlockCheck(c, true) + setupTestData(c) + performKeepBlockCheck(kc, "badblobsigningkey", "", []string{TestHash, TestHash2}) + checkErrorLog(c, []string{TestHash, TestHash2}, "HTTP 403") +} + +// Test keep-block-check initialization with keepServicesJSON +func (s *ServerRequiredSuite) TestKeepBlockCheck_InitializeWithKeepServicesJSON(c *C) { + keepServicesJSON = testKeepServicesJSON + setupKeepBlockCheck(c, false) + found := 0 + for k := range kc.LocalRoots() { + if k == "zzzzz-bi6l4-123456789012340" || k == "zzzzz-bi6l4-123456789012341" { + found++ + } + } + c.Check(found, Equals, 2) +} + +// Test loadConfig func +func (s *ServerRequiredSuite) TestLoadConfig(c *C) { + // Setup config file + configFile := setupConfigFile(c, "config") + defer os.Remove(configFile) + + // load configuration from the file + config, blobSigningKey, err := loadConfig(configFile) + c.Check(err, IsNil) + + c.Assert(config.APIHost, Equals, os.Getenv("ARVADOS_API_HOST")) + c.Assert(config.APIToken, Equals, arvadostest.DataManagerToken) + c.Assert(config.APIHostInsecure, Equals, matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))) + c.Assert(config.ExternalClient, Equals, false) + c.Assert(blobSigningKey, Equals, "abcdefg") +} + +func (s *DoMainTestSuite) Test_doMain_WithNoConfig(c *C) { + args := []string{"-prefix", "a"} + os.Args = append(os.Args, args...) + err := doMain() + c.Check(err, NotNil) + c.Assert(strings.Contains(err.Error(), "config file not specified"), Equals, true) +} + +func (s *DoMainTestSuite) Test_doMain_WithNoSuchConfigFile(c *C) { + args := []string{"-config", "no-such-file"} + os.Args = append(os.Args, args...) + err := doMain() + c.Check(err, NotNil) + c.Assert(strings.Contains(err.Error(), "no such file or directory"), Equals, true) +} + +func (s *DoMainTestSuite) Test_doMain_WithNoBlockHashFile(c *C) { + config := setupConfigFile(c, "config") + defer os.Remove(config) + + args := []string{"-config", config} + os.Args = append(os.Args, args...) + + // Start keepservers. + arvadostest.StartKeep(2, false) + defer arvadostest.StopKeep(2) + + err := doMain() + c.Assert(strings.Contains(err.Error(), "block-hash-file not specified"), Equals, true) +} + +func (s *DoMainTestSuite) Test_doMain_WithNoSuchBlockHashFile(c *C) { + config := setupConfigFile(c, "config") + defer os.Remove(config) + + args := []string{"-config", config, "-block-hash-file", "no-such-file"} + os.Args = append(os.Args, args...) + + // Start keepservers. + arvadostest.StartKeep(2, false) + defer arvadostest.StopKeep(2) + + err := doMain() + c.Assert(strings.Contains(err.Error(), "no such file or directory"), Equals, true) +} + +func (s *DoMainTestSuite) Test_doMain(c *C) { + config := setupConfigFile(c, "config") + defer os.Remove(config) + + locatorFile := setupBlockHashFile(c, "block-hash", []string{TestHash, TestHash2}) + defer os.Remove(locatorFile) + + // Start keepservers. + arvadostest.StartKeep(2, false) + defer arvadostest.StopKeep(2) + + args := []string{"-config", config, "-block-hash-file", locatorFile} + os.Args = append(os.Args, args...) + + err := doMain() + c.Check(err, IsNil) + checkErrorLog(c, []string{TestHash, TestHash2}, "Block not found") +} -- 2.30.2