--- /dev/null
+package main
+
+import (
+ "crypto/tls"
+ "errors"
+ "flag"
+ "fmt"
+ "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/keepclient"
+ "io/ioutil"
+ "log"
+ "net/http"
+ "os"
+ "regexp"
+ "strings"
+ "time"
+)
+
+func main() {
+ err := doMain(os.Args[1:])
+ if err != nil {
+ log.Fatalf("%v", err)
+ }
+}
+
+func doMain(args []string) error {
+ flags := flag.NewFlagSet("keep-block-check", flag.ExitOnError)
+
+ configFile := flags.String(
+ "config",
+ "",
+ "Configuration filename. May be either a pathname to a config file, or (for example) 'foo' as shorthand for $HOME/.config/arvados/foo.conf file. This file is expected to specify the values for ARVADOS_API_TOKEN, ARVADOS_API_HOST, ARVADOS_API_HOST_INSECURE, and ARVADOS_BLOB_SIGNING_KEY for the source.")
+
+ keepServicesJSON := flags.String(
+ "keep-services-json",
+ "",
+ "An optional list of available keepservices. "+
+ "If not provided, this list is obtained from api server configured in config-file.")
+
+ locatorFile := flags.String(
+ "block-hash-file",
+ "",
+ "Filename containing the block hashes to be checked. This is required. "+
+ "This file contains the block hashes one per line.")
+
+ prefix := flags.String(
+ "prefix",
+ "",
+ "Block hash prefix. When a prefix is specified, only hashes listed in the file with this prefix will be checked.")
+
+ verbose := flags.Bool(
+ "v",
+ false,
+ "Log progress of each block verification")
+
+ // Parse args; omit the first arg which is the command name
+ flags.Parse(args)
+
+ config, blobSigningKey, err := loadConfig(*configFile)
+ if err != nil {
+ return fmt.Errorf("Error loading configuration from file: %s", err.Error())
+ }
+
+ // get list of block locators to be checked
+ blockLocators, err := getBlockLocators(*locatorFile, *prefix)
+ if err != nil {
+ return fmt.Errorf("Error reading block hashes to be checked from file: %s", err.Error())
+ }
+
+ // setup keepclient
+ kc, err := setupKeepClient(config, *keepServicesJSON)
+ if err != nil {
+ return fmt.Errorf("Error configuring keepclient: %s", err.Error())
+ }
+
+ return performKeepBlockCheck(kc, blobSigningKey, blockLocators, *verbose)
+}
+
+type apiConfig struct {
+ APIToken string
+ APIHost string
+ APIHostInsecure bool
+ ExternalClient bool
+}
+
+// Load config from given file
+func loadConfig(configFile string) (config apiConfig, blobSigningKey string, err error) {
+ if configFile == "" {
+ err = errors.New("Client config file not specified")
+ return
+ }
+
+ config, blobSigningKey, err = readConfigFromFile(configFile)
+ return
+}
+
+var matchTrue = regexp.MustCompile("^(?i:1|yes|true)$")
+
+// Read config from file
+func readConfigFromFile(filename string) (config apiConfig, blobSigningKey string, err error) {
+ if !strings.Contains(filename, "/") {
+ filename = os.Getenv("HOME") + "/.config/arvados/" + filename + ".conf"
+ }
+
+ content, err := ioutil.ReadFile(filename)
+
+ if err != nil {
+ return
+ }
+
+ lines := strings.Split(string(content), "\n")
+ for _, line := range lines {
+ if line == "" {
+ continue
+ }
+
+ kv := strings.SplitN(line, "=", 2)
+ if len(kv) == 2 {
+ key := strings.TrimSpace(kv[0])
+ value := strings.TrimSpace(kv[1])
+
+ switch key {
+ case "ARVADOS_API_TOKEN":
+ config.APIToken = value
+ case "ARVADOS_API_HOST":
+ config.APIHost = value
+ case "ARVADOS_API_HOST_INSECURE":
+ config.APIHostInsecure = matchTrue.MatchString(value)
+ case "ARVADOS_EXTERNAL_CLIENT":
+ config.ExternalClient = matchTrue.MatchString(value)
+ case "ARVADOS_BLOB_SIGNING_KEY":
+ blobSigningKey = value
+ }
+ }
+ }
+
+ return
+}
+
+// setup keepclient using the config provided
+func setupKeepClient(config apiConfig, keepServicesJSON string) (kc *keepclient.KeepClient, err error) {
+ arv := arvadosclient.ArvadosClient{
+ ApiToken: config.APIToken,
+ ApiServer: config.APIHost,
+ ApiInsecure: config.APIHostInsecure,
+ Client: &http.Client{Transport: &http.Transport{
+ TLSClientConfig: &tls.Config{InsecureSkipVerify: config.APIHostInsecure}}},
+ External: config.ExternalClient,
+ }
+
+ // if keepServicesJSON is provided, use it to load services; else, use DiscoverKeepServers
+ if keepServicesJSON == "" {
+ kc, err = keepclient.MakeKeepClient(&arv)
+ if err != nil {
+ return
+ }
+ } else {
+ kc = keepclient.New(&arv)
+ err = kc.LoadKeepServicesFromJSON(keepServicesJSON)
+ if err != nil {
+ return
+ }
+ }
+
+ return
+}
+
+// Get list of unique block locators from the given file
+func getBlockLocators(locatorFile, prefix string) (locators []string, err error) {
+ if locatorFile == "" {
+ err = errors.New("block-hash-file not specified")
+ return
+ }
+
+ content, err := ioutil.ReadFile(locatorFile)
+ if err != nil {
+ return
+ }
+
+ locatorMap := make(map[string]bool)
+ for _, line := range strings.Split(string(content), "\n") {
+ line = strings.TrimSpace(line)
+ if line == "" || !strings.HasPrefix(line, prefix) || locatorMap[line] {
+ continue
+ }
+ locators = append(locators, line)
+ locatorMap[line] = true
+ }
+
+ return
+}
+
+// Get block headers from keep. Log any errors.
+func performKeepBlockCheck(kc *keepclient.KeepClient, blobSigningKey string, blockLocators []string, verbose bool) error {
+ totalBlocks := len(blockLocators)
+ notFoundBlocks := 0
+ current := 0
+ for _, locator := range blockLocators {
+ current++
+ if verbose {
+ log.Printf("Verifying block %d of %d: %v", current, totalBlocks, locator)
+ }
+ getLocator := locator
+ if blobSigningKey != "" {
+ expiresAt := time.Now().AddDate(0, 0, 1)
+ getLocator = keepclient.SignLocator(locator, kc.Arvados.ApiToken, expiresAt, []byte(blobSigningKey))
+ }
+
+ _, _, err := kc.Ask(getLocator)
+ if err != nil {
+ notFoundBlocks++
+ log.Printf("Error verifying block %v: %v", locator, err)
+ }
+ }
+
+ log.Printf("Verify block totals: %d attempts, %d successes, %d errors", totalBlocks, totalBlocks-notFoundBlocks, notFoundBlocks)
+
+ if notFoundBlocks > 0 {
+ return fmt.Errorf("Block verification failed for %d out of %d blocks with matching prefix.", notFoundBlocks, totalBlocks)
+ }
+
+ return nil
+}
--- /dev/null
+package main
+
+import (
+ "bytes"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "log"
+ "os"
+ "regexp"
+ "strings"
+ "testing"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ "git.curoverse.com/arvados.git/sdk/go/keepclient"
+
+ . "gopkg.in/check.v1"
+)
+
+// Gocheck boilerplate
+func Test(t *testing.T) {
+ TestingT(t)
+}
+
+// Gocheck boilerplate
+var _ = Suite(&ServerRequiredSuite{})
+var _ = Suite(&DoMainTestSuite{})
+
+type ServerRequiredSuite struct{}
+type DoMainTestSuite struct{}
+
+var kc *keepclient.KeepClient
+var logBuffer bytes.Buffer
+
+var TestHash = "aaaa09c290d0fb1ca068ffaddf22cbd0"
+var TestHash2 = "aaaac516f788aec4f30932ffb6395c39"
+
+func (s *ServerRequiredSuite) SetUpSuite(c *C) {
+ arvadostest.StartAPI()
+}
+
+func (s *ServerRequiredSuite) TearDownSuite(c *C) {
+ arvadostest.StopAPI()
+ arvadostest.ResetEnv()
+}
+
+func (s *ServerRequiredSuite) SetUpTest(c *C) {
+ logOutput := io.MultiWriter(&logBuffer)
+ log.SetOutput(logOutput)
+}
+
+func (s *ServerRequiredSuite) TearDownTest(c *C) {
+ arvadostest.StopKeep(2)
+ log.SetOutput(os.Stdout)
+ log.Printf("%v", logBuffer.String())
+}
+
+func (s *DoMainTestSuite) SetUpSuite(c *C) {
+}
+
+func (s *DoMainTestSuite) SetUpTest(c *C) {
+ logOutput := io.MultiWriter(&logBuffer)
+ log.SetOutput(logOutput)
+}
+
+func (s *DoMainTestSuite) TearDownTest(c *C) {
+ log.SetOutput(os.Stdout)
+ log.Printf("%v", logBuffer.String())
+}
+
+func setupKeepBlockCheck(c *C, enforcePermissions bool, keepServicesJSON string) {
+ var config apiConfig
+ config.APIHost = os.Getenv("ARVADOS_API_HOST")
+ config.APIToken = arvadostest.DataManagerToken
+ config.APIHostInsecure = matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))
+
+ // Start Keep servers
+ arvadostest.StartKeep(2, enforcePermissions)
+
+ // setup keepclients
+ var err error
+ kc, err = setupKeepClient(config, keepServicesJSON)
+ c.Check(err, IsNil)
+}
+
+// Setup test data
+func setupTestData(c *C) []string {
+ allLocators := []string{}
+
+ // Put a few blocks
+ for i := 0; i < 5; i++ {
+ hash, _, err := kc.PutB([]byte(fmt.Sprintf("keep-block-check-test-data-%d", i)))
+ c.Check(err, IsNil)
+ allLocators = append(allLocators, strings.Split(hash, "+A")[0])
+ }
+
+ return allLocators
+}
+
+func setupConfigFile(c *C, fileName string) string {
+ // Setup a config file
+ file, err := ioutil.TempFile(os.TempDir(), fileName)
+ c.Check(err, IsNil)
+
+ // Add config to file. While at it, throw some extra white space
+ fileContent := "ARVADOS_API_HOST=" + os.Getenv("ARVADOS_API_HOST") + "\n"
+ fileContent += "ARVADOS_API_TOKEN=" + arvadostest.DataManagerToken + "\n"
+ fileContent += "\n"
+ fileContent += "ARVADOS_API_HOST_INSECURE=" + os.Getenv("ARVADOS_API_HOST_INSECURE") + "\n"
+ fileContent += " ARVADOS_EXTERNAL_CLIENT = false \n"
+ fileContent += " NotANameValuePairAndShouldGetIgnored \n"
+ fileContent += "ARVADOS_BLOB_SIGNING_KEY=abcdefg\n"
+
+ _, err = file.Write([]byte(fileContent))
+ c.Check(err, IsNil)
+
+ return file.Name()
+}
+
+func setupBlockHashFile(c *C, name string, blocks []string) string {
+ // Setup a block hash file
+ file, err := ioutil.TempFile(os.TempDir(), name)
+ c.Check(err, IsNil)
+
+ // Add the hashes to the file. While at it, throw some extra white space
+ fileContent := ""
+ for _, hash := range blocks {
+ fileContent += fmt.Sprintf(" %s \n", hash)
+ }
+ fileContent += "\n"
+ _, err = file.Write([]byte(fileContent))
+ c.Check(err, IsNil)
+
+ return file.Name()
+}
+
+func checkErrorLog(c *C, blocks []string, prefix, suffix string) {
+ for _, hash := range blocks {
+ expected := prefix + `.*` + hash + `.*` + suffix
+ match, _ := regexp.MatchString(expected, logBuffer.String())
+ c.Assert(match, Equals, true)
+ }
+}
+
+func checkNoErrorsLogged(c *C, prefix, suffix string) {
+ expected := prefix + `.*` + suffix
+ match, _ := regexp.MatchString(expected, logBuffer.String())
+ c.Assert(match, Equals, false)
+}
+
+func (s *ServerRequiredSuite) TestBlockCheck(c *C) {
+ setupKeepBlockCheck(c, false, "")
+ allLocators := setupTestData(c)
+ err := performKeepBlockCheck(kc, "", allLocators, true)
+ c.Check(err, IsNil)
+ checkNoErrorsLogged(c, "Error verifying block", "Block not found")
+}
+
+func (s *ServerRequiredSuite) TestBlockCheckWithBlobSigning(c *C) {
+ setupKeepBlockCheck(c, true, "")
+ allLocators := setupTestData(c)
+ err := performKeepBlockCheck(kc, arvadostest.BlobSigningKey, allLocators, true)
+ c.Check(err, IsNil)
+ checkNoErrorsLogged(c, "Error verifying block", "Block not found")
+}
+
+func (s *ServerRequiredSuite) TestBlockCheck_NoSuchBlock(c *C) {
+ setupKeepBlockCheck(c, false, "")
+ allLocators := setupTestData(c)
+ allLocators = append(allLocators, TestHash)
+ allLocators = append(allLocators, TestHash2)
+ err := performKeepBlockCheck(kc, "", allLocators, true)
+ c.Check(err, NotNil)
+ c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 7 blocks with matching prefix.")
+ checkErrorLog(c, []string{TestHash, TestHash2}, "Error verifying block", "Block not found")
+}
+
+func (s *ServerRequiredSuite) TestBlockCheck_NoSuchBlock_WithMatchingPrefix(c *C) {
+ setupKeepBlockCheck(c, false, "")
+ allLocators := setupTestData(c)
+ allLocators = append(allLocators, TestHash)
+ allLocators = append(allLocators, TestHash2)
+ locatorFile := setupBlockHashFile(c, "block-hash", allLocators)
+ defer os.Remove(locatorFile)
+ locators, err := getBlockLocators(locatorFile, "aaa")
+ c.Check(err, IsNil)
+ err = performKeepBlockCheck(kc, "", locators, true)
+ c.Check(err, NotNil)
+ // Of the 7 blocks in allLocators, only two match the prefix and hence only those are checked
+ c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix.")
+ checkErrorLog(c, []string{TestHash, TestHash2}, "Error verifying block", "Block not found")
+}
+
+func (s *ServerRequiredSuite) TestBlockCheck_NoSuchBlock_WithPrefixMismatch(c *C) {
+ setupKeepBlockCheck(c, false, "")
+ allLocators := setupTestData(c)
+ allLocators = append(allLocators, TestHash)
+ allLocators = append(allLocators, TestHash2)
+ locatorFile := setupBlockHashFile(c, "block-hash", allLocators)
+ defer os.Remove(locatorFile)
+ locators, err := getBlockLocators(locatorFile, "999")
+ c.Check(err, IsNil)
+ err = performKeepBlockCheck(kc, "", locators, true)
+ c.Check(err, IsNil) // there were no matching locators in file and hence nothing was checked
+}
+
+func (s *ServerRequiredSuite) TestBlockCheck_BadSignature(c *C) {
+ setupKeepBlockCheck(c, true, "")
+ setupTestData(c)
+ err := performKeepBlockCheck(kc, "badblobsigningkey", []string{TestHash, TestHash2}, false)
+ c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix.")
+ checkErrorLog(c, []string{TestHash, TestHash2}, "Error verifying block", "HTTP 403")
+ // verbose logging not requested
+ c.Assert(strings.Contains(logBuffer.String(), "Verifying block 1 of 2"), Equals, false)
+}
+
+var testKeepServicesJSON = `{
+ "kind":"arvados#keepServiceList",
+ "etag":"",
+ "self_link":"",
+ "offset":null, "limit":null,
+ "items":[
+ {"href":"/keep_services/zzzzz-bi6l4-123456789012340",
+ "kind":"arvados#keepService",
+ "uuid":"zzzzz-bi6l4-123456789012340",
+ "service_host":"keep0.zzzzz.arvadosapi.com",
+ "service_port":25107,
+ "service_ssl_flag":false,
+ "service_type":"disk",
+ "read_only":false },
+ {"href":"/keep_services/zzzzz-bi6l4-123456789012341",
+ "kind":"arvados#keepService",
+ "uuid":"zzzzz-bi6l4-123456789012341",
+ "service_host":"keep0.zzzzz.arvadosapi.com",
+ "service_port":25108,
+ "service_ssl_flag":false,
+ "service_type":"disk",
+ "read_only":false }
+ ],
+ "items_available":2 }`
+
+// Setup block-check using keepServicesJSON with fake keepservers.
+// Expect error during performKeepBlockCheck due to unreachable keepservers.
+func (s *ServerRequiredSuite) TestErrorDuringKeepBlockCheck_FakeKeepservers(c *C) {
+ setupKeepBlockCheck(c, false, testKeepServicesJSON)
+ err := performKeepBlockCheck(kc, "", []string{TestHash, TestHash2}, true)
+ c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix.")
+ checkErrorLog(c, []string{TestHash, TestHash2}, "Error verifying block", "")
+}
+
+// Test keep-block-check initialization with keepServicesJSON
+func (s *ServerRequiredSuite) TestKeepBlockCheck_InitializeWithKeepServicesJSON(c *C) {
+ setupKeepBlockCheck(c, false, testKeepServicesJSON)
+ found := 0
+ for k := range kc.LocalRoots() {
+ if k == "zzzzz-bi6l4-123456789012340" || k == "zzzzz-bi6l4-123456789012341" {
+ found++
+ }
+ }
+ c.Check(found, Equals, 2)
+}
+
+// Test loadConfig func
+func (s *ServerRequiredSuite) TestLoadConfig(c *C) {
+ // Setup config file
+ configFile := setupConfigFile(c, "config")
+ defer os.Remove(configFile)
+
+ // load configuration from the file
+ config, blobSigningKey, err := loadConfig(configFile)
+ c.Check(err, IsNil)
+
+ c.Assert(config.APIHost, Equals, os.Getenv("ARVADOS_API_HOST"))
+ c.Assert(config.APIToken, Equals, arvadostest.DataManagerToken)
+ c.Assert(config.APIHostInsecure, Equals, matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE")))
+ c.Assert(config.ExternalClient, Equals, false)
+ c.Assert(blobSigningKey, Equals, "abcdefg")
+}
+
+func (s *DoMainTestSuite) Test_doMain_WithNoConfig(c *C) {
+ args := []string{"-prefix", "a"}
+ err := doMain(args)
+ c.Check(err, NotNil)
+ c.Assert(strings.Contains(err.Error(), "config file not specified"), Equals, true)
+}
+
+func (s *DoMainTestSuite) Test_doMain_WithNoSuchConfigFile(c *C) {
+ args := []string{"-config", "no-such-file"}
+ err := doMain(args)
+ c.Check(err, NotNil)
+ c.Assert(strings.Contains(err.Error(), "no such file or directory"), Equals, true)
+}
+
+func (s *DoMainTestSuite) Test_doMain_WithNoBlockHashFile(c *C) {
+ config := setupConfigFile(c, "config")
+ defer os.Remove(config)
+
+ // Start keepservers.
+ arvadostest.StartKeep(2, false)
+ defer arvadostest.StopKeep(2)
+
+ args := []string{"-config", config}
+ err := doMain(args)
+ c.Assert(strings.Contains(err.Error(), "block-hash-file not specified"), Equals, true)
+}
+
+func (s *DoMainTestSuite) Test_doMain_WithNoSuchBlockHashFile(c *C) {
+ config := setupConfigFile(c, "config")
+ defer os.Remove(config)
+
+ arvadostest.StartKeep(2, false)
+ defer arvadostest.StopKeep(2)
+
+ args := []string{"-config", config, "-block-hash-file", "no-such-file"}
+ err := doMain(args)
+ c.Assert(strings.Contains(err.Error(), "no such file or directory"), Equals, true)
+}
+
+func (s *DoMainTestSuite) Test_doMain(c *C) {
+ // Start keepservers.
+ arvadostest.StartKeep(2, false)
+ defer arvadostest.StopKeep(2)
+
+ config := setupConfigFile(c, "config")
+ defer os.Remove(config)
+
+ locatorFile := setupBlockHashFile(c, "block-hash", []string{TestHash, TestHash2})
+ defer os.Remove(locatorFile)
+
+ args := []string{"-config", config, "-block-hash-file", locatorFile, "-v"}
+ err := doMain(args)
+ c.Check(err, NotNil)
+ c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix.")
+ checkErrorLog(c, []string{TestHash, TestHash2}, "Error verifying block", "Block not found")
+ c.Assert(strings.Contains(logBuffer.String(), "Verifying block 1 of 2"), Equals, true)
+}
projects / arvados.git / commitdiff