X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/bbb0760e909b3ceca850b3aa319011fba2c98ed0..65efe2535e49058ccf8250f93e958ea653f55cd8:/tools/keep-block-check/keep-block-check.go

diff --git a/tools/keep-block-check/keep-block-check.go b/tools/keep-block-check/keep-block-check.go
index caed9d0469..646d417d88 100644
--- a/tools/keep-block-check/keep-block-check.go
+++ b/tools/keep-block-check/keep-block-check.go
@@ -17,13 +17,13 @@ import (
 )
 
 func main() {
-	err := doMain()
+	err := doMain(os.Args[1:])
 	if err != nil {
 		log.Fatalf("%v", err)
 	}
 }
 
-func doMain() error {
+func doMain(args []string) error {
 	flags := flag.NewFlagSet("keep-block-check", flag.ExitOnError)
 
 	configFile := flags.String(
@@ -48,8 +48,13 @@ func doMain() error {
 		"",
 		"Block hash prefix. When a prefix is specified, only hashes listed in the file with this prefix will be checked.")
 
+	verbose := flags.Bool(
+		"v",
+		false,
+		"Log progress of each block verification")
+
 	// Parse args; omit the first arg which is the command name
-	flags.Parse(os.Args[1:])
+	flags.Parse(args)
 
 	config, blobSigningKey, err := loadConfig(*configFile)
 	if err != nil {
@@ -57,7 +62,7 @@ func doMain() error {
 	}
 
 	// get list of block locators to be checked
-	blockLocators, err := getBlockLocators(*locatorFile)
+	blockLocators, err := getBlockLocators(*locatorFile, *prefix)
 	if err != nil {
 		return fmt.Errorf("Error reading block hashes to be checked from file: %s", err.Error())
 	}
@@ -68,7 +73,7 @@ func doMain() error {
 		return fmt.Errorf("Error configuring keepclient: %s", err.Error())
 	}
 
-	return performKeepBlockCheck(kc, blobSigningKey, *prefix, blockLocators)
+	return performKeepBlockCheck(kc, blobSigningKey, blockLocators, *verbose)
 }
 
 type apiConfig struct {
@@ -81,7 +86,7 @@ type apiConfig struct {
 // Load config from given file
 func loadConfig(configFile string) (config apiConfig, blobSigningKey string, err error) {
 	if configFile == "" {
-		err = errors.New("API config file not specified")
+		err = errors.New("Client config file not specified")
 		return
 	}
 
@@ -110,20 +115,22 @@ func readConfigFromFile(filename string) (config apiConfig, blobSigningKey strin
 		}
 
 		kv := strings.SplitN(line, "=", 2)
-		key := strings.TrimSpace(kv[0])
-		value := strings.TrimSpace(kv[1])
-
-		switch key {
-		case "ARVADOS_API_TOKEN":
-			config.APIToken = value
-		case "ARVADOS_API_HOST":
-			config.APIHost = value
-		case "ARVADOS_API_HOST_INSECURE":
-			config.APIHostInsecure = matchTrue.MatchString(value)
-		case "ARVADOS_EXTERNAL_CLIENT":
-			config.ExternalClient = matchTrue.MatchString(value)
-		case "ARVADOS_BLOB_SIGNING_KEY":
-			blobSigningKey = value
+		if len(kv) == 2 {
+			key := strings.TrimSpace(kv[0])
+			value := strings.TrimSpace(kv[1])
+
+			switch key {
+			case "ARVADOS_API_TOKEN":
+				config.APIToken = value
+			case "ARVADOS_API_HOST":
+				config.APIHost = value
+			case "ARVADOS_API_HOST_INSECURE":
+				config.APIHostInsecure = matchTrue.MatchString(value)
+			case "ARVADOS_EXTERNAL_CLIENT":
+				config.ExternalClient = matchTrue.MatchString(value)
+			case "ARVADOS_BLOB_SIGNING_KEY":
+				blobSigningKey = value
+			}
 		}
 	}
 
@@ -159,45 +166,40 @@ func setupKeepClient(config apiConfig, keepServicesJSON string) (kc *keepclient.
 }
 
 // Get list of unique block locators from the given file
-func getBlockLocators(locatorFile string) (locators []string, err error) {
+func getBlockLocators(locatorFile, prefix string) (locators []string, err error) {
 	if locatorFile == "" {
 		err = errors.New("block-hash-file not specified")
 		return
 	}
 
 	content, err := ioutil.ReadFile(locatorFile)
-
 	if err != nil {
 		return
 	}
 
-	locatorMap := make(map[string]string)
-	lines := strings.Split(string(content), "\n")
-	for _, line := range lines {
-		if line == "" {
+	locatorMap := make(map[string]bool)
+	for _, line := range strings.Split(string(content), "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" || !strings.HasPrefix(line, prefix) || locatorMap[line] {
 			continue
 		}
-		trimmedLine := strings.TrimSpace(line)
-		locatorMap[trimmedLine] = trimmedLine
-	}
-
-	for _, locator := range locatorMap {
-		locators = append(locators, locator)
+		locators = append(locators, line)
+		locatorMap[line] = true
 	}
 
 	return
 }
 
 // Get block headers from keep. Log any errors.
-func performKeepBlockCheck(kc *keepclient.KeepClient, blobSigningKey, prefix string, blockLocators []string) error {
-	totalBlocks := 0
+func performKeepBlockCheck(kc *keepclient.KeepClient, blobSigningKey string, blockLocators []string, verbose bool) error {
+	totalBlocks := len(blockLocators)
 	notFoundBlocks := 0
+	current := 0
 	for _, locator := range blockLocators {
-		if !strings.HasPrefix(locator, prefix) {
-			continue
+		current++
+		if verbose {
+			log.Printf("Checking block %d of %d: %v", current, totalBlocks, locator)
 		}
-
-		totalBlocks++
 		getLocator := locator
 		if blobSigningKey != "" {
 			expiresAt := time.Now().AddDate(0, 0, 1)
@@ -207,11 +209,14 @@ func performKeepBlockCheck(kc *keepclient.KeepClient, blobSigningKey, prefix str
 		_, _, err := kc.Ask(getLocator)
 		if err != nil {
 			notFoundBlocks++
-			log.Printf("Error getting head info for block: %v %v", locator, err)
+			log.Printf("Error verifying block %v: %v", locator, err)
 		}
 	}
+
+	log.Printf("Verify block totals: %d attempts, %d successes, %d errors", totalBlocks, totalBlocks-notFoundBlocks, notFoundBlocks)
+
 	if notFoundBlocks > 0 {
-		return fmt.Errorf("Head information not found for %d out of %d blocks with matching prefix.", notFoundBlocks, totalBlocks)
+		return fmt.Errorf("Block verification failed for %d out of %d blocks with matching prefix.", notFoundBlocks, totalBlocks)
 	}
 
 	return nil