X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/f1e315aa701757ee8bf497805033b039b21515ba..dddaa9f008f8cfb6d1dec5379b3ef2e32ca54565:/tools/keep-block-check/keep-block-check.go diff --git a/tools/keep-block-check/keep-block-check.go b/tools/keep-block-check/keep-block-check.go index 431703545a..995a1fd559 100644 --- a/tools/keep-block-check/keep-block-check.go +++ b/tools/keep-block-check/keep-block-check.go @@ -1,3 +1,7 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: AGPL-3.0 + package main import ( @@ -5,25 +9,26 @@ import ( "errors" "flag" "fmt" - "git.curoverse.com/arvados.git/sdk/go/arvadosclient" - "git.curoverse.com/arvados.git/sdk/go/keepclient" + "io" "io/ioutil" "log" "net/http" "os" - "regexp" "strings" "time" + + "git.arvados.org/arvados.git/lib/cmd" + "git.arvados.org/arvados.git/sdk/go/arvadosclient" + "git.arvados.org/arvados.git/sdk/go/keepclient" ) +var version = "dev" + func main() { - err := doMain() - if err != nil { - log.Fatalf("%v", err) - } + os.Exit(doMain(os.Args[1:], os.Stderr)) } -func doMain() error { +func doMain(args []string, stderr io.Writer) int { flags := flag.NewFlagSet("keep-block-check", flag.ExitOnError) configFile := flags.String( @@ -48,28 +53,55 @@ func doMain() error { "", "Block hash prefix. When a prefix is specified, only hashes listed in the file with this prefix will be checked.") - // Parse args; omit the first arg which is the command name - flags.Parse(os.Args[1:]) + blobSignatureTTLFlag := flags.Duration( + "blob-signature-ttl", + 0, + "Lifetime of blob permission signatures on the keepservers. If not provided, this will be retrieved from the API server's discovery document.") + + verbose := flags.Bool( + "v", + false, + "Log progress of each block verification") + + getVersion := flags.Bool( + "version", + false, + "Print version information and exit.") + + if ok, code := cmd.ParseFlags(flags, os.Args[0], args, "", stderr); !ok { + return code + } else if *getVersion { + fmt.Printf("%s %s\n", os.Args[0], version) + return 0 + } config, blobSigningKey, err := loadConfig(*configFile) if err != nil { - return fmt.Errorf("Error loading configuration from file: %s", err.Error()) + fmt.Fprintf(stderr, "Error loading configuration from file: %s\n", err) + return 1 } // get list of block locators to be checked - blockLocators, err := getBlockLocators(*locatorFile) + blockLocators, err := getBlockLocators(*locatorFile, *prefix) if err != nil { - return fmt.Errorf("Error reading block hashes to be checked from file: %s", err.Error()) + fmt.Fprintf(stderr, "Error reading block hashes to be checked from file: %s\n", err) + return 1 } // setup keepclient - kc, err := setupKeepClient(config, *keepServicesJSON) + kc, blobSignatureTTL, err := setupKeepClient(config, *keepServicesJSON, *blobSignatureTTLFlag) if err != nil { - return fmt.Errorf("Error configuring keepclient: %s", err.Error()) + fmt.Fprintf(stderr, "Error configuring keepclient: %s\n", err) + return 1 } - performKeepBlockCheck(kc, blobSigningKey, *prefix, blockLocators) - return nil + err = performKeepBlockCheck(kc, blobSignatureTTL, blobSigningKey, blockLocators, *verbose) + if err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + + return 0 } type apiConfig struct { @@ -82,7 +114,7 @@ type apiConfig struct { // Load config from given file func loadConfig(configFile string) (config apiConfig, blobSigningKey string, err error) { if configFile == "" { - err = errors.New("API config file not specified") + err = errors.New("Client config file not specified") return } @@ -90,8 +122,6 @@ func loadConfig(configFile string) (config apiConfig, blobSigningKey string, err return } -var matchTrue = regexp.MustCompile("^(?i:1|yes|true)$") - // Read config from file func readConfigFromFile(filename string) (config apiConfig, blobSigningKey string, err error) { if !strings.Contains(filename, "/") { @@ -111,20 +141,22 @@ func readConfigFromFile(filename string) (config apiConfig, blobSigningKey strin } kv := strings.SplitN(line, "=", 2) - key := strings.TrimSpace(kv[0]) - value := strings.TrimSpace(kv[1]) - - switch key { - case "ARVADOS_API_TOKEN": - config.APIToken = value - case "ARVADOS_API_HOST": - config.APIHost = value - case "ARVADOS_API_HOST_INSECURE": - config.APIHostInsecure = matchTrue.MatchString(value) - case "ARVADOS_EXTERNAL_CLIENT": - config.ExternalClient = matchTrue.MatchString(value) - case "ARVADOS_BLOB_SIGNING_KEY": - blobSigningKey = value + if len(kv) == 2 { + key := strings.TrimSpace(kv[0]) + value := strings.TrimSpace(kv[1]) + + switch key { + case "ARVADOS_API_TOKEN": + config.APIToken = value + case "ARVADOS_API_HOST": + config.APIHost = value + case "ARVADOS_API_HOST_INSECURE": + config.APIHostInsecure = arvadosclient.StringBool(value) + case "ARVADOS_EXTERNAL_CLIENT": + config.ExternalClient = arvadosclient.StringBool(value) + case "ARVADOS_BLOB_SIGNING_KEY": + blobSigningKey = value + } } } @@ -132,7 +164,7 @@ func readConfigFromFile(filename string) (config apiConfig, blobSigningKey strin } // setup keepclient using the config provided -func setupKeepClient(config apiConfig, keepServicesJSON string) (kc *keepclient.KeepClient, err error) { +func setupKeepClient(config apiConfig, keepServicesJSON string, blobSignatureTTL time.Duration) (kc *keepclient.KeepClient, ttl time.Duration, err error) { arv := arvadosclient.ArvadosClient{ ApiToken: config.APIToken, ApiServer: config.APIHost, @@ -142,7 +174,7 @@ func setupKeepClient(config apiConfig, keepServicesJSON string) (kc *keepclient. External: config.ExternalClient, } - // if keepServicesJSON is provided, use it to load services; else, use DiscoverKeepServers + // If keepServicesJSON is provided, use it instead of service discovery if keepServicesJSON == "" { kc, err = keepclient.MakeKeepClient(&arv) if err != nil { @@ -156,48 +188,73 @@ func setupKeepClient(config apiConfig, keepServicesJSON string) (kc *keepclient. } } + // Get if blobSignatureTTL is not provided + ttl = blobSignatureTTL + if blobSignatureTTL == 0 { + value, err := arv.Discovery("blobSignatureTtl") + if err == nil { + ttl = time.Duration(int(value.(float64))) * time.Second + } else { + return nil, 0, err + } + } + return } -// Get list of block locators from the given file -func getBlockLocators(locatorFile string) (locators []string, err error) { +// Get list of unique block locators from the given file +func getBlockLocators(locatorFile, prefix string) (locators []string, err error) { if locatorFile == "" { err = errors.New("block-hash-file not specified") return } content, err := ioutil.ReadFile(locatorFile) - if err != nil { return } - lines := strings.Split(string(content), "\n") - for _, line := range lines { - if line == "" { + locatorMap := make(map[string]bool) + for _, line := range strings.Split(string(content), "\n") { + line = strings.TrimSpace(line) + if line == "" || !strings.HasPrefix(line, prefix) || locatorMap[line] { continue } - locators = append(locators, strings.TrimSpace(line)) + locators = append(locators, line) + locatorMap[line] = true } return } // Get block headers from keep. Log any errors. -func performKeepBlockCheck(kc *keepclient.KeepClient, blobSigningKey, prefix string, blockLocators []string) { +func performKeepBlockCheck(kc *keepclient.KeepClient, blobSignatureTTL time.Duration, blobSigningKey string, blockLocators []string, verbose bool) error { + totalBlocks := len(blockLocators) + notFoundBlocks := 0 + current := 0 for _, locator := range blockLocators { - if !strings.HasPrefix(locator, prefix) { - continue + current++ + if verbose { + log.Printf("Verifying block %d of %d: %v", current, totalBlocks, locator) } getLocator := locator if blobSigningKey != "" { expiresAt := time.Now().AddDate(0, 0, 1) - getLocator = keepclient.SignLocator(locator, kc.Arvados.ApiToken, expiresAt, []byte(blobSigningKey)) + getLocator = keepclient.SignLocator(locator, kc.Arvados.ApiToken, expiresAt, blobSignatureTTL, []byte(blobSigningKey)) } _, _, err := kc.Ask(getLocator) if err != nil { - log.Printf("Error getting head info for block: %v %v", locator, err) + notFoundBlocks++ + log.Printf("Error verifying block %v: %v", locator, err) } } + + log.Printf("Verify block totals: %d attempts, %d successes, %d errors", totalBlocks, totalBlocks-notFoundBlocks, notFoundBlocks) + + if notFoundBlocks > 0 { + return fmt.Errorf("Block verification failed for %d out of %d blocks with matching prefix", notFoundBlocks, totalBlocks) + } + + return nil }