Merge branch '18874-merge-wb2'

[arvados.git] / tools / keep-block-check / keep-block-check.go

diff --git a/tools/keep-block-check/keep-block-check.go b/tools/keep-block-check/keep-block-check.go
index f27a4bc4c1af221b455b6ca4066ff4a85960839a..6127485835388d359b64e5ffae4cd2607c298cc6 100644 (file)
--- a/tools/keep-block-check/keep-block-check.go
+++ b/tools/keep-block-check/keep-block-check.go
@@ -1,3 +1,7 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+

 package main
 
 import (
 package main
 
 import (


@@ -5,25 +9,26 @@ import (
        "errors"
        "flag"
        "fmt"
        "errors"
        "flag"
        "fmt"

-       "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
-       "git.curoverse.com/arvados.git/sdk/go/keepclient"
+       "io"

        "io/ioutil"
        "log"
        "net/http"
        "os"
        "io/ioutil"
        "log"
        "net/http"
        "os"

-       "regexp"

        "strings"
        "time"
        "strings"
        "time"

+
+       "git.arvados.org/arvados.git/lib/cmd"
+       "git.arvados.org/arvados.git/sdk/go/arvadosclient"
+       "git.arvados.org/arvados.git/sdk/go/keepclient"

 )
 
 )
 

+var version = "dev"
+

 func main() {
 func main() {

-       err := doMain(os.Args[1:])
-       if err != nil {
-               log.Fatalf("%v", err)
-       }
+       os.Exit(doMain(os.Args[1:], os.Stderr))

 }
 
 }
 

-func doMain(args []string) error {
+func doMain(args []string, stderr io.Writer) int {

        flags := flag.NewFlagSet("keep-block-check", flag.ExitOnError)
 
        configFile := flags.String(
        flags := flag.NewFlagSet("keep-block-check", flag.ExitOnError)
 
        configFile := flags.String(


@@ -48,8 +53,8 @@ func doMain(args []string) error {
                "",
                "Block hash prefix. When a prefix is specified, only hashes listed in the file with this prefix will be checked.")
 
                "",
                "Block hash prefix. When a prefix is specified, only hashes listed in the file with this prefix will be checked.")
 

-       blobSignatureTTL := flags.Duration(
-               "blob-signing-ttl",
+       blobSignatureTTLFlag := flags.Duration(
+               "blob-signature-ttl",

                0,
                "Lifetime of blob permission signatures on the keepservers. If not provided, this will be retrieved from the API server's discovery document.")
 
                0,
                "Lifetime of blob permission signatures on the keepservers. If not provided, this will be retrieved from the API server's discovery document.")
 


@@ -58,34 +63,51 @@ func doMain(args []string) error {
                false,
                "Log progress of each block verification")
 
                false,
                "Log progress of each block verification")
 

-       // Parse args; omit the first arg which is the command name
-       flags.Parse(args)
+       getVersion := flags.Bool(
+               "version",
+               false,
+               "Print version information and exit.")
+
+       if ok, code := cmd.ParseFlags(flags, os.Args[0], args, "", stderr); !ok {
+               return code
+       } else if *getVersion {
+               fmt.Printf("%s %s\n", os.Args[0], version)
+               return 0
+       }

 
        config, blobSigningKey, err := loadConfig(*configFile)
        if err != nil {
 
        config, blobSigningKey, err := loadConfig(*configFile)
        if err != nil {

-               return fmt.Errorf("Error loading configuration from file: %s", err.Error())
+               fmt.Fprintf(stderr, "Error loading configuration from file: %s\n", err)
+               return 1

        }
 
        // get list of block locators to be checked
        blockLocators, err := getBlockLocators(*locatorFile, *prefix)
        if err != nil {
        }
 
        // get list of block locators to be checked
        blockLocators, err := getBlockLocators(*locatorFile, *prefix)
        if err != nil {

-               return fmt.Errorf("Error reading block hashes to be checked from file: %s", err.Error())
+               fmt.Fprintf(stderr, "Error reading block hashes to be checked from file: %s\n", err)
+               return 1

        }
 
        // setup keepclient
        }
 
        // setup keepclient

-       kc, err := setupKeepClient(config, *keepServicesJSON, *blobSignatureTTL)
+       kc, blobSignatureTTL, err := setupKeepClient(config, *keepServicesJSON, *blobSignatureTTLFlag)
+       if err != nil {
+               fmt.Fprintf(stderr, "Error configuring keepclient: %s\n", err)
+               return 1
+       }
+
+       err = performKeepBlockCheck(kc, blobSignatureTTL, blobSigningKey, blockLocators, *verbose)

        if err != nil {
        if err != nil {

-               return fmt.Errorf("Error configuring keepclient: %s", err.Error())
+               fmt.Fprintln(stderr, err)
+               return 1

        }
 
        }
 

-       return performKeepBlockCheck(kc, *blobSignatureTTL, blobSigningKey, blockLocators, *verbose)
+       return 0

 }
 
 type apiConfig struct {
        APIToken        string
        APIHost         string
        APIHostInsecure bool
 }
 
 type apiConfig struct {
        APIToken        string
        APIHost         string
        APIHostInsecure bool

-       ExternalClient  bool

 }
 
 // Load config from given file
 }
 
 // Load config from given file


@@ -99,8 +121,6 @@ func loadConfig(configFile string) (config apiConfig, blobSigningKey string, err
        return
 }
 
        return
 }
 

-var matchTrue = regexp.MustCompile("^(?i:1|yes|true)$")
-

 // Read config from file
 func readConfigFromFile(filename string) (config apiConfig, blobSigningKey string, err error) {
        if !strings.Contains(filename, "/") {
 // Read config from file
 func readConfigFromFile(filename string) (config apiConfig, blobSigningKey string, err error) {
        if !strings.Contains(filename, "/") {


@@ -130,9 +150,7 @@ func readConfigFromFile(filename string) (config apiConfig, blobSigningKey strin
                        case "ARVADOS_API_HOST":
                                config.APIHost = value
                        case "ARVADOS_API_HOST_INSECURE":
                        case "ARVADOS_API_HOST":
                                config.APIHost = value
                        case "ARVADOS_API_HOST_INSECURE":

-                               config.APIHostInsecure = matchTrue.MatchString(value)
-                       case "ARVADOS_EXTERNAL_CLIENT":
-                               config.ExternalClient = matchTrue.MatchString(value)
+                               config.APIHostInsecure = arvadosclient.StringBool(value)

                        case "ARVADOS_BLOB_SIGNING_KEY":
                                blobSigningKey = value
                        }
                        case "ARVADOS_BLOB_SIGNING_KEY":
                                blobSigningKey = value
                        }


@@ -143,17 +161,16 @@ func readConfigFromFile(filename string) (config apiConfig, blobSigningKey strin
 }
 
 // setup keepclient using the config provided
 }
 
 // setup keepclient using the config provided

-func setupKeepClient(config apiConfig, keepServicesJSON string, blobSignatureTTL time.Duration) (kc *keepclient.KeepClient, err error) {
+func setupKeepClient(config apiConfig, keepServicesJSON string, blobSignatureTTL time.Duration) (kc *keepclient.KeepClient, ttl time.Duration, err error) {

        arv := arvadosclient.ArvadosClient{
                ApiToken:    config.APIToken,
                ApiServer:   config.APIHost,
                ApiInsecure: config.APIHostInsecure,
                Client: &http.Client{Transport: &http.Transport{
                        TLSClientConfig: &tls.Config{InsecureSkipVerify: config.APIHostInsecure}}},
        arv := arvadosclient.ArvadosClient{
                ApiToken:    config.APIToken,
                ApiServer:   config.APIHost,
                ApiInsecure: config.APIHostInsecure,
                Client: &http.Client{Transport: &http.Transport{
                        TLSClientConfig: &tls.Config{InsecureSkipVerify: config.APIHostInsecure}}},

-               External: config.ExternalClient,

        }
 
        }
 

-       // if keepServicesJSON is provided, use it to load services; else, use DiscoverKeepServers
+       // If keepServicesJSON is provided, use it instead of service discovery

        if keepServicesJSON == "" {
                kc, err = keepclient.MakeKeepClient(&arv)
                if err != nil {
        if keepServicesJSON == "" {
                kc, err = keepclient.MakeKeepClient(&arv)
                if err != nil {


@@ -168,12 +185,13 @@ func setupKeepClient(config apiConfig, keepServicesJSON string, blobSignatureTTL
        }
 
        // Get if blobSignatureTTL is not provided
        }
 
        // Get if blobSignatureTTL is not provided

+       ttl = blobSignatureTTL

        if blobSignatureTTL == 0 {
                value, err := arv.Discovery("blobSignatureTtl")
                if err == nil {
        if blobSignatureTTL == 0 {
                value, err := arv.Discovery("blobSignatureTtl")
                if err == nil {

-                       blobSignatureTTL = time.Duration(int(value.(float64))) * time.Second
+                       ttl = time.Duration(int(value.(float64))) * time.Second

                } else {
                } else {

-                       return nil, err
+                       return nil, 0, err

                }
        }
 
                }
        }
 


@@ -231,7 +249,7 @@ func performKeepBlockCheck(kc *keepclient.KeepClient, blobSignatureTTL time.Dura
        log.Printf("Verify block totals: %d attempts, %d successes, %d errors", totalBlocks, totalBlocks-notFoundBlocks, notFoundBlocks)
 
        if notFoundBlocks > 0 {
        log.Printf("Verify block totals: %d attempts, %d successes, %d errors", totalBlocks, totalBlocks-notFoundBlocks, notFoundBlocks)
 
        if notFoundBlocks > 0 {

-               return fmt.Errorf("Block verification failed for %d out of %d blocks with matching prefix.", notFoundBlocks, totalBlocks)
+               return fmt.Errorf("Block verification failed for %d out of %d blocks with matching prefix", notFoundBlocks, totalBlocks)

        }
 
        return nil
        }
 
        return nil