9005: Keep service discovery up to date automatically.

[arvados.git] / tools / keep-block-check / keep-block-check.go

diff --git a/tools/keep-block-check/keep-block-check.go b/tools/keep-block-check/keep-block-check.go
index 646d417d880e2941ab7b785dab4b8c3328fe3df4..e57a9abed4b336ca345808ec85bcbf8733b02a08 100644 (file)
--- a/tools/keep-block-check/keep-block-check.go
+++ b/tools/keep-block-check/keep-block-check.go
@@ -5,8 +5,6 @@ import (
        "errors"
        "flag"
        "fmt"
-       "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
-       "git.curoverse.com/arvados.git/sdk/go/keepclient"
        "io/ioutil"
        "log"
        "net/http"
@@ -14,6 +12,9 @@ import (
        "regexp"
        "strings"
        "time"
+
+       "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+       "git.curoverse.com/arvados.git/sdk/go/keepclient"
 )
 
 func main() {
@@ -48,6 +49,11 @@ func doMain(args []string) error {
                "",
                "Block hash prefix. When a prefix is specified, only hashes listed in the file with this prefix will be checked.")
 
+       blobSignatureTTLFlag := flags.Duration(
+               "blob-signature-ttl",
+               0,
+               "Lifetime of blob permission signatures on the keepservers. If not provided, this will be retrieved from the API server's discovery document.")
+
        verbose := flags.Bool(
                "v",
                false,
@@ -68,12 +74,12 @@ func doMain(args []string) error {
        }
 
        // setup keepclient
-       kc, err := setupKeepClient(config, *keepServicesJSON)
+       kc, blobSignatureTTL, err := setupKeepClient(config, *keepServicesJSON, *blobSignatureTTLFlag)
        if err != nil {
                return fmt.Errorf("Error configuring keepclient: %s", err.Error())
        }
 
-       return performKeepBlockCheck(kc, blobSigningKey, blockLocators, *verbose)
+       return performKeepBlockCheck(kc, blobSignatureTTL, blobSigningKey, blockLocators, *verbose)
 }
 
 type apiConfig struct {
@@ -138,7 +144,7 @@ func readConfigFromFile(filename string) (config apiConfig, blobSigningKey strin
 }
 
 // setup keepclient using the config provided
-func setupKeepClient(config apiConfig, keepServicesJSON string) (kc *keepclient.KeepClient, err error) {
+func setupKeepClient(config apiConfig, keepServicesJSON string, blobSignatureTTL time.Duration) (kc *keepclient.KeepClient, ttl time.Duration, err error) {
        arv := arvadosclient.ArvadosClient{
                ApiToken:    config.APIToken,
                ApiServer:   config.APIHost,
@@ -148,7 +154,7 @@ func setupKeepClient(config apiConfig, keepServicesJSON string) (kc *keepclient.
                External: config.ExternalClient,
        }
 
-       // if keepServicesJSON is provided, use it to load services; else, use DiscoverKeepServers
+       // If keepServicesJSON is provided, use it instead of service discovery
        if keepServicesJSON == "" {
                kc, err = keepclient.MakeKeepClient(&arv)
                if err != nil {
@@ -162,6 +168,17 @@ func setupKeepClient(config apiConfig, keepServicesJSON string) (kc *keepclient.
                }
        }
 
+       // Get if blobSignatureTTL is not provided
+       ttl = blobSignatureTTL
+       if blobSignatureTTL == 0 {
+               value, err := arv.Discovery("blobSignatureTtl")
+               if err == nil {
+                       ttl = time.Duration(int(value.(float64))) * time.Second
+               } else {
+                       return nil, 0, err
+               }
+       }
+
        return
 }
 
@@ -191,19 +208,19 @@ func getBlockLocators(locatorFile, prefix string) (locators []string, err error)
 }
 
 // Get block headers from keep. Log any errors.
-func performKeepBlockCheck(kc *keepclient.KeepClient, blobSigningKey string, blockLocators []string, verbose bool) error {
+func performKeepBlockCheck(kc *keepclient.KeepClient, blobSignatureTTL time.Duration, blobSigningKey string, blockLocators []string, verbose bool) error {
        totalBlocks := len(blockLocators)
        notFoundBlocks := 0
        current := 0
        for _, locator := range blockLocators {
                current++
                if verbose {
-                       log.Printf("Checking block %d of %d: %v", current, totalBlocks, locator)
+                       log.Printf("Verifying block %d of %d: %v", current, totalBlocks, locator)
                }
                getLocator := locator
                if blobSigningKey != "" {
                        expiresAt := time.Now().AddDate(0, 0, 1)
-                       getLocator = keepclient.SignLocator(locator, kc.Arvados.ApiToken, expiresAt, []byte(blobSigningKey))
+                       getLocator = keepclient.SignLocator(locator, kc.Arvados.ApiToken, expiresAt, blobSignatureTTL, []byte(blobSigningKey))
                }
 
                _, _, err := kc.Ask(getLocator)