From f5617be935a121dc339effcaafebb569109ebe5b Mon Sep 17 00:00:00 2001 From: radhika Date: Fri, 22 Apr 2016 18:51:17 -0400 Subject: [PATCH] 8936: update keep-block-check and keep-rsync to properly use blob-signature-ttl to perform the signature verification. --- tools/keep-block-check/keep-block-check.go | 13 ++++++----- .../keep-block-check/keep-block-check_test.go | 15 +++++++++++- tools/keep-rsync/keep-rsync.go | 23 ++++++++++--------- tools/keep-rsync/keep-rsync_test.go | 4 ++-- 4 files changed, 35 insertions(+), 20 deletions(-) diff --git a/tools/keep-block-check/keep-block-check.go b/tools/keep-block-check/keep-block-check.go index 569046b70f..6cf11a7280 100644 --- a/tools/keep-block-check/keep-block-check.go +++ b/tools/keep-block-check/keep-block-check.go @@ -48,7 +48,7 @@ func doMain(args []string) error { "", "Block hash prefix. When a prefix is specified, only hashes listed in the file with this prefix will be checked.") - blobSignatureTTL := flags.Duration( + blobSignatureTTLFlag := flags.Duration( "blob-signature-ttl", 0, "Lifetime of blob permission signatures on the keepservers. If not provided, this will be retrieved from the API server's discovery document.") @@ -73,12 +73,12 @@ func doMain(args []string) error { } // setup keepclient - kc, err := setupKeepClient(config, *keepServicesJSON, *blobSignatureTTL) + kc, blobSignatureTTL, err := setupKeepClient(config, *keepServicesJSON, *blobSignatureTTLFlag) if err != nil { return fmt.Errorf("Error configuring keepclient: %s", err.Error()) } - return performKeepBlockCheck(kc, *blobSignatureTTL, blobSigningKey, blockLocators, *verbose) + return performKeepBlockCheck(kc, blobSignatureTTL, blobSigningKey, blockLocators, *verbose) } type apiConfig struct { @@ -143,7 +143,7 @@ func readConfigFromFile(filename string) (config apiConfig, blobSigningKey strin } // setup keepclient using the config provided -func setupKeepClient(config apiConfig, keepServicesJSON string, blobSignatureTTL time.Duration) (kc *keepclient.KeepClient, err error) { +func setupKeepClient(config apiConfig, keepServicesJSON string, blobSignatureTTL time.Duration) (kc *keepclient.KeepClient, ttl time.Duration, err error) { arv := arvadosclient.ArvadosClient{ ApiToken: config.APIToken, ApiServer: config.APIHost, @@ -168,12 +168,13 @@ func setupKeepClient(config apiConfig, keepServicesJSON string, blobSignatureTTL } // Get if blobSignatureTTL is not provided + ttl = blobSignatureTTL if blobSignatureTTL == 0 { value, err := arv.Discovery("blobSignatureTtl") if err == nil { - blobSignatureTTL = time.Duration(int(value.(float64))) * time.Second + ttl = time.Duration(int(value.(float64))) * time.Second } else { - return nil, err + return nil, 0, err } } diff --git a/tools/keep-block-check/keep-block-check_test.go b/tools/keep-block-check/keep-block-check_test.go index 6c776126ab..e49fe68616 100644 --- a/tools/keep-block-check/keep-block-check_test.go +++ b/tools/keep-block-check/keep-block-check_test.go @@ -72,6 +72,10 @@ func (s *DoMainTestSuite) TearDownTest(c *C) { } func setupKeepBlockCheck(c *C, enforcePermissions bool, keepServicesJSON string) { + setupKeepBlockCheckWithTTL(c, enforcePermissions, keepServicesJSON, blobSignatureTTL) +} + +func setupKeepBlockCheckWithTTL(c *C, enforcePermissions bool, keepServicesJSON string, ttl time.Duration) { var config apiConfig config.APIHost = os.Getenv("ARVADOS_API_HOST") config.APIToken = arvadostest.DataManagerToken @@ -82,7 +86,8 @@ func setupKeepBlockCheck(c *C, enforcePermissions bool, keepServicesJSON string) // setup keepclients var err error - kc, err = setupKeepClient(config, keepServicesJSON, blobSignatureTTL) + kc, ttl, err = setupKeepClient(config, keepServicesJSON, ttl) + c.Assert(ttl, Equals, blobSignatureTTL) c.Check(err, IsNil) } @@ -167,6 +172,14 @@ func (s *ServerRequiredSuite) TestBlockCheckWithBlobSigning(c *C) { checkNoErrorsLogged(c, "Error verifying block", "Block not found") } +func (s *ServerRequiredSuite) TestBlockCheckWithBlobSigningAndTTLFromDiscovery(c *C) { + setupKeepBlockCheckWithTTL(c, true, "", 0) + allLocators := setupTestData(c) + err := performKeepBlockCheck(kc, blobSignatureTTL, arvadostest.BlobSigningKey, allLocators, true) + c.Check(err, IsNil) + checkNoErrorsLogged(c, "Error verifying block", "Block not found") +} + func (s *ServerRequiredSuite) TestBlockCheck_NoSuchBlock(c *C) { setupKeepBlockCheck(c, false, "") allLocators := setupTestData(c) diff --git a/tools/keep-rsync/keep-rsync.go b/tools/keep-rsync/keep-rsync.go index 36f7320435..c6e7665caa 100644 --- a/tools/keep-rsync/keep-rsync.go +++ b/tools/keep-rsync/keep-rsync.go @@ -60,7 +60,7 @@ func doMain() error { "", "Index prefix") - srcBlobSignatureTTL := flags.Duration( + srcBlobSignatureTTLFlag := flags.Duration( "src-blob-signature-ttl", 0, "Lifetime of blob permission signatures on source keepservers. If not provided, this will be retrieved from the API server's discovery document.") @@ -79,18 +79,18 @@ func doMain() error { } // setup src and dst keepclients - kcSrc, err := setupKeepClient(srcConfig, *srcKeepServicesJSON, false, 0, *srcBlobSignatureTTL) + kcSrc, srcBlobSignatureTTL, err := setupKeepClient(srcConfig, *srcKeepServicesJSON, false, 0, *srcBlobSignatureTTLFlag) if err != nil { return fmt.Errorf("Error configuring src keepclient: %s", err.Error()) } - kcDst, err := setupKeepClient(dstConfig, *dstKeepServicesJSON, true, *replications, 0) + kcDst, _, err := setupKeepClient(dstConfig, *dstKeepServicesJSON, true, *replications, 0) if err != nil { return fmt.Errorf("Error configuring dst keepclient: %s", err.Error()) } // Copy blocks not found in dst from src - err = performKeepRsync(kcSrc, kcDst, *srcBlobSignatureTTL, srcBlobSigningKey, *prefix) + err = performKeepRsync(kcSrc, kcDst, srcBlobSignatureTTL, srcBlobSigningKey, *prefix) if err != nil { return fmt.Errorf("Error while syncing data: %s", err.Error()) } @@ -160,7 +160,7 @@ func readConfigFromFile(filename string) (config apiConfig, blobSigningKey strin } // setup keepclient using the config provided -func setupKeepClient(config apiConfig, keepServicesJSON string, isDst bool, replications int, srcBlobSignatureTTL time.Duration) (kc *keepclient.KeepClient, err error) { +func setupKeepClient(config apiConfig, keepServicesJSON string, isDst bool, replications int, srcBlobSignatureTTL time.Duration) (kc *keepclient.KeepClient, blobSignatureTTL time.Duration, err error) { arv := arvadosclient.ArvadosClient{ ApiToken: config.APIToken, ApiServer: config.APIHost, @@ -174,13 +174,13 @@ func setupKeepClient(config apiConfig, keepServicesJSON string, isDst bool, repl if keepServicesJSON == "" { kc, err = keepclient.MakeKeepClient(&arv) if err != nil { - return nil, err + return nil, 0, err } } else { kc = keepclient.New(&arv) err = kc.LoadKeepServicesFromJSON(keepServicesJSON) if err != nil { - return kc, err + return kc, 0, err } } @@ -191,7 +191,7 @@ func setupKeepClient(config apiConfig, keepServicesJSON string, isDst bool, repl if err == nil { replications = int(value.(float64)) } else { - return nil, err + return nil, 0, err } } @@ -199,16 +199,17 @@ func setupKeepClient(config apiConfig, keepServicesJSON string, isDst bool, repl } // If srcBlobSignatureTTL is not provided, get it from API server discovery doc + blobSignatureTTL = srcBlobSignatureTTL if !isDst && srcBlobSignatureTTL == 0 { value, err := arv.Discovery("blobSignatureTtl") if err == nil { - srcBlobSignatureTTL = time.Duration(int(value.(float64))) * time.Second + blobSignatureTTL = time.Duration(int(value.(float64))) * time.Second } else { - return nil, err + return nil, 0, err } } - return kc, nil + return kc, blobSignatureTTL, nil } // Get unique block locators from src and dst diff --git a/tools/keep-rsync/keep-rsync_test.go b/tools/keep-rsync/keep-rsync_test.go index f1f6a5f7ab..bdae39a134 100644 --- a/tools/keep-rsync/keep-rsync_test.go +++ b/tools/keep-rsync/keep-rsync_test.go @@ -100,10 +100,10 @@ func setupRsync(c *C, enforcePermissions bool, replications int) { // setup keepclients var err error - kcSrc, err = setupKeepClient(srcConfig, srcKeepServicesJSON, false, 0, blobSignatureTTL) + kcSrc, _, err = setupKeepClient(srcConfig, srcKeepServicesJSON, false, 0, blobSignatureTTL) c.Check(err, IsNil) - kcDst, err = setupKeepClient(dstConfig, dstKeepServicesJSON, true, replications, 0) + kcDst, _, err = setupKeepClient(dstConfig, dstKeepServicesJSON, true, replications, 0) c.Check(err, IsNil) for uuid := range kcSrc.LocalRoots() { -- 2.30.2