8936: update keep-block-check and keep-rsync to properly use blob-signature-ttl to...

diff --git a/tools/keep-block-check/keep-block-check.go b/tools/keep-block-check/keep-block-check.go
index 569046b70f4c3969dbabde049ffcbed2044dd454..6cf11a728075c60990c1b049c0fb916cea5cd5d5 100644 (file)
--- a/tools/keep-block-check/keep-block-check.go
+++ b/tools/keep-block-check/keep-block-check.go
@@ -48,7 +48,7 @@ func doMain(args []string) error {
                "",
                "Block hash prefix. When a prefix is specified, only hashes listed in the file with this prefix will be checked.")
 
-       blobSignatureTTL := flags.Duration(
+       blobSignatureTTLFlag := flags.Duration(
                "blob-signature-ttl",
                0,
                "Lifetime of blob permission signatures on the keepservers. If not provided, this will be retrieved from the API server's discovery document.")
@@ -73,12 +73,12 @@ func doMain(args []string) error {
        }
 
        // setup keepclient
-       kc, err := setupKeepClient(config, *keepServicesJSON, *blobSignatureTTL)
+       kc, blobSignatureTTL, err := setupKeepClient(config, *keepServicesJSON, *blobSignatureTTLFlag)
        if err != nil {
                return fmt.Errorf("Error configuring keepclient: %s", err.Error())
        }
 
-       return performKeepBlockCheck(kc, *blobSignatureTTL, blobSigningKey, blockLocators, *verbose)
+       return performKeepBlockCheck(kc, blobSignatureTTL, blobSigningKey, blockLocators, *verbose)
 }
 
 type apiConfig struct {
@@ -143,7 +143,7 @@ func readConfigFromFile(filename string) (config apiConfig, blobSigningKey strin
 }
 
 // setup keepclient using the config provided
-func setupKeepClient(config apiConfig, keepServicesJSON string, blobSignatureTTL time.Duration) (kc *keepclient.KeepClient, err error) {
+func setupKeepClient(config apiConfig, keepServicesJSON string, blobSignatureTTL time.Duration) (kc *keepclient.KeepClient, ttl time.Duration, err error) {
        arv := arvadosclient.ArvadosClient{
                ApiToken:    config.APIToken,
                ApiServer:   config.APIHost,
@@ -168,12 +168,13 @@ func setupKeepClient(config apiConfig, keepServicesJSON string, blobSignatureTTL
        }
 
        // Get if blobSignatureTTL is not provided
+       ttl = blobSignatureTTL
        if blobSignatureTTL == 0 {
                value, err := arv.Discovery("blobSignatureTtl")
                if err == nil {
-                       blobSignatureTTL = time.Duration(int(value.(float64))) * time.Second
+                       ttl = time.Duration(int(value.(float64))) * time.Second
                } else {
-                       return nil, err
+                       return nil, 0, err
                }
        }
 

diff --git a/tools/keep-block-check/keep-block-check_test.go b/tools/keep-block-check/keep-block-check_test.go
index 6c776126ab3f6627130930c1f4cde4ec1032f256..e49fe68616626275f00ba5bce21877dd969207dd 100644 (file)
--- a/tools/keep-block-check/keep-block-check_test.go
+++ b/tools/keep-block-check/keep-block-check_test.go
@@ -72,6 +72,10 @@ func (s *DoMainTestSuite) TearDownTest(c *C) {
 }
 
 func setupKeepBlockCheck(c *C, enforcePermissions bool, keepServicesJSON string) {
+       setupKeepBlockCheckWithTTL(c, enforcePermissions, keepServicesJSON, blobSignatureTTL)
+}
+
+func setupKeepBlockCheckWithTTL(c *C, enforcePermissions bool, keepServicesJSON string, ttl time.Duration) {
        var config apiConfig
        config.APIHost = os.Getenv("ARVADOS_API_HOST")
        config.APIToken = arvadostest.DataManagerToken
@@ -82,7 +86,8 @@ func setupKeepBlockCheck(c *C, enforcePermissions bool, keepServicesJSON string)
 
        // setup keepclients
        var err error
-       kc, err = setupKeepClient(config, keepServicesJSON, blobSignatureTTL)
+       kc, ttl, err = setupKeepClient(config, keepServicesJSON, ttl)
+       c.Assert(ttl, Equals, blobSignatureTTL)
        c.Check(err, IsNil)
 }
 
@@ -167,6 +172,14 @@ func (s *ServerRequiredSuite) TestBlockCheckWithBlobSigning(c *C) {
        checkNoErrorsLogged(c, "Error verifying block", "Block not found")
 }
 
+func (s *ServerRequiredSuite) TestBlockCheckWithBlobSigningAndTTLFromDiscovery(c *C) {
+       setupKeepBlockCheckWithTTL(c, true, "", 0)
+       allLocators := setupTestData(c)
+       err := performKeepBlockCheck(kc, blobSignatureTTL, arvadostest.BlobSigningKey, allLocators, true)
+       c.Check(err, IsNil)
+       checkNoErrorsLogged(c, "Error verifying block", "Block not found")
+}
+
 func (s *ServerRequiredSuite) TestBlockCheck_NoSuchBlock(c *C) {
        setupKeepBlockCheck(c, false, "")
        allLocators := setupTestData(c)

diff --git a/tools/keep-rsync/keep-rsync.go b/tools/keep-rsync/keep-rsync.go
index 36f73204359736286f40aa9a753aa104428df7e0..c6e7665caa2a312c327b8a603159a7da07941450 100644 (file)
--- a/tools/keep-rsync/keep-rsync.go
+++ b/tools/keep-rsync/keep-rsync.go
@@ -60,7 +60,7 @@ func doMain() error {
                "",
                "Index prefix")
 
-       srcBlobSignatureTTL := flags.Duration(
+       srcBlobSignatureTTLFlag := flags.Duration(
                "src-blob-signature-ttl",
                0,
                "Lifetime of blob permission signatures on source keepservers. If not provided, this will be retrieved from the API server's discovery document.")
@@ -79,18 +79,18 @@ func doMain() error {
        }
 
        // setup src and dst keepclients
-       kcSrc, err := setupKeepClient(srcConfig, *srcKeepServicesJSON, false, 0, *srcBlobSignatureTTL)
+       kcSrc, srcBlobSignatureTTL, err := setupKeepClient(srcConfig, *srcKeepServicesJSON, false, 0, *srcBlobSignatureTTLFlag)
        if err != nil {
                return fmt.Errorf("Error configuring src keepclient: %s", err.Error())
        }
 
-       kcDst, err := setupKeepClient(dstConfig, *dstKeepServicesJSON, true, *replications, 0)
+       kcDst, _, err := setupKeepClient(dstConfig, *dstKeepServicesJSON, true, *replications, 0)
        if err != nil {
                return fmt.Errorf("Error configuring dst keepclient: %s", err.Error())
        }
 
        // Copy blocks not found in dst from src
-       err = performKeepRsync(kcSrc, kcDst, *srcBlobSignatureTTL, srcBlobSigningKey, *prefix)
+       err = performKeepRsync(kcSrc, kcDst, srcBlobSignatureTTL, srcBlobSigningKey, *prefix)
        if err != nil {
                return fmt.Errorf("Error while syncing data: %s", err.Error())
        }
@@ -160,7 +160,7 @@ func readConfigFromFile(filename string) (config apiConfig, blobSigningKey strin
 }
 
 // setup keepclient using the config provided
-func setupKeepClient(config apiConfig, keepServicesJSON string, isDst bool, replications int, srcBlobSignatureTTL time.Duration) (kc *keepclient.KeepClient, err error) {
+func setupKeepClient(config apiConfig, keepServicesJSON string, isDst bool, replications int, srcBlobSignatureTTL time.Duration) (kc *keepclient.KeepClient, blobSignatureTTL time.Duration, err error) {
        arv := arvadosclient.ArvadosClient{
                ApiToken:    config.APIToken,
                ApiServer:   config.APIHost,
@@ -174,13 +174,13 @@ func setupKeepClient(config apiConfig, keepServicesJSON string, isDst bool, repl
        if keepServicesJSON == "" {
                kc, err = keepclient.MakeKeepClient(&arv)
                if err != nil {
-                       return nil, err
+                       return nil, 0, err
                }
        } else {
                kc = keepclient.New(&arv)
                err = kc.LoadKeepServicesFromJSON(keepServicesJSON)
                if err != nil {
-                       return kc, err
+                       return kc, 0, err
                }
        }
 
@@ -191,7 +191,7 @@ func setupKeepClient(config apiConfig, keepServicesJSON string, isDst bool, repl
                        if err == nil {
                                replications = int(value.(float64))
                        } else {
-                               return nil, err
+                               return nil, 0, err
                        }
                }
 
@@ -199,16 +199,17 @@ func setupKeepClient(config apiConfig, keepServicesJSON string, isDst bool, repl
        }
 
        // If srcBlobSignatureTTL is not provided, get it from API server discovery doc
+       blobSignatureTTL = srcBlobSignatureTTL
        if !isDst && srcBlobSignatureTTL == 0 {
                value, err := arv.Discovery("blobSignatureTtl")
                if err == nil {
-                       srcBlobSignatureTTL = time.Duration(int(value.(float64))) * time.Second
+                       blobSignatureTTL = time.Duration(int(value.(float64))) * time.Second
                } else {
-                       return nil, err
+                       return nil, 0, err
                }
        }
 
-       return kc, nil
+       return kc, blobSignatureTTL, nil
 }
 
 // Get unique block locators from src and dst

diff --git a/tools/keep-rsync/keep-rsync_test.go b/tools/keep-rsync/keep-rsync_test.go
index f1f6a5f7ab9d6008c1d27642767708ac979ce568..bdae39a134d6e666accb8e37cb87c728e124b60a 100644 (file)
--- a/tools/keep-rsync/keep-rsync_test.go
+++ b/tools/keep-rsync/keep-rsync_test.go
@@ -100,10 +100,10 @@ func setupRsync(c *C, enforcePermissions bool, replications int) {
 
        // setup keepclients
        var err error
-       kcSrc, err = setupKeepClient(srcConfig, srcKeepServicesJSON, false, 0, blobSignatureTTL)
+       kcSrc, _, err = setupKeepClient(srcConfig, srcKeepServicesJSON, false, 0, blobSignatureTTL)
        c.Check(err, IsNil)
 
-       kcDst, err = setupKeepClient(dstConfig, dstKeepServicesJSON, true, replications, 0)
+       kcDst, _, err = setupKeepClient(dstConfig, dstKeepServicesJSON, true, replications, 0)
        c.Check(err, IsNil)
 
        for uuid := range kcSrc.LocalRoots() {