9005: Keep service discovery up to date automatically.

[arvados.git] / tools / keep-rsync / keep-rsync_test.go

diff --git a/tools/keep-rsync/keep-rsync_test.go b/tools/keep-rsync/keep-rsync_test.go
index 6fbb535a03a3069249b1ab22fa6aaae85740998f..6f128c69a1a6a1c52d16b7c5d95bd7560f37ab5d 100644 (file)
--- a/tools/keep-rsync/keep-rsync_test.go
+++ b/tools/keep-rsync/keep-rsync_test.go
@@ -4,6 +4,7 @@ import (
        "crypto/md5"
        "fmt"
        "io/ioutil"
+       "log"
        "os"
        "strings"
        "testing"
@@ -48,6 +49,7 @@ func (s *DoMainTestSuite) SetUpSuite(c *C) {
 
 var kcSrc, kcDst *keepclient.KeepClient
 var srcKeepServicesJSON, dstKeepServicesJSON, blobSigningKey string
+var blobSignatureTTL = time.Duration(2*7*24) * time.Hour
 
 func (s *ServerRequiredSuite) SetUpTest(c *C) {
        // reset all variables between tests
@@ -56,6 +58,7 @@ func (s *ServerRequiredSuite) SetUpTest(c *C) {
        dstKeepServicesJSON = ""
        kcSrc = &keepclient.KeepClient{}
        kcDst = &keepclient.KeepClient{}
+       keepclient.ClearCache()
 }
 
 func (s *ServerRequiredSuite) TearDownTest(c *C) {
@@ -63,6 +66,7 @@ func (s *ServerRequiredSuite) TearDownTest(c *C) {
 }
 
 func (s *DoMainTestSuite) SetUpTest(c *C) {
+       keepclient.ClearCache()
        args := []string{"keep-rsync"}
        os.Args = args
 }
@@ -80,17 +84,17 @@ func setupRsync(c *C, enforcePermissions bool, replications int) {
        // srcConfig
        var srcConfig apiConfig
        srcConfig.APIHost = os.Getenv("ARVADOS_API_HOST")
-       srcConfig.APIToken = os.Getenv("ARVADOS_API_TOKEN")
+       srcConfig.APIToken = arvadostest.DataManagerToken
        srcConfig.APIHostInsecure = matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))
 
        // dstConfig
        var dstConfig apiConfig
        dstConfig.APIHost = os.Getenv("ARVADOS_API_HOST")
-       dstConfig.APIToken = os.Getenv("ARVADOS_API_TOKEN")
+       dstConfig.APIToken = arvadostest.DataManagerToken
        dstConfig.APIHostInsecure = matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))
 
        if enforcePermissions {
-               blobSigningKey = "zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc"
+               blobSigningKey = arvadostest.BlobSigningKey
        }
 
        // Start Keep servers
@@ -98,10 +102,10 @@ func setupRsync(c *C, enforcePermissions bool, replications int) {
 
        // setup keepclients
        var err error
-       kcSrc, err = setupKeepClient(srcConfig, srcKeepServicesJSON, false, 0)
+       kcSrc, _, err = setupKeepClient(srcConfig, srcKeepServicesJSON, false, 0, blobSignatureTTL)
        c.Check(err, IsNil)
 
-       kcDst, err = setupKeepClient(dstConfig, dstKeepServicesJSON, true, replications)
+       kcDst, _, err = setupKeepClient(dstConfig, dstKeepServicesJSON, true, replications, 0)
        c.Check(err, IsNil)
 
        for uuid := range kcSrc.LocalRoots() {
@@ -173,7 +177,7 @@ func testNoCrosstalk(c *C, testData string, kc1, kc2 *keepclient.KeepClient) {
        c.Assert(err, Equals, nil)
 
        locator = strings.Split(locator, "+")[0]
-       _, _, _, err = kc2.Get(keepclient.SignLocator(locator, kc2.Arvados.ApiToken, time.Now().AddDate(0, 0, 1), []byte(blobSigningKey)))
+       _, _, _, err = kc2.Get(keepclient.SignLocator(locator, kc2.Arvados.ApiToken, time.Now().AddDate(0, 0, 1), blobSignatureTTL, []byte(blobSigningKey)))
        c.Assert(err, NotNil)
        c.Check(err.Error(), Equals, "Block not found")
 }
@@ -255,7 +259,7 @@ func testKeepRsync(c *C, enforcePermissions bool, prefix string) {
        // setupTestData
        setupTestData(c, prefix)
 
-       err := performKeepRsync(kcSrc, kcDst, blobSigningKey, prefix)
+       err := performKeepRsync(kcSrc, kcDst, blobSignatureTTL, blobSigningKey, prefix)
        c.Check(err, IsNil)
 
        // Now GetIndex from dst and verify that all 5 from src and the 2 extra blocks are found
@@ -326,8 +330,9 @@ func (s *ServerRequiredSuite) TestErrorDuringRsync_FakeSrcKeepservers(c *C) {
 
        setupRsync(c, false, 1)
 
-       err := performKeepRsync(kcSrc, kcDst, "", "")
-       c.Check(strings.HasSuffix(err.Error(), "no such host"), Equals, true)
+       err := performKeepRsync(kcSrc, kcDst, blobSignatureTTL, "", "")
+       log.Printf("Err = %v", err)
+       c.Check(strings.Contains(err.Error(), "no such host"), Equals, true)
 }
 
 // Setup rsync using dstKeepServicesJSON with fake keepservers.
@@ -337,8 +342,9 @@ func (s *ServerRequiredSuite) TestErrorDuringRsync_FakeDstKeepservers(c *C) {
 
        setupRsync(c, false, 1)
 
-       err := performKeepRsync(kcSrc, kcDst, "", "")
-       c.Check(strings.HasSuffix(err.Error(), "no such host"), Equals, true)
+       err := performKeepRsync(kcSrc, kcDst, blobSignatureTTL, "", "")
+       log.Printf("Err = %v", err)
+       c.Check(strings.Contains(err.Error(), "no such host"), Equals, true)
 }
 
 // Test rsync with signature error during Get from src.
@@ -351,8 +357,8 @@ func (s *ServerRequiredSuite) TestErrorDuringRsync_ErrorGettingBlockFromSrc(c *C
        // Change blob signing key to a fake key, so that Get from src fails
        blobSigningKey = "thisisfakeblobsigningkey"
 
-       err := performKeepRsync(kcSrc, kcDst, blobSigningKey, "")
-       c.Check(strings.HasSuffix(err.Error(), "Block not found"), Equals, true)
+       err := performKeepRsync(kcSrc, kcDst, blobSignatureTTL, blobSigningKey, "")
+       c.Check(strings.Contains(err.Error(), "HTTP 403 \"Forbidden\""), Equals, true)
 }
 
 // Test rsync with error during Put to src.
@@ -365,8 +371,8 @@ func (s *ServerRequiredSuite) TestErrorDuringRsync_ErrorPuttingBlockInDst(c *C)
        // Increase Want_replicas on dst to result in insufficient replicas error during Put
        kcDst.Want_replicas = 2
 
-       err := performKeepRsync(kcSrc, kcDst, blobSigningKey, "")
-       c.Check(strings.HasSuffix(err.Error(), "Could not write sufficient replicas"), Equals, true)
+       err := performKeepRsync(kcSrc, kcDst, blobSignatureTTL, blobSigningKey, "")
+       c.Check(strings.Contains(err.Error(), "Could not write sufficient replicas"), Equals, true)
 }
 
 // Test loadConfig func
@@ -386,7 +392,7 @@ func (s *ServerNotRequiredSuite) TestLoadConfig(c *C) {
        c.Check(err, IsNil)
 
        c.Assert(srcConfig.APIHost, Equals, os.Getenv("ARVADOS_API_HOST"))
-       c.Assert(srcConfig.APIToken, Equals, os.Getenv("ARVADOS_API_TOKEN"))
+       c.Assert(srcConfig.APIToken, Equals, arvadostest.DataManagerToken)
        c.Assert(srcConfig.APIHostInsecure, Equals, matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE")))
        c.Assert(srcConfig.ExternalClient, Equals, false)
 
@@ -394,7 +400,7 @@ func (s *ServerNotRequiredSuite) TestLoadConfig(c *C) {
        c.Check(err, IsNil)
 
        c.Assert(dstConfig.APIHost, Equals, os.Getenv("ARVADOS_API_HOST"))
-       c.Assert(dstConfig.APIToken, Equals, os.Getenv("ARVADOS_API_TOKEN"))
+       c.Assert(dstConfig.APIToken, Equals, arvadostest.DataManagerToken)
        c.Assert(dstConfig.APIHostInsecure, Equals, matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE")))
        c.Assert(dstConfig.ExternalClient, Equals, false)
 
@@ -410,7 +416,19 @@ func (s *ServerNotRequiredSuite) TestLoadConfig_MissingSrcConfig(c *C) {
 // Test loadConfig func - error reading config
 func (s *ServerNotRequiredSuite) TestLoadConfig_ErrorLoadingSrcConfig(c *C) {
        _, _, err := loadConfig("no-such-config-file")
-       c.Assert(strings.HasSuffix(err.Error(), "no such file or directory"), Equals, true)
+       c.Assert(strings.Contains(err.Error(), "no such file or directory"), Equals, true)
+}
+
+func (s *ServerNotRequiredSuite) TestSetupKeepClient_NoBlobSignatureTTL(c *C) {
+       var srcConfig apiConfig
+       srcConfig.APIHost = os.Getenv("ARVADOS_API_HOST")
+       srcConfig.APIToken = arvadostest.DataManagerToken
+       srcConfig.APIHostInsecure = matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))
+       arvadostest.StartKeep(2, false)
+
+       _, ttl, err := setupKeepClient(srcConfig, srcKeepServicesJSON, false, 0, 0)
+       c.Check(err, IsNil)
+       c.Assert(ttl, Equals, blobSignatureTTL)
 }
 
 func setupConfigFile(c *C, name string) *os.File {
@@ -419,7 +437,7 @@ func setupConfigFile(c *C, name string) *os.File {
        c.Check(err, IsNil)
 
        fileContent := "ARVADOS_API_HOST=" + os.Getenv("ARVADOS_API_HOST") + "\n"
-       fileContent += "ARVADOS_API_TOKEN=" + os.Getenv("ARVADOS_API_TOKEN") + "\n"
+       fileContent += "ARVADOS_API_TOKEN=" + arvadostest.DataManagerToken + "\n"
        fileContent += "ARVADOS_API_HOST_INSECURE=" + os.Getenv("ARVADOS_API_HOST_INSECURE") + "\n"
        fileContent += "ARVADOS_EXTERNAL_CLIENT=false\n"
        fileContent += "ARVADOS_BLOB_SIGNING_KEY=abcdefg"
@@ -467,9 +485,11 @@ func (s *DoMainTestSuite) Test_doMainWithSrcAndDstConfig(c *C) {
        args := []string{"-src", srcConfig.Name(), "-dst", dstConfig.Name()}
        os.Args = append(os.Args, args...)
 
-       // Start keepservers. Since we are not doing any tweaking as in setupRsync func,
-       // kcSrc and kcDst will be the same and no actual copying to dst will happen, but that's ok.
+       // Start keepservers. Since we are not doing any tweaking as
+       // in setupRsync func, kcSrc and kcDst will be the same and no
+       // actual copying to dst will happen, but that's ok.
        arvadostest.StartKeep(2, false)
+       defer arvadostest.StopKeep(2)
 
        err := doMain()
        c.Check(err, IsNil)