X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/845f7b9048d4b674a05e04fe919f2eb37479b1a0..7c7dc19ffa0a20fff6d97e51c874bfaca9596b24:/tools/keep-rsync/keep-rsync_test.go diff --git a/tools/keep-rsync/keep-rsync_test.go b/tools/keep-rsync/keep-rsync_test.go index bb9d4a6dfe..6f128c69a1 100644 --- a/tools/keep-rsync/keep-rsync_test.go +++ b/tools/keep-rsync/keep-rsync_test.go @@ -4,6 +4,7 @@ import ( "crypto/md5" "fmt" "io/ioutil" + "log" "os" "strings" "testing" @@ -23,10 +24,12 @@ func Test(t *testing.T) { // Gocheck boilerplate var _ = Suite(&ServerRequiredSuite{}) var _ = Suite(&ServerNotRequiredSuite{}) +var _ = Suite(&DoMainTestSuite{}) // Tests that require the Keep server running type ServerRequiredSuite struct{} type ServerNotRequiredSuite struct{} +type DoMainTestSuite struct{} func (s *ServerRequiredSuite) SetUpSuite(c *C) { // Start API server @@ -38,8 +41,15 @@ func (s *ServerRequiredSuite) TearDownSuite(c *C) { arvadostest.ResetEnv() } +var initialArgs []string + +func (s *DoMainTestSuite) SetUpSuite(c *C) { + initialArgs = os.Args +} + var kcSrc, kcDst *keepclient.KeepClient var srcKeepServicesJSON, dstKeepServicesJSON, blobSigningKey string +var blobSignatureTTL = time.Duration(2*7*24) * time.Hour func (s *ServerRequiredSuite) SetUpTest(c *C) { // reset all variables between tests @@ -48,10 +58,21 @@ func (s *ServerRequiredSuite) SetUpTest(c *C) { dstKeepServicesJSON = "" kcSrc = &keepclient.KeepClient{} kcDst = &keepclient.KeepClient{} + keepclient.ClearCache() } func (s *ServerRequiredSuite) TearDownTest(c *C) { - arvadostest.StopKeepServers(3) + arvadostest.StopKeep(3) +} + +func (s *DoMainTestSuite) SetUpTest(c *C) { + keepclient.ClearCache() + args := []string{"keep-rsync"} + os.Args = args +} + +func (s *DoMainTestSuite) TearDownTest(c *C) { + os.Args = initialArgs } var testKeepServicesJSON = "{ \"kind\":\"arvados#keepServiceList\", \"etag\":\"\", \"self_link\":\"\", \"offset\":null, \"limit\":null, \"items\":[ { \"href\":\"/keep_services/zzzzz-bi6l4-123456789012340\", \"kind\":\"arvados#keepService\", \"etag\":\"641234567890enhj7hzx432e5\", \"uuid\":\"zzzzz-bi6l4-123456789012340\", \"owner_uuid\":\"zzzzz-tpzed-123456789012345\", \"service_host\":\"keep0.zzzzz.arvadosapi.com\", \"service_port\":25107, \"service_ssl_flag\":false, \"service_type\":\"disk\", \"read_only\":false }, { \"href\":\"/keep_services/zzzzz-bi6l4-123456789012341\", \"kind\":\"arvados#keepService\", \"etag\":\"641234567890enhj7hzx432e5\", \"uuid\":\"zzzzz-bi6l4-123456789012341\", \"owner_uuid\":\"zzzzz-tpzed-123456789012345\", \"service_host\":\"keep0.zzzzz.arvadosapi.com\", \"service_port\":25108, \"service_ssl_flag\":false, \"service_type\":\"disk\", \"read_only\":false } ], \"items_available\":2 }" @@ -63,29 +84,28 @@ func setupRsync(c *C, enforcePermissions bool, replications int) { // srcConfig var srcConfig apiConfig srcConfig.APIHost = os.Getenv("ARVADOS_API_HOST") - srcConfig.APIToken = os.Getenv("ARVADOS_API_TOKEN") + srcConfig.APIToken = arvadostest.DataManagerToken srcConfig.APIHostInsecure = matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE")) // dstConfig var dstConfig apiConfig dstConfig.APIHost = os.Getenv("ARVADOS_API_HOST") - dstConfig.APIToken = os.Getenv("ARVADOS_API_TOKEN") + dstConfig.APIToken = arvadostest.DataManagerToken dstConfig.APIHostInsecure = matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE")) if enforcePermissions { - blobSigningKey = "zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc" + blobSigningKey = arvadostest.BlobSigningKey } // Start Keep servers - arvadostest.StartAPI() - arvadostest.StartKeepWithParams(3, enforcePermissions) + arvadostest.StartKeep(3, enforcePermissions) // setup keepclients var err error - kcSrc, err = setupKeepClient(srcConfig, srcKeepServicesJSON, false, 0) + kcSrc, _, err = setupKeepClient(srcConfig, srcKeepServicesJSON, false, 0, blobSignatureTTL) c.Check(err, IsNil) - kcDst, err = setupKeepClient(dstConfig, dstKeepServicesJSON, true, replications) + kcDst, _, err = setupKeepClient(dstConfig, dstKeepServicesJSON, true, replications, 0) c.Check(err, IsNil) for uuid := range kcSrc.LocalRoots() { @@ -157,7 +177,7 @@ func testNoCrosstalk(c *C, testData string, kc1, kc2 *keepclient.KeepClient) { c.Assert(err, Equals, nil) locator = strings.Split(locator, "+")[0] - _, _, _, err = kc2.Get(keepclient.SignLocator(locator, kc2.Arvados.ApiToken, time.Now().AddDate(0, 0, 1), []byte(blobSigningKey))) + _, _, _, err = kc2.Get(keepclient.SignLocator(locator, kc2.Arvados.ApiToken, time.Now().AddDate(0, 0, 1), blobSignatureTTL, []byte(blobSigningKey))) c.Assert(err, NotNil) c.Check(err.Error(), Equals, "Block not found") } @@ -239,7 +259,7 @@ func testKeepRsync(c *C, enforcePermissions bool, prefix string) { // setupTestData setupTestData(c, prefix) - err := performKeepRsync(kcSrc, kcDst, blobSigningKey, prefix) + err := performKeepRsync(kcSrc, kcDst, blobSignatureTTL, blobSigningKey, prefix) c.Check(err, IsNil) // Now GetIndex from dst and verify that all 5 from src and the 2 extra blocks are found @@ -310,8 +330,9 @@ func (s *ServerRequiredSuite) TestErrorDuringRsync_FakeSrcKeepservers(c *C) { setupRsync(c, false, 1) - err := performKeepRsync(kcSrc, kcDst, "", "") - c.Check(strings.HasSuffix(err.Error(), "no such host"), Equals, true) + err := performKeepRsync(kcSrc, kcDst, blobSignatureTTL, "", "") + log.Printf("Err = %v", err) + c.Check(strings.Contains(err.Error(), "no such host"), Equals, true) } // Setup rsync using dstKeepServicesJSON with fake keepservers. @@ -321,8 +342,9 @@ func (s *ServerRequiredSuite) TestErrorDuringRsync_FakeDstKeepservers(c *C) { setupRsync(c, false, 1) - err := performKeepRsync(kcSrc, kcDst, "", "") - c.Check(strings.HasSuffix(err.Error(), "no such host"), Equals, true) + err := performKeepRsync(kcSrc, kcDst, blobSignatureTTL, "", "") + log.Printf("Err = %v", err) + c.Check(strings.Contains(err.Error(), "no such host"), Equals, true) } // Test rsync with signature error during Get from src. @@ -335,8 +357,8 @@ func (s *ServerRequiredSuite) TestErrorDuringRsync_ErrorGettingBlockFromSrc(c *C // Change blob signing key to a fake key, so that Get from src fails blobSigningKey = "thisisfakeblobsigningkey" - err := performKeepRsync(kcSrc, kcDst, blobSigningKey, "") - c.Check(strings.HasSuffix(err.Error(), "Block not found"), Equals, true) + err := performKeepRsync(kcSrc, kcDst, blobSignatureTTL, blobSigningKey, "") + c.Check(strings.Contains(err.Error(), "HTTP 403 \"Forbidden\""), Equals, true) } // Test rsync with error during Put to src. @@ -349,8 +371,8 @@ func (s *ServerRequiredSuite) TestErrorDuringRsync_ErrorPuttingBlockInDst(c *C) // Increase Want_replicas on dst to result in insufficient replicas error during Put kcDst.Want_replicas = 2 - err := performKeepRsync(kcSrc, kcDst, blobSigningKey, "") - c.Check(strings.HasSuffix(err.Error(), "Could not write sufficient replicas"), Equals, true) + err := performKeepRsync(kcSrc, kcDst, blobSignatureTTL, blobSigningKey, "") + c.Check(strings.Contains(err.Error(), "Could not write sufficient replicas"), Equals, true) } // Test loadConfig func @@ -369,17 +391,17 @@ func (s *ServerNotRequiredSuite) TestLoadConfig(c *C) { srcConfig, srcBlobSigningKey, err := loadConfig(srcConfigFile) c.Check(err, IsNil) - c.Assert(srcConfig.APIHost, Equals, "testhost") - c.Assert(srcConfig.APIToken, Equals, "testtoken") - c.Assert(srcConfig.APIHostInsecure, Equals, true) + c.Assert(srcConfig.APIHost, Equals, os.Getenv("ARVADOS_API_HOST")) + c.Assert(srcConfig.APIToken, Equals, arvadostest.DataManagerToken) + c.Assert(srcConfig.APIHostInsecure, Equals, matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))) c.Assert(srcConfig.ExternalClient, Equals, false) dstConfig, _, err := loadConfig(dstConfigFile) c.Check(err, IsNil) - c.Assert(dstConfig.APIHost, Equals, "testhost") - c.Assert(dstConfig.APIToken, Equals, "testtoken") - c.Assert(dstConfig.APIHostInsecure, Equals, true) + c.Assert(dstConfig.APIHost, Equals, os.Getenv("ARVADOS_API_HOST")) + c.Assert(dstConfig.APIToken, Equals, arvadostest.DataManagerToken) + c.Assert(dstConfig.APIHostInsecure, Equals, matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))) c.Assert(dstConfig.ExternalClient, Equals, false) c.Assert(srcBlobSigningKey, Equals, "abcdefg") @@ -394,7 +416,19 @@ func (s *ServerNotRequiredSuite) TestLoadConfig_MissingSrcConfig(c *C) { // Test loadConfig func - error reading config func (s *ServerNotRequiredSuite) TestLoadConfig_ErrorLoadingSrcConfig(c *C) { _, _, err := loadConfig("no-such-config-file") - c.Assert(strings.HasSuffix(err.Error(), "no such file or directory"), Equals, true) + c.Assert(strings.Contains(err.Error(), "no such file or directory"), Equals, true) +} + +func (s *ServerNotRequiredSuite) TestSetupKeepClient_NoBlobSignatureTTL(c *C) { + var srcConfig apiConfig + srcConfig.APIHost = os.Getenv("ARVADOS_API_HOST") + srcConfig.APIToken = arvadostest.DataManagerToken + srcConfig.APIHostInsecure = matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE")) + arvadostest.StartKeep(2, false) + + _, ttl, err := setupKeepClient(srcConfig, srcKeepServicesJSON, false, 0, 0) + c.Check(err, IsNil) + c.Assert(ttl, Equals, blobSignatureTTL) } func setupConfigFile(c *C, name string) *os.File { @@ -402,9 +436,10 @@ func setupConfigFile(c *C, name string) *os.File { file, err := ioutil.TempFile(os.TempDir(), name) c.Check(err, IsNil) - fileContent := "ARVADOS_API_HOST=testhost\n" - fileContent += "ARVADOS_API_TOKEN=testtoken\n" - fileContent += "ARVADOS_API_HOST_INSECURE=true\n" + fileContent := "ARVADOS_API_HOST=" + os.Getenv("ARVADOS_API_HOST") + "\n" + fileContent += "ARVADOS_API_TOKEN=" + arvadostest.DataManagerToken + "\n" + fileContent += "ARVADOS_API_HOST_INSECURE=" + os.Getenv("ARVADOS_API_HOST_INSECURE") + "\n" + fileContent += "ARVADOS_EXTERNAL_CLIENT=false\n" fileContent += "ARVADOS_BLOB_SIGNING_KEY=abcdefg" _, err = file.Write([]byte(fileContent)) @@ -412,3 +447,50 @@ func setupConfigFile(c *C, name string) *os.File { return file } + +func (s *DoMainTestSuite) Test_doMain_NoSrcConfig(c *C) { + err := doMain() + c.Check(err, NotNil) + c.Assert(err.Error(), Equals, "Error loading src configuration from file: config file not specified") +} + +func (s *DoMainTestSuite) Test_doMain_SrcButNoDstConfig(c *C) { + srcConfig := setupConfigFile(c, "src") + args := []string{"-replications", "3", "-src", srcConfig.Name()} + os.Args = append(os.Args, args...) + err := doMain() + c.Check(err, NotNil) + c.Assert(err.Error(), Equals, "Error loading dst configuration from file: config file not specified") +} + +func (s *DoMainTestSuite) Test_doMain_BadSrcConfig(c *C) { + args := []string{"-src", "abcd"} + os.Args = append(os.Args, args...) + err := doMain() + c.Check(err, NotNil) + c.Assert(strings.HasPrefix(err.Error(), "Error loading src configuration from file: Error reading config file"), Equals, true) +} + +func (s *DoMainTestSuite) Test_doMain_WithReplicationsButNoSrcConfig(c *C) { + args := []string{"-replications", "3"} + os.Args = append(os.Args, args...) + err := doMain() + c.Check(err, NotNil) + c.Assert(err.Error(), Equals, "Error loading src configuration from file: config file not specified") +} + +func (s *DoMainTestSuite) Test_doMainWithSrcAndDstConfig(c *C) { + srcConfig := setupConfigFile(c, "src") + dstConfig := setupConfigFile(c, "dst") + args := []string{"-src", srcConfig.Name(), "-dst", dstConfig.Name()} + os.Args = append(os.Args, args...) + + // Start keepservers. Since we are not doing any tweaking as + // in setupRsync func, kcSrc and kcDst will be the same and no + // actual copying to dst will happen, but that's ok. + arvadostest.StartKeep(2, false) + defer arvadostest.StopKeep(2) + + err := doMain() + c.Check(err, IsNil) +}