X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/ef42f0396f5b51fb8f87b2f7e605d50f32f256b4..55b017847fe87e00e7fb9c6bfd0444b83f8ca12c:/tools/keep-rsync/keep-rsync_test.go diff --git a/tools/keep-rsync/keep-rsync_test.go b/tools/keep-rsync/keep-rsync_test.go index 9a6480e864..afc27f513a 100644 --- a/tools/keep-rsync/keep-rsync_test.go +++ b/tools/keep-rsync/keep-rsync_test.go @@ -4,6 +4,7 @@ import ( "crypto/md5" "fmt" "io/ioutil" + "log" "os" "strings" "testing" @@ -22,19 +23,34 @@ func Test(t *testing.T) { // Gocheck boilerplate var _ = Suite(&ServerRequiredSuite{}) +var _ = Suite(&ServerNotRequiredSuite{}) +var _ = Suite(&DoMainTestSuite{}) // Tests that require the Keep server running type ServerRequiredSuite struct{} +type ServerNotRequiredSuite struct{} +type DoMainTestSuite struct{} func (s *ServerRequiredSuite) SetUpSuite(c *C) { + // Start API server + arvadostest.StartAPI() +} + +func (s *ServerRequiredSuite) TearDownSuite(c *C) { + arvadostest.StopAPI() + arvadostest.ResetEnv() +} + +var initialArgs []string + +func (s *DoMainTestSuite) SetUpSuite(c *C) { + initialArgs = os.Args } var kcSrc, kcDst *keepclient.KeepClient var srcKeepServicesJSON, dstKeepServicesJSON, blobSigningKey string func (s *ServerRequiredSuite) SetUpTest(c *C) { - arvadostest.ResetEnv() - // reset all variables between tests blobSigningKey = "" srcKeepServicesJSON = "" @@ -43,9 +59,17 @@ func (s *ServerRequiredSuite) SetUpTest(c *C) { kcDst = &keepclient.KeepClient{} } -func (s *ServerRequiredSuite) TearDownSuite(c *C) { - arvadostest.StopKeepServers(3) - arvadostest.StopAPI() +func (s *ServerRequiredSuite) TearDownTest(c *C) { + arvadostest.StopKeep(3) +} + +func (s *DoMainTestSuite) SetUpTest(c *C) { + args := []string{"keep-rsync"} + os.Args = args +} + +func (s *DoMainTestSuite) TearDownTest(c *C) { + os.Args = initialArgs } var testKeepServicesJSON = "{ \"kind\":\"arvados#keepServiceList\", \"etag\":\"\", \"self_link\":\"\", \"offset\":null, \"limit\":null, \"items\":[ { \"href\":\"/keep_services/zzzzz-bi6l4-123456789012340\", \"kind\":\"arvados#keepService\", \"etag\":\"641234567890enhj7hzx432e5\", \"uuid\":\"zzzzz-bi6l4-123456789012340\", \"owner_uuid\":\"zzzzz-tpzed-123456789012345\", \"service_host\":\"keep0.zzzzz.arvadosapi.com\", \"service_port\":25107, \"service_ssl_flag\":false, \"service_type\":\"disk\", \"read_only\":false }, { \"href\":\"/keep_services/zzzzz-bi6l4-123456789012341\", \"kind\":\"arvados#keepService\", \"etag\":\"641234567890enhj7hzx432e5\", \"uuid\":\"zzzzz-bi6l4-123456789012341\", \"owner_uuid\":\"zzzzz-tpzed-123456789012345\", \"service_host\":\"keep0.zzzzz.arvadosapi.com\", \"service_port\":25108, \"service_ssl_flag\":false, \"service_type\":\"disk\", \"read_only\":false } ], \"items_available\":2 }" @@ -53,26 +77,25 @@ var testKeepServicesJSON = "{ \"kind\":\"arvados#keepServiceList\", \"etag\":\"\ // Testing keep-rsync needs two sets of keep services: src and dst. // The test setup hence creates 3 servers instead of the default 2, // and uses the first 2 as src and the 3rd as dst keep servers. -func setupRsync(c *C, enforcePermissions, updateDstReplications bool, replications int) { +func setupRsync(c *C, enforcePermissions bool, replications int) { // srcConfig var srcConfig apiConfig srcConfig.APIHost = os.Getenv("ARVADOS_API_HOST") - srcConfig.APIToken = os.Getenv("ARVADOS_API_TOKEN") + srcConfig.APIToken = arvadostest.DataManagerToken srcConfig.APIHostInsecure = matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE")) // dstConfig var dstConfig apiConfig dstConfig.APIHost = os.Getenv("ARVADOS_API_HOST") - dstConfig.APIToken = os.Getenv("ARVADOS_API_TOKEN") + dstConfig.APIToken = arvadostest.DataManagerToken dstConfig.APIHostInsecure = matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE")) if enforcePermissions { - blobSigningKey = "zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc" + blobSigningKey = arvadostest.BlobSigningKey } - // Start API and Keep servers - arvadostest.StartAPI() - arvadostest.StartKeepWithParams(3, enforcePermissions) + // Start Keep servers + arvadostest.StartKeep(3, enforcePermissions) // setup keepclients var err error @@ -114,63 +137,53 @@ func setupRsync(c *C, enforcePermissions, updateDstReplications bool, replicatio } } - if updateDstReplications { - kcDst.Want_replicas = replications + if replications == 0 { + // Must have got default replications value of 2 from dst discovery document + c.Assert(kcDst.Want_replicas, Equals, 2) + } else { + // Since replications value is provided, it is used + c.Assert(kcDst.Want_replicas, Equals, replications) } } -// Test keep-rsync initialization, with src and dst keep servers. -// Do a Put and Get in src, both of which should succeed. -// Do a Put and Get in dst, both of which should succeed. -// Do a Get in dst for the src hash, which should raise block not found error. -// Do a Get in src for the dst hash, which should raise block not found error. func (s *ServerRequiredSuite) TestRsyncPutInOne_GetFromOtherShouldFail(c *C) { - setupRsync(c, false, true, 1) - - // Put a block in src using kcSrc and Get it - srcData := []byte("test-data1") - locatorInSrc := fmt.Sprintf("%x", md5.Sum(srcData)) + setupRsync(c, false, 1) - hash, rep, err := kcSrc.PutB(srcData) - c.Check(hash, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, locatorInSrc)) - c.Check(rep, Equals, 2) - c.Check(err, Equals, nil) + // Put a block in src and verify that it is not found in dst + testNoCrosstalk(c, "test-data-1", kcSrc, kcDst) - reader, blocklen, _, err := kcSrc.Get(locatorInSrc) - c.Check(err, IsNil) - c.Check(blocklen, Equals, int64(10)) - all, err := ioutil.ReadAll(reader) - c.Check(all, DeepEquals, srcData) + // Put a block in dst and verify that it is not found in src + testNoCrosstalk(c, "test-data-2", kcDst, kcSrc) +} - // Put a different block in src using kcSrc and Get it - dstData := []byte("test-data2") - locatorInDst := fmt.Sprintf("%x", md5.Sum(dstData)) +func (s *ServerRequiredSuite) TestRsyncWithBlobSigning_PutInOne_GetFromOtherShouldFail(c *C) { + setupRsync(c, true, 1) - hash, rep, err = kcDst.PutB(dstData) - c.Check(hash, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, locatorInDst)) - c.Check(rep, Equals, 1) - c.Check(err, Equals, nil) + // Put a block in src and verify that it is not found in dst + testNoCrosstalk(c, "test-data-1", kcSrc, kcDst) - reader, blocklen, _, err = kcDst.Get(locatorInDst) - c.Check(err, IsNil) - c.Check(blocklen, Equals, int64(10)) - all, err = ioutil.ReadAll(reader) - c.Check(all, DeepEquals, dstData) - - // Get srcLocator using kcDst should fail with Not Found error - _, _, _, err = kcDst.Get(locatorInSrc) - c.Assert(err.Error(), Equals, "Block not found") + // Put a block in dst and verify that it is not found in src + testNoCrosstalk(c, "test-data-2", kcDst, kcSrc) +} - // Get dstLocator using kcSrc should fail with Not Found error - _, _, _, err = kcSrc.Get(locatorInDst) - c.Assert(err.Error(), Equals, "Block not found") +// Do a Put in the first and Get from the second, +// which should raise block not found error. +func testNoCrosstalk(c *C, testData string, kc1, kc2 *keepclient.KeepClient) { + // Put a block using kc1 + locator, _, err := kc1.PutB([]byte(testData)) + c.Assert(err, Equals, nil) + + locator = strings.Split(locator, "+")[0] + _, _, _, err = kc2.Get(keepclient.SignLocator(locator, kc2.Arvados.ApiToken, time.Now().AddDate(0, 0, 1), []byte(blobSigningKey))) + c.Assert(err, NotNil) + c.Check(err.Error(), Equals, "Block not found") } // Test keep-rsync initialization, with srcKeepServicesJSON func (s *ServerRequiredSuite) TestRsyncInitializeWithKeepServicesJSON(c *C) { srcKeepServicesJSON = testKeepServicesJSON - setupRsync(c, false, true, 1) + setupRsync(c, false, 1) localRoots := kcSrc.LocalRoots() c.Check(localRoots, NotNil) @@ -192,74 +205,14 @@ func (s *ServerRequiredSuite) TestRsyncInitializeWithKeepServicesJSON(c *C) { c.Check(foundIt, Equals, true) } -// Test keep-rsync initialization, with src and dst keep servers with blobSigningKey. -// Do a Put and Get in src, both of which should succeed. -// Do a Put and Get in dst, both of which should succeed. -// Do a Get in dst for the src hash, which should raise block not found error. -// Do a Get in src for the dst hash, which should raise block not found error. -func (s *ServerRequiredSuite) TestRsyncWithBlobSigning_PutInOne_GetFromOtherShouldFail(c *C) { - setupRsync(c, true, true, 1) - - // Put a block in src using kcSrc and Get it - srcData := []byte("test-data1") - locatorInSrc := fmt.Sprintf("%x", md5.Sum(srcData)) - - hash, rep, err := kcSrc.PutB(srcData) - c.Check(hash, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, locatorInSrc)) - c.Check(rep, Equals, 2) - c.Check(err, Equals, nil) - - tomorrow := time.Now().AddDate(0, 0, 1) - signedLocator := keepclient.SignLocator(locatorInSrc, kcSrc.Arvados.ApiToken, tomorrow, []byte(blobSigningKey)) - - reader, blocklen, _, err := kcSrc.Get(signedLocator) - c.Check(err, IsNil) - c.Check(blocklen, Equals, int64(10)) - all, err := ioutil.ReadAll(reader) - c.Check(all, DeepEquals, srcData) - - // Put a different block in src using kcSrc and Get it - dstData := []byte("test-data2") - locatorInDst := fmt.Sprintf("%x", md5.Sum(dstData)) - - hash, rep, err = kcDst.PutB(dstData) - c.Check(hash, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, locatorInDst)) - c.Check(rep, Equals, 1) - c.Check(err, Equals, nil) - - signedLocator = keepclient.SignLocator(locatorInDst, kcDst.Arvados.ApiToken, tomorrow, []byte(blobSigningKey)) - - reader, blocklen, _, err = kcDst.Get(signedLocator) - c.Check(err, IsNil) - c.Check(blocklen, Equals, int64(10)) - all, err = ioutil.ReadAll(reader) - c.Check(all, DeepEquals, dstData) - - // Get srcLocator using kcDst should fail with Not Found error - signedLocator = keepclient.SignLocator(locatorInSrc, kcDst.Arvados.ApiToken, tomorrow, []byte(blobSigningKey)) - _, _, _, err = kcDst.Get(locatorInSrc) - c.Assert(err.Error(), Equals, "Block not found") - - // Get dstLocator using kcSrc should fail with Not Found error - signedLocator = keepclient.SignLocator(locatorInDst, kcSrc.Arvados.ApiToken, tomorrow, []byte(blobSigningKey)) - _, _, _, err = kcSrc.Get(locatorInDst) - c.Assert(err.Error(), Equals, "Block not found") -} - // Test keep-rsync initialization with default replications count func (s *ServerRequiredSuite) TestInitializeRsyncDefaultReplicationsCount(c *C) { - setupRsync(c, false, false, 0) - - // Must have got default replications value as 2 from dst discovery document - c.Assert(kcDst.Want_replicas, Equals, 2) + setupRsync(c, false, 0) } // Test keep-rsync initialization with replications count argument func (s *ServerRequiredSuite) TestInitializeRsyncReplicationsCount(c *C) { - setupRsync(c, false, false, 3) - - // Since replications value is provided, default is not used - c.Assert(kcDst.Want_replicas, Equals, 3) + setupRsync(c, false, 3) } // Put some blocks in Src and some more in Dst @@ -282,6 +235,7 @@ func (s *ServerRequiredSuite) TestKeepRsync_WithPrefix(c *C) { hash := fmt.Sprintf("%x", md5.Sum(data)) testKeepRsync(c, false, hash[0:3]) + c.Check(len(dstIndex) > len(dstLocators), Equals, true) } // Put some blocks in Src and some more in Dst @@ -289,6 +243,7 @@ func (s *ServerRequiredSuite) TestKeepRsync_WithPrefix(c *C) { // And copy missing blocks from Src to Dst func (s *ServerRequiredSuite) TestKeepRsync_WithNoSuchPrefixInSrc(c *C) { testKeepRsync(c, false, "999") + c.Check(len(dstIndex), Equals, len(dstLocators)) } // Put 5 blocks in src. Put 2 of those blocks in dst @@ -296,7 +251,7 @@ func (s *ServerRequiredSuite) TestKeepRsync_WithNoSuchPrefixInSrc(c *C) { // Also, put 2 extra blocks in dst; they are hence only in dst // Run rsync and verify that those 7 blocks are now available in dst func testKeepRsync(c *C, enforcePermissions bool, prefix string) { - setupRsync(c, enforcePermissions, true, 1) + setupRsync(c, enforcePermissions, 1) // setupTestData setupTestData(c, prefix) @@ -305,19 +260,12 @@ func testKeepRsync(c *C, enforcePermissions bool, prefix string) { c.Check(err, IsNil) // Now GetIndex from dst and verify that all 5 from src and the 2 extra blocks are found - dstIndex, err := getUniqueLocators(kcDst, "") + dstIndex, err = getUniqueLocators(kcDst, "") c.Check(err, IsNil) - if prefix == "" { - for _, locator := range srcLocators { - _, ok := dstIndex[locator] - c.Assert(ok, Equals, true) - } - } else { - for _, locator := range srcLocatorsMatchingPrefix { - _, ok := dstIndex[locator] - c.Assert(ok, Equals, true) - } + for _, locator := range srcLocatorsMatchingPrefix { + _, ok := dstIndex[locator] + c.Assert(ok, Equals, true) } for _, locator := range extraDstLocators { @@ -329,19 +277,21 @@ func testKeepRsync(c *C, enforcePermissions bool, prefix string) { // all blocks from src and the two extra blocks c.Assert(len(dstIndex), Equals, len(srcLocators)+len(extraDstLocators)) } else { - // one matching prefix, 2 that were initially copied into dst along with src, and the extra blocks + // 1 matching prefix and copied over, 2 that were initially copied into dst along with src, and the 2 extra blocks c.Assert(len(dstIndex), Equals, len(srcLocatorsMatchingPrefix)+len(extraDstLocators)+2) } } // Setup test data in src and dst. var srcLocators, srcLocatorsMatchingPrefix, dstLocators, extraDstLocators []string +var dstIndex map[string]bool func setupTestData(c *C, indexPrefix string) { srcLocators = []string{} srcLocatorsMatchingPrefix = []string{} dstLocators = []string{} extraDstLocators = []string{} + dstIndex = make(map[string]bool) // Put a few blocks in src using kcSrc for i := 0; i < 5; i++ { @@ -375,10 +325,11 @@ func setupTestData(c *C, indexPrefix string) { func (s *ServerRequiredSuite) TestErrorDuringRsync_FakeSrcKeepservers(c *C) { srcKeepServicesJSON = testKeepServicesJSON - setupRsync(c, false, false, 1) + setupRsync(c, false, 1) err := performKeepRsync(kcSrc, kcDst, "", "") - c.Check(strings.HasSuffix(err.Error(), "no such host"), Equals, true) + log.Printf("Err = %v", err) + c.Check(strings.Contains(err.Error(), "no such host"), Equals, true) } // Setup rsync using dstKeepServicesJSON with fake keepservers. @@ -386,15 +337,16 @@ func (s *ServerRequiredSuite) TestErrorDuringRsync_FakeSrcKeepservers(c *C) { func (s *ServerRequiredSuite) TestErrorDuringRsync_FakeDstKeepservers(c *C) { dstKeepServicesJSON = testKeepServicesJSON - setupRsync(c, false, false, 1) + setupRsync(c, false, 1) err := performKeepRsync(kcSrc, kcDst, "", "") - c.Check(strings.HasSuffix(err.Error(), "no such host"), Equals, true) + log.Printf("Err = %v", err) + c.Check(strings.Contains(err.Error(), "no such host"), Equals, true) } // Test rsync with signature error during Get from src. func (s *ServerRequiredSuite) TestErrorDuringRsync_ErrorGettingBlockFromSrc(c *C) { - setupRsync(c, true, true, 1) + setupRsync(c, true, 1) // put some blocks in src and dst setupTestData(c, "") @@ -403,12 +355,12 @@ func (s *ServerRequiredSuite) TestErrorDuringRsync_ErrorGettingBlockFromSrc(c *C blobSigningKey = "thisisfakeblobsigningkey" err := performKeepRsync(kcSrc, kcDst, blobSigningKey, "") - c.Check(strings.HasSuffix(err.Error(), "Block not found"), Equals, true) + c.Check(strings.Contains(err.Error(), "HTTP 403 \"Forbidden\""), Equals, true) } // Test rsync with error during Put to src. func (s *ServerRequiredSuite) TestErrorDuringRsync_ErrorPuttingBlockInDst(c *C) { - setupRsync(c, false, true, 1) + setupRsync(c, false, 1) // put some blocks in src and dst setupTestData(c, "") @@ -417,11 +369,11 @@ func (s *ServerRequiredSuite) TestErrorDuringRsync_ErrorPuttingBlockInDst(c *C) kcDst.Want_replicas = 2 err := performKeepRsync(kcSrc, kcDst, blobSigningKey, "") - c.Check(strings.HasSuffix(err.Error(), "Could not write sufficient replicas"), Equals, true) + c.Check(strings.Contains(err.Error(), "Could not write sufficient replicas"), Equals, true) } // Test loadConfig func -func (s *ServerRequiredSuite) TestLoadConfig(c *C) { +func (s *ServerNotRequiredSuite) TestLoadConfig(c *C) { // Setup a src config file srcFile := setupConfigFile(c, "src-config") defer os.Remove(srcFile.Name()) @@ -436,32 +388,32 @@ func (s *ServerRequiredSuite) TestLoadConfig(c *C) { srcConfig, srcBlobSigningKey, err := loadConfig(srcConfigFile) c.Check(err, IsNil) - c.Assert(srcConfig.APIHost, Equals, "testhost") - c.Assert(srcConfig.APIToken, Equals, "testtoken") - c.Assert(srcConfig.APIHostInsecure, Equals, true) + c.Assert(srcConfig.APIHost, Equals, os.Getenv("ARVADOS_API_HOST")) + c.Assert(srcConfig.APIToken, Equals, arvadostest.DataManagerToken) + c.Assert(srcConfig.APIHostInsecure, Equals, matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))) c.Assert(srcConfig.ExternalClient, Equals, false) dstConfig, _, err := loadConfig(dstConfigFile) c.Check(err, IsNil) - c.Assert(dstConfig.APIHost, Equals, "testhost") - c.Assert(dstConfig.APIToken, Equals, "testtoken") - c.Assert(dstConfig.APIHostInsecure, Equals, true) + c.Assert(dstConfig.APIHost, Equals, os.Getenv("ARVADOS_API_HOST")) + c.Assert(dstConfig.APIToken, Equals, arvadostest.DataManagerToken) + c.Assert(dstConfig.APIHostInsecure, Equals, matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))) c.Assert(dstConfig.ExternalClient, Equals, false) c.Assert(srcBlobSigningKey, Equals, "abcdefg") } // Test loadConfig func without setting up the config files -func (s *ServerRequiredSuite) TestLoadConfig_MissingSrcConfig(c *C) { +func (s *ServerNotRequiredSuite) TestLoadConfig_MissingSrcConfig(c *C) { _, _, err := loadConfig("") c.Assert(err.Error(), Equals, "config file not specified") } // Test loadConfig func - error reading config -func (s *ServerRequiredSuite) TestLoadConfig_ErrorLoadingSrcConfig(c *C) { +func (s *ServerNotRequiredSuite) TestLoadConfig_ErrorLoadingSrcConfig(c *C) { _, _, err := loadConfig("no-such-config-file") - c.Assert(strings.HasSuffix(err.Error(), "no such file or directory"), Equals, true) + c.Assert(strings.Contains(err.Error(), "no such file or directory"), Equals, true) } func setupConfigFile(c *C, name string) *os.File { @@ -469,9 +421,10 @@ func setupConfigFile(c *C, name string) *os.File { file, err := ioutil.TempFile(os.TempDir(), name) c.Check(err, IsNil) - fileContent := "ARVADOS_API_HOST=testhost\n" - fileContent += "ARVADOS_API_TOKEN=testtoken\n" - fileContent += "ARVADOS_API_HOST_INSECURE=true\n" + fileContent := "ARVADOS_API_HOST=" + os.Getenv("ARVADOS_API_HOST") + "\n" + fileContent += "ARVADOS_API_TOKEN=" + arvadostest.DataManagerToken + "\n" + fileContent += "ARVADOS_API_HOST_INSECURE=" + os.Getenv("ARVADOS_API_HOST_INSECURE") + "\n" + fileContent += "ARVADOS_EXTERNAL_CLIENT=false\n" fileContent += "ARVADOS_BLOB_SIGNING_KEY=abcdefg" _, err = file.Write([]byte(fileContent)) @@ -479,3 +432,50 @@ func setupConfigFile(c *C, name string) *os.File { return file } + +func (s *DoMainTestSuite) Test_doMain_NoSrcConfig(c *C) { + err := doMain() + c.Check(err, NotNil) + c.Assert(err.Error(), Equals, "Error loading src configuration from file: config file not specified") +} + +func (s *DoMainTestSuite) Test_doMain_SrcButNoDstConfig(c *C) { + srcConfig := setupConfigFile(c, "src") + args := []string{"-replications", "3", "-src", srcConfig.Name()} + os.Args = append(os.Args, args...) + err := doMain() + c.Check(err, NotNil) + c.Assert(err.Error(), Equals, "Error loading dst configuration from file: config file not specified") +} + +func (s *DoMainTestSuite) Test_doMain_BadSrcConfig(c *C) { + args := []string{"-src", "abcd"} + os.Args = append(os.Args, args...) + err := doMain() + c.Check(err, NotNil) + c.Assert(strings.HasPrefix(err.Error(), "Error loading src configuration from file: Error reading config file"), Equals, true) +} + +func (s *DoMainTestSuite) Test_doMain_WithReplicationsButNoSrcConfig(c *C) { + args := []string{"-replications", "3"} + os.Args = append(os.Args, args...) + err := doMain() + c.Check(err, NotNil) + c.Assert(err.Error(), Equals, "Error loading src configuration from file: config file not specified") +} + +func (s *DoMainTestSuite) Test_doMainWithSrcAndDstConfig(c *C) { + srcConfig := setupConfigFile(c, "src") + dstConfig := setupConfigFile(c, "dst") + args := []string{"-src", srcConfig.Name(), "-dst", dstConfig.Name()} + os.Args = append(os.Args, args...) + + // Start keepservers. Since we are not doing any tweaking as + // in setupRsync func, kcSrc and kcDst will be the same and no + // actual copying to dst will happen, but that's ok. + arvadostest.StartKeep(2, false) + defer arvadostest.StopKeep(2) + + err := doMain() + c.Check(err, IsNil) +}