X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/f1ac845d8fdaa73ef8298ffdc1a6f82a515fb2bf..63be99931dcf5301ca75e9bc8eb049c86174eb0c:/tools/keep-rsync/keep-rsync_test.go diff --git a/tools/keep-rsync/keep-rsync_test.go b/tools/keep-rsync/keep-rsync_test.go index f722e9af81..dc5b957125 100644 --- a/tools/keep-rsync/keep-rsync_test.go +++ b/tools/keep-rsync/keep-rsync_test.go @@ -1,3 +1,7 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: AGPL-3.0 + package main import ( @@ -9,44 +13,66 @@ import ( "testing" "time" - "git.curoverse.com/arvados.git/sdk/go/arvadosclient" - "git.curoverse.com/arvados.git/sdk/go/arvadostest" - "git.curoverse.com/arvados.git/sdk/go/keepclient" + "git.arvados.org/arvados.git/sdk/go/arvadosclient" + "git.arvados.org/arvados.git/sdk/go/arvadostest" + "git.arvados.org/arvados.git/sdk/go/keepclient" . "gopkg.in/check.v1" ) +var kcSrc, kcDst *keepclient.KeepClient +var srcKeepServicesJSON, dstKeepServicesJSON, blobSigningKey string +var blobSignatureTTL = time.Duration(2*7*24) * time.Hour + +func resetGlobals() { + blobSigningKey = "" + srcKeepServicesJSON = "" + dstKeepServicesJSON = "" + kcSrc = nil + kcDst = nil +} + // Gocheck boilerplate func Test(t *testing.T) { TestingT(t) } -// Gocheck boilerplate var _ = Suite(&ServerRequiredSuite{}) +var _ = Suite(&ServerNotRequiredSuite{}) +var _ = Suite(&DoMainTestSuite{}) -// Tests that require the Keep server running type ServerRequiredSuite struct{} -func (s *ServerRequiredSuite) SetUpSuite(c *C) { +func (s *ServerRequiredSuite) TearDownSuite(c *C) { + arvadostest.ResetEnv() } -var kcSrc, kcDst *keepclient.KeepClient -var srcKeepServicesJSON, dstKeepServicesJSON, blobSigningKey string - func (s *ServerRequiredSuite) SetUpTest(c *C) { - arvadostest.ResetEnv() + resetGlobals() +} - // reset all variables between tests - blobSigningKey = "" - srcKeepServicesJSON = "" - dstKeepServicesJSON = "" - kcSrc = &keepclient.KeepClient{} - kcDst = &keepclient.KeepClient{} +func (s *ServerRequiredSuite) TearDownTest(c *C) { + arvadostest.StopKeep(3) } -func (s *ServerRequiredSuite) TearDownSuite(c *C) { - arvadostest.StopKeep() - arvadostest.StopAPI() +func (s *ServerNotRequiredSuite) SetUpTest(c *C) { + resetGlobals() +} + +type ServerNotRequiredSuite struct{} + +type DoMainTestSuite struct { + initialArgs []string +} + +func (s *DoMainTestSuite) SetUpTest(c *C) { + s.initialArgs = os.Args + os.Args = []string{"keep-rsync"} + resetGlobals() +} + +func (s *DoMainTestSuite) TearDownTest(c *C) { + os.Args = s.initialArgs } var testKeepServicesJSON = "{ \"kind\":\"arvados#keepServiceList\", \"etag\":\"\", \"self_link\":\"\", \"offset\":null, \"limit\":null, \"items\":[ { \"href\":\"/keep_services/zzzzz-bi6l4-123456789012340\", \"kind\":\"arvados#keepService\", \"etag\":\"641234567890enhj7hzx432e5\", \"uuid\":\"zzzzz-bi6l4-123456789012340\", \"owner_uuid\":\"zzzzz-tpzed-123456789012345\", \"service_host\":\"keep0.zzzzz.arvadosapi.com\", \"service_port\":25107, \"service_ssl_flag\":false, \"service_type\":\"disk\", \"read_only\":false }, { \"href\":\"/keep_services/zzzzz-bi6l4-123456789012341\", \"kind\":\"arvados#keepService\", \"etag\":\"641234567890enhj7hzx432e5\", \"uuid\":\"zzzzz-bi6l4-123456789012341\", \"owner_uuid\":\"zzzzz-tpzed-123456789012345\", \"service_host\":\"keep0.zzzzz.arvadosapi.com\", \"service_port\":25108, \"service_ssl_flag\":false, \"service_type\":\"disk\", \"read_only\":false } ], \"items_available\":2 }" @@ -54,210 +80,113 @@ var testKeepServicesJSON = "{ \"kind\":\"arvados#keepServiceList\", \"etag\":\"\ // Testing keep-rsync needs two sets of keep services: src and dst. // The test setup hence creates 3 servers instead of the default 2, // and uses the first 2 as src and the 3rd as dst keep servers. -func setupRsync(c *C, enforcePermissions, updateDstReplications bool, replications int) { +func setupRsync(c *C, enforcePermissions bool, replications int) { // srcConfig - var srcConfig arvadosclient.APIConfig + var srcConfig apiConfig srcConfig.APIHost = os.Getenv("ARVADOS_API_HOST") - srcConfig.APIToken = os.Getenv("ARVADOS_API_TOKEN") - srcConfig.APIHostInsecure = matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE")) + srcConfig.APIToken = arvadostest.SystemRootToken + srcConfig.APIHostInsecure = arvadosclient.StringBool(os.Getenv("ARVADOS_API_HOST_INSECURE")) // dstConfig - var dstConfig arvadosclient.APIConfig + var dstConfig apiConfig dstConfig.APIHost = os.Getenv("ARVADOS_API_HOST") - dstConfig.APIToken = os.Getenv("ARVADOS_API_TOKEN") - dstConfig.APIHostInsecure = matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE")) + dstConfig.APIToken = arvadostest.SystemRootToken + dstConfig.APIHostInsecure = arvadosclient.StringBool(os.Getenv("ARVADOS_API_HOST_INSECURE")) if enforcePermissions { - blobSigningKey = "zfhgfenhffzltr9dixws36j1yhksjoll2grmku38mi7yxd66h5j4q9w4jzanezacp8s6q0ro3hxakfye02152hncy6zml2ed0uc" + blobSigningKey = arvadostest.BlobSigningKey } - // Start API and Keep servers - arvadostest.StartAPI() - arvadostest.StartKeepWithParams(true, enforcePermissions) + // Start Keep servers + arvadostest.StartKeep(3, enforcePermissions) + keepclient.RefreshServiceDiscovery() // setup keepclients var err error - kcSrc, kcDst, err = setupKeepClients(srcConfig, dstConfig, srcKeepServicesJSON, dstKeepServicesJSON, replications) - c.Check(err, IsNil) + kcSrc, _, err = setupKeepClient(srcConfig, srcKeepServicesJSON, false, 0, blobSignatureTTL) + c.Assert(err, IsNil) - for uuid := range kcSrc.LocalRoots() { - if strings.HasSuffix(uuid, "02") { - delete(kcSrc.LocalRoots(), uuid) - } - } - for uuid := range kcSrc.GatewayRoots() { - if strings.HasSuffix(uuid, "02") { - delete(kcSrc.GatewayRoots(), uuid) - } - } - for uuid := range kcSrc.WritableLocalRoots() { - if strings.HasSuffix(uuid, "02") { - delete(kcSrc.WritableLocalRoots(), uuid) - } - } + kcDst, _, err = setupKeepClient(dstConfig, dstKeepServicesJSON, true, replications, 0) + c.Assert(err, IsNil) - for uuid := range kcDst.LocalRoots() { - if strings.HasSuffix(uuid, "00") || strings.HasSuffix(uuid, "01") { - delete(kcDst.LocalRoots(), uuid) + srcRoots := map[string]string{} + dstRoots := map[string]string{} + for uuid, root := range kcSrc.LocalRoots() { + if strings.HasSuffix(uuid, "02") { + dstRoots[uuid] = root + } else { + srcRoots[uuid] = root } } - for uuid := range kcDst.GatewayRoots() { - if strings.HasSuffix(uuid, "00") || strings.HasSuffix(uuid, "01") { - delete(kcDst.GatewayRoots(), uuid) - } + if srcKeepServicesJSON == "" { + kcSrc.SetServiceRoots(srcRoots, srcRoots, srcRoots) } - for uuid := range kcDst.WritableLocalRoots() { - if strings.HasSuffix(uuid, "00") || strings.HasSuffix(uuid, "01") { - delete(kcDst.WritableLocalRoots(), uuid) - } + if dstKeepServicesJSON == "" { + kcDst.SetServiceRoots(dstRoots, dstRoots, dstRoots) } - if updateDstReplications { - kcDst.Want_replicas = replications + if replications == 0 { + // Must have got default replications value of 2 from dst discovery document + c.Assert(kcDst.Want_replicas, Equals, 2) + } else { + // Since replications value is provided, it is used + c.Assert(kcDst.Want_replicas, Equals, replications) } } -// Test keep-rsync initialization, with src and dst keep servers. -// Do a Put and Get in src, both of which should succeed. -// Do a Put and Get in dst, both of which should succeed. -// Do a Get in dst for the src hash, which should raise block not found error. -// Do a Get in src for the dst hash, which should raise block not found error. func (s *ServerRequiredSuite) TestRsyncPutInOne_GetFromOtherShouldFail(c *C) { - setupRsync(c, false, true, 1) - - // Put a block in src using kcSrc and Get it - srcData := []byte("test-data1") - locatorInSrc := fmt.Sprintf("%x", md5.Sum(srcData)) + setupRsync(c, false, 1) - hash, rep, err := kcSrc.PutB(srcData) - c.Check(hash, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, locatorInSrc)) - c.Check(rep, Equals, 2) - c.Check(err, Equals, nil) - - reader, blocklen, _, err := kcSrc.Get(locatorInSrc) - c.Check(err, IsNil) - c.Check(blocklen, Equals, int64(10)) - all, err := ioutil.ReadAll(reader) - c.Check(all, DeepEquals, srcData) + // Put a block in src and verify that it is not found in dst + testNoCrosstalk(c, "test-data-1", kcSrc, kcDst) - // Put a different block in src using kcSrc and Get it - dstData := []byte("test-data2") - locatorInDst := fmt.Sprintf("%x", md5.Sum(dstData)) + // Put a block in dst and verify that it is not found in src + testNoCrosstalk(c, "test-data-2", kcDst, kcSrc) +} - hash, rep, err = kcDst.PutB(dstData) - c.Check(hash, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, locatorInDst)) - c.Check(rep, Equals, 1) - c.Check(err, Equals, nil) +func (s *ServerRequiredSuite) TestRsyncWithBlobSigning_PutInOne_GetFromOtherShouldFail(c *C) { + setupRsync(c, true, 1) - reader, blocklen, _, err = kcDst.Get(locatorInDst) - c.Check(err, IsNil) - c.Check(blocklen, Equals, int64(10)) - all, err = ioutil.ReadAll(reader) - c.Check(all, DeepEquals, dstData) + // Put a block in src and verify that it is not found in dst + testNoCrosstalk(c, "test-data-1", kcSrc, kcDst) - // Get srcLocator using kcDst should fail with Not Found error - _, _, _, err = kcDst.Get(locatorInSrc) - c.Assert(err.Error(), Equals, "Block not found") + // Put a block in dst and verify that it is not found in src + testNoCrosstalk(c, "test-data-2", kcDst, kcSrc) +} - // Get dstLocator using kcSrc should fail with Not Found error - _, _, _, err = kcSrc.Get(locatorInDst) - c.Assert(err.Error(), Equals, "Block not found") +// Do a Put in the first and Get from the second, +// which should raise block not found error. +func testNoCrosstalk(c *C, testData string, kc1, kc2 *keepclient.KeepClient) { + // Put a block using kc1 + locator, _, err := kc1.PutB([]byte(testData)) + c.Assert(err, Equals, nil) + + locator = strings.Split(locator, "+")[0] + _, _, _, err = kc2.Get(keepclient.SignLocator(locator, kc2.Arvados.ApiToken, time.Now().AddDate(0, 0, 1), blobSignatureTTL, []byte(blobSigningKey))) + c.Assert(err, NotNil) + c.Check(err.Error(), Equals, "Block not found") } // Test keep-rsync initialization, with srcKeepServicesJSON func (s *ServerRequiredSuite) TestRsyncInitializeWithKeepServicesJSON(c *C) { srcKeepServicesJSON = testKeepServicesJSON - setupRsync(c, false, true, 1) + setupRsync(c, false, 1) localRoots := kcSrc.LocalRoots() c.Check(localRoots, NotNil) - - foundIt := false - for k := range localRoots { - if k == "zzzzz-bi6l4-123456789012340" { - foundIt = true - } - } - c.Check(foundIt, Equals, true) - - foundIt = false - for k := range localRoots { - if k == "zzzzz-bi6l4-123456789012341" { - foundIt = true - } - } - c.Check(foundIt, Equals, true) -} - -// Test keep-rsync initialization, with src and dst keep servers with blobSigningKey. -// Do a Put and Get in src, both of which should succeed. -// Do a Put and Get in dst, both of which should succeed. -// Do a Get in dst for the src hash, which should raise block not found error. -// Do a Get in src for the dst hash, which should raise block not found error. -func (s *ServerRequiredSuite) TestRsyncWithBlobSigning_PutInOne_GetFromOtherShouldFail(c *C) { - setupRsync(c, true, true, 1) - - // Put a block in src using kcSrc and Get it - srcData := []byte("test-data1") - locatorInSrc := fmt.Sprintf("%x", md5.Sum(srcData)) - - hash, rep, err := kcSrc.PutB(srcData) - c.Check(hash, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, locatorInSrc)) - c.Check(rep, Equals, 2) - c.Check(err, Equals, nil) - - tomorrow := time.Now().AddDate(0, 0, 1) - signedLocator := keepclient.SignLocator(locatorInSrc, kcSrc.Arvados.ApiToken, tomorrow, []byte(blobSigningKey)) - - reader, blocklen, _, err := kcSrc.Get(signedLocator) - c.Check(err, IsNil) - c.Check(blocklen, Equals, int64(10)) - all, err := ioutil.ReadAll(reader) - c.Check(all, DeepEquals, srcData) - - // Put a different block in src using kcSrc and Get it - dstData := []byte("test-data2") - locatorInDst := fmt.Sprintf("%x", md5.Sum(dstData)) - - hash, rep, err = kcDst.PutB(dstData) - c.Check(hash, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, locatorInDst)) - c.Check(rep, Equals, 1) - c.Check(err, Equals, nil) - - signedLocator = keepclient.SignLocator(locatorInDst, kcDst.Arvados.ApiToken, tomorrow, []byte(blobSigningKey)) - - reader, blocklen, _, err = kcDst.Get(signedLocator) - c.Check(err, IsNil) - c.Check(blocklen, Equals, int64(10)) - all, err = ioutil.ReadAll(reader) - c.Check(all, DeepEquals, dstData) - - // Get srcLocator using kcDst should fail with Not Found error - signedLocator = keepclient.SignLocator(locatorInSrc, kcDst.Arvados.ApiToken, tomorrow, []byte(blobSigningKey)) - _, _, _, err = kcDst.Get(locatorInSrc) - c.Assert(err.Error(), Equals, "Block not found") - - // Get dstLocator using kcSrc should fail with Not Found error - signedLocator = keepclient.SignLocator(locatorInDst, kcSrc.Arvados.ApiToken, tomorrow, []byte(blobSigningKey)) - _, _, _, err = kcSrc.Get(locatorInDst) - c.Assert(err.Error(), Equals, "Block not found") + c.Check(localRoots["zzzzz-bi6l4-123456789012340"], Not(Equals), "") + c.Check(localRoots["zzzzz-bi6l4-123456789012341"], Not(Equals), "") } // Test keep-rsync initialization with default replications count func (s *ServerRequiredSuite) TestInitializeRsyncDefaultReplicationsCount(c *C) { - setupRsync(c, false, false, 0) - - // Must have got default replications value as 2 from dst discovery document - c.Assert(kcDst.Want_replicas, Equals, 2) + setupRsync(c, false, 0) } // Test keep-rsync initialization with replications count argument func (s *ServerRequiredSuite) TestInitializeRsyncReplicationsCount(c *C) { - setupRsync(c, false, false, 3) - - // Since replications value is provided, default is not used - c.Assert(kcDst.Want_replicas, Equals, 3) + setupRsync(c, false, 3) } // Put some blocks in Src and some more in Dst @@ -280,6 +209,7 @@ func (s *ServerRequiredSuite) TestKeepRsync_WithPrefix(c *C) { hash := fmt.Sprintf("%x", md5.Sum(data)) testKeepRsync(c, false, hash[0:3]) + c.Check(len(dstIndex) > len(dstLocators), Equals, true) } // Put some blocks in Src and some more in Dst @@ -287,6 +217,7 @@ func (s *ServerRequiredSuite) TestKeepRsync_WithPrefix(c *C) { // And copy missing blocks from Src to Dst func (s *ServerRequiredSuite) TestKeepRsync_WithNoSuchPrefixInSrc(c *C) { testKeepRsync(c, false, "999") + c.Check(len(dstIndex), Equals, len(dstLocators)) } // Put 5 blocks in src. Put 2 of those blocks in dst @@ -294,28 +225,21 @@ func (s *ServerRequiredSuite) TestKeepRsync_WithNoSuchPrefixInSrc(c *C) { // Also, put 2 extra blocks in dst; they are hence only in dst // Run rsync and verify that those 7 blocks are now available in dst func testKeepRsync(c *C, enforcePermissions bool, prefix string) { - setupRsync(c, enforcePermissions, true, 1) + setupRsync(c, enforcePermissions, 1) // setupTestData setupTestData(c, prefix) - err := performKeepRsync(kcSrc, kcDst, blobSigningKey, prefix) + err := performKeepRsync(kcSrc, kcDst, blobSignatureTTL, blobSigningKey, prefix) c.Check(err, IsNil) // Now GetIndex from dst and verify that all 5 from src and the 2 extra blocks are found - dstIndex, err := getUniqueLocators(kcDst, "") + dstIndex, err = getUniqueLocators(kcDst, "") c.Check(err, IsNil) - if prefix == "" { - for _, locator := range srcLocators { - _, ok := dstIndex[locator] - c.Assert(ok, Equals, true) - } - } else { - for _, locator := range srcLocatorsMatchingPrefix { - _, ok := dstIndex[locator] - c.Assert(ok, Equals, true) - } + for _, locator := range srcLocatorsMatchingPrefix { + _, ok := dstIndex[locator] + c.Assert(ok, Equals, true) } for _, locator := range extraDstLocators { @@ -327,19 +251,21 @@ func testKeepRsync(c *C, enforcePermissions bool, prefix string) { // all blocks from src and the two extra blocks c.Assert(len(dstIndex), Equals, len(srcLocators)+len(extraDstLocators)) } else { - // one matching prefix, 2 that were initially copied into dst along with src, and the extra blocks + // 1 matching prefix and copied over, 2 that were initially copied into dst along with src, and the 2 extra blocks c.Assert(len(dstIndex), Equals, len(srcLocatorsMatchingPrefix)+len(extraDstLocators)+2) } } // Setup test data in src and dst. var srcLocators, srcLocatorsMatchingPrefix, dstLocators, extraDstLocators []string +var dstIndex map[string]bool func setupTestData(c *C, indexPrefix string) { srcLocators = []string{} srcLocatorsMatchingPrefix = []string{} dstLocators = []string{} extraDstLocators = []string{} + dstIndex = make(map[string]bool) // Put a few blocks in src using kcSrc for i := 0; i < 5; i++ { @@ -373,10 +299,11 @@ func setupTestData(c *C, indexPrefix string) { func (s *ServerRequiredSuite) TestErrorDuringRsync_FakeSrcKeepservers(c *C) { srcKeepServicesJSON = testKeepServicesJSON - setupRsync(c, false, false, 1) + setupRsync(c, false, 1) - err := performKeepRsync(kcSrc, kcDst, "", "") - c.Check(strings.HasSuffix(err.Error(), "no such host"), Equals, true) + err := performKeepRsync(kcSrc, kcDst, blobSignatureTTL, "", "") + c.Assert(err, NotNil) + c.Check(err.Error(), Matches, ".*no such host.*") } // Setup rsync using dstKeepServicesJSON with fake keepservers. @@ -384,15 +311,16 @@ func (s *ServerRequiredSuite) TestErrorDuringRsync_FakeSrcKeepservers(c *C) { func (s *ServerRequiredSuite) TestErrorDuringRsync_FakeDstKeepservers(c *C) { dstKeepServicesJSON = testKeepServicesJSON - setupRsync(c, false, false, 1) + setupRsync(c, false, 1) - err := performKeepRsync(kcSrc, kcDst, "", "") - c.Check(strings.HasSuffix(err.Error(), "no such host"), Equals, true) + err := performKeepRsync(kcSrc, kcDst, blobSignatureTTL, "", "") + c.Assert(err, NotNil) + c.Check(err.Error(), Matches, ".*no such host.*") } // Test rsync with signature error during Get from src. func (s *ServerRequiredSuite) TestErrorDuringRsync_ErrorGettingBlockFromSrc(c *C) { - setupRsync(c, true, true, 1) + setupRsync(c, true, 1) // put some blocks in src and dst setupTestData(c, "") @@ -400,13 +328,14 @@ func (s *ServerRequiredSuite) TestErrorDuringRsync_ErrorGettingBlockFromSrc(c *C // Change blob signing key to a fake key, so that Get from src fails blobSigningKey = "thisisfakeblobsigningkey" - err := performKeepRsync(kcSrc, kcDst, blobSigningKey, "") - c.Check(strings.HasSuffix(err.Error(), "Block not found"), Equals, true) + err := performKeepRsync(kcSrc, kcDst, blobSignatureTTL, blobSigningKey, "") + c.Assert(err, NotNil) + c.Check(err.Error(), Matches, ".*HTTP 403 \"Forbidden\".*") } // Test rsync with error during Put to src. func (s *ServerRequiredSuite) TestErrorDuringRsync_ErrorPuttingBlockInDst(c *C) { - setupRsync(c, false, true, 1) + setupRsync(c, false, 1) // put some blocks in src and dst setupTestData(c, "") @@ -414,12 +343,13 @@ func (s *ServerRequiredSuite) TestErrorDuringRsync_ErrorPuttingBlockInDst(c *C) // Increase Want_replicas on dst to result in insufficient replicas error during Put kcDst.Want_replicas = 2 - err := performKeepRsync(kcSrc, kcDst, blobSigningKey, "") - c.Check(strings.HasSuffix(err.Error(), "Could not write sufficient replicas"), Equals, true) + err := performKeepRsync(kcSrc, kcDst, blobSignatureTTL, blobSigningKey, "") + c.Assert(err, NotNil) + c.Check(err.Error(), Matches, ".*Could not write sufficient replicas.*") } // Test loadConfig func -func (s *ServerRequiredSuite) TestLoadConfig(c *C) { +func (s *ServerNotRequiredSuite) TestLoadConfig(c *C) { // Setup a src config file srcFile := setupConfigFile(c, "src-config") defer os.Remove(srcFile.Name()) @@ -431,56 +361,45 @@ func (s *ServerRequiredSuite) TestLoadConfig(c *C) { dstConfigFile := dstFile.Name() // load configuration from those files - srcConfig, dstConfig, srcBlobSigningKey, _, err := loadConfig(srcConfigFile, dstConfigFile) + srcConfig, srcBlobSigningKey, err := loadConfig(srcConfigFile) c.Check(err, IsNil) - c.Assert(srcConfig.APIHost, Equals, "testhost") - c.Assert(srcConfig.APIToken, Equals, "testtoken") - c.Assert(srcConfig.APIHostInsecure, Equals, true) - c.Assert(srcConfig.ExternalClient, Equals, false) + c.Assert(srcConfig.APIHost, Equals, os.Getenv("ARVADOS_API_HOST")) + c.Assert(srcConfig.APIToken, Equals, arvadostest.SystemRootToken) + c.Assert(srcConfig.APIHostInsecure, Equals, arvadosclient.StringBool(os.Getenv("ARVADOS_API_HOST_INSECURE"))) - c.Assert(dstConfig.APIHost, Equals, "testhost") - c.Assert(dstConfig.APIToken, Equals, "testtoken") - c.Assert(dstConfig.APIHostInsecure, Equals, true) - c.Assert(dstConfig.ExternalClient, Equals, false) + dstConfig, _, err := loadConfig(dstConfigFile) + c.Check(err, IsNil) + + c.Assert(dstConfig.APIHost, Equals, os.Getenv("ARVADOS_API_HOST")) + c.Assert(dstConfig.APIToken, Equals, arvadostest.SystemRootToken) + c.Assert(dstConfig.APIHostInsecure, Equals, arvadosclient.StringBool(os.Getenv("ARVADOS_API_HOST_INSECURE"))) c.Assert(srcBlobSigningKey, Equals, "abcdefg") } // Test loadConfig func without setting up the config files -func (s *ServerRequiredSuite) TestLoadConfig_MissingSrcConfig(c *C) { - _, _, _, _, err := loadConfig("", "") - c.Assert(err.Error(), Equals, "-src-config-file must be specified") +func (s *ServerNotRequiredSuite) TestLoadConfig_MissingSrcConfig(c *C) { + _, _, err := loadConfig("") + c.Assert(err.Error(), Equals, "config file not specified") } -// Test loadConfig func - error reading src config -func (s *ServerRequiredSuite) TestLoadConfig_ErrorLoadingSrcConfig(c *C) { - _, _, _, _, err := loadConfig("no-such-config-file", "") - c.Assert(strings.HasSuffix(err.Error(), "no such file or directory"), Equals, true) +// Test loadConfig func - error reading config +func (s *ServerNotRequiredSuite) TestLoadConfig_ErrorLoadingSrcConfig(c *C) { + _, _, err := loadConfig("no-such-config-file") + c.Assert(err, NotNil) + c.Check(err.Error(), Matches, ".*no such file or directory.*") } -// Test loadConfig func with no dst config file specified -func (s *ServerRequiredSuite) TestLoadConfig_MissingDstConfig(c *C) { - // Setup a src config file - srcFile := setupConfigFile(c, "src-config") - defer os.Remove(srcFile.Name()) - srcConfigFile := srcFile.Name() - - // load configuration - _, _, _, _, err := loadConfig(srcConfigFile, "") - c.Assert(err.Error(), Equals, "-dst-config-file must be specified") -} - -// Test loadConfig func -func (s *ServerRequiredSuite) TestLoadConfig_ErrorLoadingDstConfig(c *C) { - // Setup a src config file - srcFile := setupConfigFile(c, "src-config") - defer os.Remove(srcFile.Name()) - srcConfigFile := srcFile.Name() +func (s *ServerNotRequiredSuite) TestSetupKeepClient_NoBlobSignatureTTL(c *C) { + var srcConfig apiConfig + srcConfig.APIHost = os.Getenv("ARVADOS_API_HOST") + srcConfig.APIToken = arvadostest.SystemRootToken + srcConfig.APIHostInsecure = arvadosclient.StringBool(os.Getenv("ARVADOS_API_HOST_INSECURE")) - // load configuration - _, _, _, _, err := loadConfig(srcConfigFile, "no-such-config-file") - c.Assert(strings.HasSuffix(err.Error(), "no such file or directory"), Equals, true) + _, ttl, err := setupKeepClient(srcConfig, srcKeepServicesJSON, false, 0, 0) + c.Check(err, IsNil) + c.Assert(ttl, Equals, blobSignatureTTL) } func setupConfigFile(c *C, name string) *os.File { @@ -488,9 +407,9 @@ func setupConfigFile(c *C, name string) *os.File { file, err := ioutil.TempFile(os.TempDir(), name) c.Check(err, IsNil) - fileContent := "ARVADOS_API_HOST=testhost\n" - fileContent += "ARVADOS_API_TOKEN=testtoken\n" - fileContent += "ARVADOS_API_HOST_INSECURE=true\n" + fileContent := "ARVADOS_API_HOST=" + os.Getenv("ARVADOS_API_HOST") + "\n" + fileContent += "ARVADOS_API_TOKEN=" + arvadostest.SystemRootToken + "\n" + fileContent += "ARVADOS_API_HOST_INSECURE=" + os.Getenv("ARVADOS_API_HOST_INSECURE") + "\n" fileContent += "ARVADOS_BLOB_SIGNING_KEY=abcdefg" _, err = file.Write([]byte(fileContent)) @@ -498,3 +417,51 @@ func setupConfigFile(c *C, name string) *os.File { return file } + +func (s *DoMainTestSuite) Test_doMain_NoSrcConfig(c *C) { + err := doMain() + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "Error loading src configuration from file: config file not specified") +} + +func (s *DoMainTestSuite) Test_doMain_SrcButNoDstConfig(c *C) { + srcConfig := setupConfigFile(c, "src") + args := []string{"-replications", "3", "-src", srcConfig.Name()} + os.Args = append(os.Args, args...) + err := doMain() + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, "Error loading dst configuration from file: config file not specified") +} + +func (s *DoMainTestSuite) Test_doMain_BadSrcConfig(c *C) { + args := []string{"-src", "abcd"} + os.Args = append(os.Args, args...) + err := doMain() + c.Assert(err, NotNil) + c.Assert(err.Error(), Matches, "Error loading src configuration from file: Error reading config file.*") +} + +func (s *DoMainTestSuite) Test_doMain_WithReplicationsButNoSrcConfig(c *C) { + args := []string{"-replications", "3"} + os.Args = append(os.Args, args...) + err := doMain() + c.Check(err, NotNil) + c.Assert(err.Error(), Equals, "Error loading src configuration from file: config file not specified") +} + +func (s *DoMainTestSuite) Test_doMainWithSrcAndDstConfig(c *C) { + srcConfig := setupConfigFile(c, "src") + dstConfig := setupConfigFile(c, "dst") + args := []string{"-src", srcConfig.Name(), "-dst", dstConfig.Name()} + os.Args = append(os.Args, args...) + + // Start keepservers. Since we are not doing any tweaking as + // in setupRsync func, kcSrc and kcDst will be the same and no + // actual copying to dst will happen, but that's ok. + arvadostest.StartKeep(2, false) + defer arvadostest.StopKeep(2) + keepclient.RefreshServiceDiscovery() + + err := doMain() + c.Check(err, IsNil) +}