X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/3c88abd3cb33cbe80bb81a7cca779fe668036c9e..0152714fbaae387661b858ca1b761b9409ccf803:/tools/keep-rsync/keep-rsync_test.go

diff --git a/tools/keep-rsync/keep-rsync_test.go b/tools/keep-rsync/keep-rsync_test.go
index f1f6a5f7ab..45ed3f67f5 100644
--- a/tools/keep-rsync/keep-rsync_test.go
+++ b/tools/keep-rsync/keep-rsync_test.go
@@ -1,76 +1,78 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
 package main
 
 import (
 	"crypto/md5"
 	"fmt"
 	"io/ioutil"
-	"log"
 	"os"
 	"strings"
 	"testing"
 	"time"
 
-	"git.curoverse.com/arvados.git/sdk/go/arvadostest"
-	"git.curoverse.com/arvados.git/sdk/go/keepclient"
+	"git.arvados.org/arvados.git/sdk/go/arvadosclient"
+	"git.arvados.org/arvados.git/sdk/go/arvadostest"
+	"git.arvados.org/arvados.git/sdk/go/keepclient"
 
 	. "gopkg.in/check.v1"
 )
 
+var kcSrc, kcDst *keepclient.KeepClient
+var srcKeepServicesJSON, dstKeepServicesJSON, blobSigningKey string
+var blobSignatureTTL = time.Duration(2*7*24) * time.Hour
+
+func resetGlobals() {
+	blobSigningKey = ""
+	srcKeepServicesJSON = ""
+	dstKeepServicesJSON = ""
+	kcSrc = nil
+	kcDst = nil
+}
+
 // Gocheck boilerplate
 func Test(t *testing.T) {
 	TestingT(t)
 }
 
-// Gocheck boilerplate
 var _ = Suite(&ServerRequiredSuite{})
 var _ = Suite(&ServerNotRequiredSuite{})
 var _ = Suite(&DoMainTestSuite{})
 
-// Tests that require the Keep server running
 type ServerRequiredSuite struct{}
-type ServerNotRequiredSuite struct{}
-type DoMainTestSuite struct{}
-
-func (s *ServerRequiredSuite) SetUpSuite(c *C) {
-	// Start API server
-	arvadostest.StartAPI()
-}
 
 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
-	arvadostest.StopAPI()
 	arvadostest.ResetEnv()
 }
 
-var initialArgs []string
-
-func (s *DoMainTestSuite) SetUpSuite(c *C) {
-	initialArgs = os.Args
-}
-
-var kcSrc, kcDst *keepclient.KeepClient
-var srcKeepServicesJSON, dstKeepServicesJSON, blobSigningKey string
-var blobSignatureTTL = time.Duration(2*7*24) * time.Hour
-
 func (s *ServerRequiredSuite) SetUpTest(c *C) {
-	// reset all variables between tests
-	blobSigningKey = ""
-	srcKeepServicesJSON = ""
-	dstKeepServicesJSON = ""
-	kcSrc = &keepclient.KeepClient{}
-	kcDst = &keepclient.KeepClient{}
+	resetGlobals()
 }
 
 func (s *ServerRequiredSuite) TearDownTest(c *C) {
 	arvadostest.StopKeep(3)
 }
 
+func (s *ServerNotRequiredSuite) SetUpTest(c *C) {
+	resetGlobals()
+}
+
+type ServerNotRequiredSuite struct{}
+
+type DoMainTestSuite struct {
+	initialArgs []string
+}
+
 func (s *DoMainTestSuite) SetUpTest(c *C) {
-	args := []string{"keep-rsync"}
-	os.Args = args
+	s.initialArgs = os.Args
+	os.Args = []string{"keep-rsync"}
+	resetGlobals()
 }
 
 func (s *DoMainTestSuite) TearDownTest(c *C) {
-	os.Args = initialArgs
+	os.Args = s.initialArgs
 }
 
 var testKeepServicesJSON = "{ \"kind\":\"arvados#keepServiceList\", \"etag\":\"\", \"self_link\":\"\", \"offset\":null, \"limit\":null, \"items\":[ { \"href\":\"/keep_services/zzzzz-bi6l4-123456789012340\", \"kind\":\"arvados#keepService\", \"etag\":\"641234567890enhj7hzx432e5\", \"uuid\":\"zzzzz-bi6l4-123456789012340\", \"owner_uuid\":\"zzzzz-tpzed-123456789012345\", \"service_host\":\"keep0.zzzzz.arvadosapi.com\", \"service_port\":25107, \"service_ssl_flag\":false, \"service_type\":\"disk\", \"read_only\":false }, { \"href\":\"/keep_services/zzzzz-bi6l4-123456789012341\", \"kind\":\"arvados#keepService\", \"etag\":\"641234567890enhj7hzx432e5\", \"uuid\":\"zzzzz-bi6l4-123456789012341\", \"owner_uuid\":\"zzzzz-tpzed-123456789012345\", \"service_host\":\"keep0.zzzzz.arvadosapi.com\", \"service_port\":25108, \"service_ssl_flag\":false, \"service_type\":\"disk\", \"read_only\":false } ], \"items_available\":2 }"
@@ -82,14 +84,14 @@ func setupRsync(c *C, enforcePermissions bool, replications int) {
 	// srcConfig
 	var srcConfig apiConfig
 	srcConfig.APIHost = os.Getenv("ARVADOS_API_HOST")
-	srcConfig.APIToken = arvadostest.DataManagerToken
-	srcConfig.APIHostInsecure = matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))
+	srcConfig.APIToken = arvadostest.SystemRootToken
+	srcConfig.APIHostInsecure = arvadosclient.StringBool(os.Getenv("ARVADOS_API_HOST_INSECURE"))
 
 	// dstConfig
 	var dstConfig apiConfig
 	dstConfig.APIHost = os.Getenv("ARVADOS_API_HOST")
-	dstConfig.APIToken = arvadostest.DataManagerToken
-	dstConfig.APIHostInsecure = matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))
+	dstConfig.APIToken = arvadostest.SystemRootToken
+	dstConfig.APIHostInsecure = arvadosclient.StringBool(os.Getenv("ARVADOS_API_HOST_INSECURE"))
 
 	if enforcePermissions {
 		blobSigningKey = arvadostest.BlobSigningKey
@@ -97,45 +99,30 @@ func setupRsync(c *C, enforcePermissions bool, replications int) {
 
 	// Start Keep servers
 	arvadostest.StartKeep(3, enforcePermissions)
+	keepclient.RefreshServiceDiscovery()
 
 	// setup keepclients
 	var err error
-	kcSrc, err = setupKeepClient(srcConfig, srcKeepServicesJSON, false, 0, blobSignatureTTL)
-	c.Check(err, IsNil)
+	kcSrc, _, err = setupKeepClient(srcConfig, srcKeepServicesJSON, false, 0, blobSignatureTTL)
+	c.Assert(err, IsNil)
 
-	kcDst, err = setupKeepClient(dstConfig, dstKeepServicesJSON, true, replications, 0)
-	c.Check(err, IsNil)
+	kcDst, _, err = setupKeepClient(dstConfig, dstKeepServicesJSON, true, replications, 0)
+	c.Assert(err, IsNil)
 
-	for uuid := range kcSrc.LocalRoots() {
-		if strings.HasSuffix(uuid, "02") {
-			delete(kcSrc.LocalRoots(), uuid)
-		}
-	}
-	for uuid := range kcSrc.GatewayRoots() {
+	srcRoots := map[string]string{}
+	dstRoots := map[string]string{}
+	for uuid, root := range kcSrc.LocalRoots() {
 		if strings.HasSuffix(uuid, "02") {
-			delete(kcSrc.GatewayRoots(), uuid)
+			dstRoots[uuid] = root
+		} else {
+			srcRoots[uuid] = root
 		}
 	}
-	for uuid := range kcSrc.WritableLocalRoots() {
-		if strings.HasSuffix(uuid, "02") {
-			delete(kcSrc.WritableLocalRoots(), uuid)
-		}
+	if srcKeepServicesJSON == "" {
+		kcSrc.SetServiceRoots(srcRoots, srcRoots, srcRoots)
 	}
-
-	for uuid := range kcDst.LocalRoots() {
-		if strings.HasSuffix(uuid, "00") || strings.HasSuffix(uuid, "01") {
-			delete(kcDst.LocalRoots(), uuid)
-		}
-	}
-	for uuid := range kcDst.GatewayRoots() {
-		if strings.HasSuffix(uuid, "00") || strings.HasSuffix(uuid, "01") {
-			delete(kcDst.GatewayRoots(), uuid)
-		}
-	}
-	for uuid := range kcDst.WritableLocalRoots() {
-		if strings.HasSuffix(uuid, "00") || strings.HasSuffix(uuid, "01") {
-			delete(kcDst.WritableLocalRoots(), uuid)
-		}
+	if dstKeepServicesJSON == "" {
+		kcDst.SetServiceRoots(dstRoots, dstRoots, dstRoots)
 	}
 
 	if replications == 0 {
@@ -188,22 +175,8 @@ func (s *ServerRequiredSuite) TestRsyncInitializeWithKeepServicesJSON(c *C) {
 
 	localRoots := kcSrc.LocalRoots()
 	c.Check(localRoots, NotNil)
-
-	foundIt := false
-	for k := range localRoots {
-		if k == "zzzzz-bi6l4-123456789012340" {
-			foundIt = true
-		}
-	}
-	c.Check(foundIt, Equals, true)
-
-	foundIt = false
-	for k := range localRoots {
-		if k == "zzzzz-bi6l4-123456789012341" {
-			foundIt = true
-		}
-	}
-	c.Check(foundIt, Equals, true)
+	c.Check(localRoots["zzzzz-bi6l4-123456789012340"], Not(Equals), "")
+	c.Check(localRoots["zzzzz-bi6l4-123456789012341"], Not(Equals), "")
 }
 
 // Test keep-rsync initialization with default replications count
@@ -329,8 +302,8 @@ func (s *ServerRequiredSuite) TestErrorDuringRsync_FakeSrcKeepservers(c *C) {
 	setupRsync(c, false, 1)
 
 	err := performKeepRsync(kcSrc, kcDst, blobSignatureTTL, "", "")
-	log.Printf("Err = %v", err)
-	c.Check(strings.Contains(err.Error(), "no such host"), Equals, true)
+	c.Assert(err, NotNil)
+	c.Check(err.Error(), Matches, ".*no such host.*")
 }
 
 // Setup rsync using dstKeepServicesJSON with fake keepservers.
@@ -341,8 +314,8 @@ func (s *ServerRequiredSuite) TestErrorDuringRsync_FakeDstKeepservers(c *C) {
 	setupRsync(c, false, 1)
 
 	err := performKeepRsync(kcSrc, kcDst, blobSignatureTTL, "", "")
-	log.Printf("Err = %v", err)
-	c.Check(strings.Contains(err.Error(), "no such host"), Equals, true)
+	c.Assert(err, NotNil)
+	c.Check(err.Error(), Matches, ".*no such host.*")
 }
 
 // Test rsync with signature error during Get from src.
@@ -356,7 +329,8 @@ func (s *ServerRequiredSuite) TestErrorDuringRsync_ErrorGettingBlockFromSrc(c *C
 	blobSigningKey = "thisisfakeblobsigningkey"
 
 	err := performKeepRsync(kcSrc, kcDst, blobSignatureTTL, blobSigningKey, "")
-	c.Check(strings.Contains(err.Error(), "HTTP 403 \"Forbidden\""), Equals, true)
+	c.Assert(err, NotNil)
+	c.Check(err.Error(), Matches, ".*HTTP 403 \"Forbidden\".*")
 }
 
 // Test rsync with error during Put to src.
@@ -370,7 +344,8 @@ func (s *ServerRequiredSuite) TestErrorDuringRsync_ErrorPuttingBlockInDst(c *C)
 	kcDst.Want_replicas = 2
 
 	err := performKeepRsync(kcSrc, kcDst, blobSignatureTTL, blobSigningKey, "")
-	c.Check(strings.Contains(err.Error(), "Could not write sufficient replicas"), Equals, true)
+	c.Assert(err, NotNil)
+	c.Check(err.Error(), Matches, ".*Could not write sufficient replicas.*")
 }
 
 // Test loadConfig func
@@ -390,16 +365,16 @@ func (s *ServerNotRequiredSuite) TestLoadConfig(c *C) {
 	c.Check(err, IsNil)
 
 	c.Assert(srcConfig.APIHost, Equals, os.Getenv("ARVADOS_API_HOST"))
-	c.Assert(srcConfig.APIToken, Equals, arvadostest.DataManagerToken)
-	c.Assert(srcConfig.APIHostInsecure, Equals, matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE")))
+	c.Assert(srcConfig.APIToken, Equals, arvadostest.SystemRootToken)
+	c.Assert(srcConfig.APIHostInsecure, Equals, arvadosclient.StringBool(os.Getenv("ARVADOS_API_HOST_INSECURE")))
 	c.Assert(srcConfig.ExternalClient, Equals, false)
 
 	dstConfig, _, err := loadConfig(dstConfigFile)
 	c.Check(err, IsNil)
 
 	c.Assert(dstConfig.APIHost, Equals, os.Getenv("ARVADOS_API_HOST"))
-	c.Assert(dstConfig.APIToken, Equals, arvadostest.DataManagerToken)
-	c.Assert(dstConfig.APIHostInsecure, Equals, matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE")))
+	c.Assert(dstConfig.APIToken, Equals, arvadostest.SystemRootToken)
+	c.Assert(dstConfig.APIHostInsecure, Equals, arvadosclient.StringBool(os.Getenv("ARVADOS_API_HOST_INSECURE")))
 	c.Assert(dstConfig.ExternalClient, Equals, false)
 
 	c.Assert(srcBlobSigningKey, Equals, "abcdefg")
@@ -414,7 +389,19 @@ func (s *ServerNotRequiredSuite) TestLoadConfig_MissingSrcConfig(c *C) {
 // Test loadConfig func - error reading config
 func (s *ServerNotRequiredSuite) TestLoadConfig_ErrorLoadingSrcConfig(c *C) {
 	_, _, err := loadConfig("no-such-config-file")
-	c.Assert(strings.Contains(err.Error(), "no such file or directory"), Equals, true)
+	c.Assert(err, NotNil)
+	c.Check(err.Error(), Matches, ".*no such file or directory.*")
+}
+
+func (s *ServerNotRequiredSuite) TestSetupKeepClient_NoBlobSignatureTTL(c *C) {
+	var srcConfig apiConfig
+	srcConfig.APIHost = os.Getenv("ARVADOS_API_HOST")
+	srcConfig.APIToken = arvadostest.SystemRootToken
+	srcConfig.APIHostInsecure = arvadosclient.StringBool(os.Getenv("ARVADOS_API_HOST_INSECURE"))
+
+	_, ttl, err := setupKeepClient(srcConfig, srcKeepServicesJSON, false, 0, 0)
+	c.Check(err, IsNil)
+	c.Assert(ttl, Equals, blobSignatureTTL)
 }
 
 func setupConfigFile(c *C, name string) *os.File {
@@ -423,7 +410,7 @@ func setupConfigFile(c *C, name string) *os.File {
 	c.Check(err, IsNil)
 
 	fileContent := "ARVADOS_API_HOST=" + os.Getenv("ARVADOS_API_HOST") + "\n"
-	fileContent += "ARVADOS_API_TOKEN=" + arvadostest.DataManagerToken + "\n"
+	fileContent += "ARVADOS_API_TOKEN=" + arvadostest.SystemRootToken + "\n"
 	fileContent += "ARVADOS_API_HOST_INSECURE=" + os.Getenv("ARVADOS_API_HOST_INSECURE") + "\n"
 	fileContent += "ARVADOS_EXTERNAL_CLIENT=false\n"
 	fileContent += "ARVADOS_BLOB_SIGNING_KEY=abcdefg"
@@ -436,7 +423,7 @@ func setupConfigFile(c *C, name string) *os.File {
 
 func (s *DoMainTestSuite) Test_doMain_NoSrcConfig(c *C) {
 	err := doMain()
-	c.Check(err, NotNil)
+	c.Assert(err, NotNil)
 	c.Assert(err.Error(), Equals, "Error loading src configuration from file: config file not specified")
 }
 
@@ -445,7 +432,7 @@ func (s *DoMainTestSuite) Test_doMain_SrcButNoDstConfig(c *C) {
 	args := []string{"-replications", "3", "-src", srcConfig.Name()}
 	os.Args = append(os.Args, args...)
 	err := doMain()
-	c.Check(err, NotNil)
+	c.Assert(err, NotNil)
 	c.Assert(err.Error(), Equals, "Error loading dst configuration from file: config file not specified")
 }
 
@@ -453,8 +440,8 @@ func (s *DoMainTestSuite) Test_doMain_BadSrcConfig(c *C) {
 	args := []string{"-src", "abcd"}
 	os.Args = append(os.Args, args...)
 	err := doMain()
-	c.Check(err, NotNil)
-	c.Assert(strings.HasPrefix(err.Error(), "Error loading src configuration from file: Error reading config file"), Equals, true)
+	c.Assert(err, NotNil)
+	c.Assert(err.Error(), Matches, "Error loading src configuration from file: Error reading config file.*")
 }
 
 func (s *DoMainTestSuite) Test_doMain_WithReplicationsButNoSrcConfig(c *C) {
@@ -476,6 +463,7 @@ func (s *DoMainTestSuite) Test_doMainWithSrcAndDstConfig(c *C) {
 	// actual copying to dst will happen, but that's ok.
 	arvadostest.StartKeep(2, false)
 	defer arvadostest.StopKeep(2)
+	keepclient.RefreshServiceDiscovery()
 
 	err := doMain()
 	c.Check(err, IsNil)