From 86df40e33e586ccb4dc506e00f773392d454804c Mon Sep 17 00:00:00 2001 From: radhika Date: Mon, 5 Oct 2015 07:46:56 -0400 Subject: [PATCH] 7167: keep-rsync parameter loading and intialization. Update test framework to allow creating two sets of keep servers, source and destination. --- sdk/go/arvadosclient/arvadosclient.go | 20 +++- sdk/go/arvadostest/run_servers.go | 7 ++ sdk/python/tests/run_test_server.py | 13 ++- tools/keep-rsync/.gitignore | 1 + tools/keep-rsync/keep-rsync.go | 141 ++++++++++++++++++++++++++ tools/keep-rsync/keep-rsync_test.go | 117 +++++++++++++++++++++ 6 files changed, 293 insertions(+), 6 deletions(-) create mode 100644 tools/keep-rsync/.gitignore create mode 100644 tools/keep-rsync/keep-rsync.go create mode 100644 tools/keep-rsync/keep-rsync_test.go diff --git a/sdk/go/arvadosclient/arvadosclient.go b/sdk/go/arvadosclient/arvadosclient.go index 1cce0a7fc9..ab2d9b29ea 100644 --- a/sdk/go/arvadosclient/arvadosclient.go +++ b/sdk/go/arvadosclient/arvadosclient.go @@ -82,13 +82,25 @@ type ArvadosClient struct { // variables ARVADOS_API_HOST, ARVADOS_API_TOKEN, and (optionally) // ARVADOS_API_HOST_INSECURE. func MakeArvadosClient() (ac ArvadosClient, err error) { + config := make(map[string]string) + config["ARVADOS_API_TOKEN"] = os.Getenv("ARVADOS_API_TOKEN") + config["ARVADOS_API_HOST"] = os.Getenv("ARVADOS_API_HOST") + config["ARVADOS_API_HOST_INSECURE"] = os.Getenv("ARVADOS_API_HOST_INSECURE") + config["ARVADOS_EXTERNAL_CLIENT"] = os.Getenv("ARVADOS_EXTERNAL_CLIENT") + + return MakeArvadosClientWithConfig(config) +} + +// Create a new ArvadosClient, using the given input parameters. +func MakeArvadosClientWithConfig(config map[string]string) (ac ArvadosClient, err error) { var matchTrue = regexp.MustCompile("^(?i:1|yes|true)$") - insecure := matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE")) - external := matchTrue.MatchString(os.Getenv("ARVADOS_EXTERNAL_CLIENT")) + + insecure := matchTrue.MatchString(config["ARVADOS_API_HOST_INSECURE"]) + external := matchTrue.MatchString(config["ARVADOS_EXTERNAL_CLIENT"]) ac = ArvadosClient{ - ApiServer: os.Getenv("ARVADOS_API_HOST"), - ApiToken: os.Getenv("ARVADOS_API_TOKEN"), + ApiServer: config["ARVADOS_API_HOST"], + ApiToken: config["ARVADOS_API_TOKEN"], ApiInsecure: insecure, Client: &http.Client{Transport: &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: insecure}}}, diff --git a/sdk/go/arvadostest/run_servers.go b/sdk/go/arvadostest/run_servers.go index cad16917db..a175136164 100644 --- a/sdk/go/arvadostest/run_servers.go +++ b/sdk/go/arvadostest/run_servers.go @@ -99,11 +99,18 @@ func StopAPI() { } func StartKeep() { + StartKeepAdditional(false) +} + +func StartKeepAdditional(keepExisting bool) { cwd, _ := os.Getwd() defer os.Chdir(cwd) chdirToPythonTests() cmd := exec.Command("python", "run_test_server.py", "start_keep") + if keepExisting { + cmd = exec.Command("python", "run_test_server.py", "start_keep", "--keep_existing", "true") + } stderr, err := cmd.StderrPipe() if err != nil { log.Fatalf("Setting up stderr pipe: %s", err) diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py index 5d0c42ad21..d5d1874749 100644 --- a/sdk/python/tests/run_test_server.py +++ b/sdk/python/tests/run_test_server.py @@ -324,7 +324,8 @@ def _start_keep(n, keep_args): return port def run_keep(blob_signing_key=None, enforce_permissions=False): - stop_keep() + if args.keep_existing is None: + stop_keep() keep_args = {} if not blob_signing_key: @@ -344,12 +345,16 @@ def run_keep(blob_signing_key=None, enforce_permissions=False): host=os.environ['ARVADOS_API_HOST'], token=os.environ['ARVADOS_API_TOKEN'], insecure=True) + for d in api.keep_services().list().execute()['items']: api.keep_services().delete(uuid=d['uuid']).execute() for d in api.keep_disks().list().execute()['items']: api.keep_disks().delete(uuid=d['uuid']).execute() - for d in range(0, 2): + start_index = 0 + if args.keep_existing is not None: + start_index = 2 + for d in range(start_index, start_index+2): port = _start_keep(d, keep_args) svc = api.keep_services().create(body={'keep_service': { 'uuid': 'zzzzz-bi6l4-keepdisk{:07d}'.format(d), @@ -374,6 +379,9 @@ def _stop_keep(n): def stop_keep(): _stop_keep(0) _stop_keep(1) + # We may have created 2 additional keep servers when keep_existing is used + _stop_keep(2) + _stop_keep(3) def run_keep_proxy(): if 'ARVADOS_TEST_PROXY_SERVICES' in os.environ: @@ -595,6 +603,7 @@ if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument('action', type=str, help="one of {}".format(actions)) parser.add_argument('--auth', type=str, metavar='FIXTURE_NAME', help='Print authorization info for given api_client_authorizations fixture') + parser.add_argument('--keep_existing', type=str, help="Used to add additional keep servers, without terminating existing servers") args = parser.parse_args() if args.action not in actions: diff --git a/tools/keep-rsync/.gitignore b/tools/keep-rsync/.gitignore new file mode 100644 index 0000000000..5ee7f3b61a --- /dev/null +++ b/tools/keep-rsync/.gitignore @@ -0,0 +1 @@ +keep-rsync diff --git a/tools/keep-rsync/keep-rsync.go b/tools/keep-rsync/keep-rsync.go new file mode 100644 index 0000000000..eff8b9c47a --- /dev/null +++ b/tools/keep-rsync/keep-rsync.go @@ -0,0 +1,141 @@ +package main + +import ( + "flag" + "git.curoverse.com/arvados.git/sdk/go/arvadosclient" + "git.curoverse.com/arvados.git/sdk/go/keepclient" + "io/ioutil" + "log" + "strings" +) + +// keep-rsync arguments +var ( + srcConfig map[string]string + destConfig map[string]string + srcKeepServicesJSON string + destKeepServicesJSON string + replications int + prefix string +) + +func main() { + var srcConfigFile string + var destConfigFile string + + flag.StringVar( + &srcConfigFile, + "src-config-file", + "", + "Source configuration filename with full path that contains "+ + "an ARVADOS_API_TOKEN which is a valid datamanager token recognized by the source keep servers, "+ + "ARVADOS_API_HOST, ARVADOS_API_HOST_INSECURE, and ARVADOS_BLOB_SIGNING_KEY.") + + flag.StringVar( + &destConfigFile, + "dest-config-file", + "", + "Destination configuration filename with full path that contains "+ + "an ARVADOS_API_TOKEN which is a valid datamanager token recognized by the destination keep servers, "+ + "ARVADOS_API_HOST, ARVADOS_API_HOST_INSECURE, and ARVADOS_BLOB_SIGNING_KEY.") + + flag.StringVar( + &srcKeepServicesJSON, + "src-keep-services-json", + "", + "An optional list of available source keepservices. "+ + "If not provided, this list is obtained from api server configured in src-config-file.") + + flag.StringVar( + &destKeepServicesJSON, + "dest-keep-services-json", + "", + "An optional list of available destination keepservices. "+ + "If not provided, this list is obtained from api server configured in dest-config-file.") + + flag.IntVar( + &replications, + "replications", + 3, + "Number of replications to write to the destination.") + + flag.StringVar( + &prefix, + "prefix", + "", + "Index prefix") + + flag.Parse() + + var err error + + if srcConfigFile == "" { + log.Fatal("-src-config-file must be specified.") + } + srcConfig, err = readConfigFromFile(srcConfigFile) + if err != nil { + log.Fatal("Error reading source configuration: %s", err.Error()) + } + + if destConfigFile == "" { + log.Fatal("-dest-config-file must be specified.") + } + destConfig, err = readConfigFromFile(destConfigFile) + if err != nil { + log.Fatal("Error reading destination configuration: %s", err.Error()) + } + + err = initializeKeepRsync() + if err != nil { + log.Fatal("Error configurating keep-rsync: %s", err.Error()) + } +} + +// Reads config from file +func readConfigFromFile(filename string) (map[string]string, error) { + content, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + + config := make(map[string]string) + lines := strings.Split(string(content), "\n") + for _, line := range lines { + if line == "" { + continue + } + kv := strings.Split(line, "=") + config[kv[0]] = kv[1] + } + return config, nil +} + +// keep-rsync source and destination clients +var ( + arvSrc arvadosclient.ArvadosClient + arvDest arvadosclient.ArvadosClient + kcSrc *keepclient.KeepClient + kcDest *keepclient.KeepClient +) + +// Initializes keep-rsync using the config provided +func initializeKeepRsync() (err error) { + arvSrc, err = arvadosclient.MakeArvadosClientWithConfig(srcConfig) + if err != nil { + return + } + + arvDest, err = arvadosclient.MakeArvadosClientWithConfig(destConfig) + if err != nil { + return + } + + kcSrc, err = keepclient.MakeKeepClient(&arvSrc) + if err != nil { + return + } + + kcDest, err = keepclient.MakeKeepClient(&arvDest) + + return +} diff --git a/tools/keep-rsync/keep-rsync_test.go b/tools/keep-rsync/keep-rsync_test.go new file mode 100644 index 0000000000..e2b1f0f0f7 --- /dev/null +++ b/tools/keep-rsync/keep-rsync_test.go @@ -0,0 +1,117 @@ +package main + +import ( + "crypto/md5" + "fmt" + "io/ioutil" + "os" + "testing" + + "git.curoverse.com/arvados.git/sdk/go/arvadostest" + "git.curoverse.com/arvados.git/sdk/go/keepclient" + + . "gopkg.in/check.v1" +) + +// Gocheck boilerplate +func Test(t *testing.T) { + TestingT(t) +} + +// Gocheck boilerplate +var _ = Suite(&ServerRequiredSuite{}) + +// Tests that require the Keep server running +type ServerRequiredSuite struct{} + +func (s *ServerRequiredSuite) SetUpSuite(c *C) { +} + +func (s *ServerRequiredSuite) SetUpTest(c *C) { + arvadostest.ResetEnv() +} + +func (s *ServerRequiredSuite) TearDownSuite(c *C) { + arvadostest.StopKeep() + arvadostest.StopAPI() +} + +// Testing keep-rsync needs two sets of keep services: src and dest. +// The test setup hence tweaks keep-rsync initialzation to achieve this. +// First invoke initializeKeepRsync and then invoke StartKeepAdditional +// to create the keep servers to be used as destination. +func setupRsync(c *C) { + // srcConfig + srcConfig = make(map[string]string) + srcConfig["ARVADOS_API_HOST"] = os.Getenv("ARVADOS_API_HOST") + srcConfig["ARVADOS_API_TOKEN"] = os.Getenv("ARVADOS_API_TOKEN") + srcConfig["ARVADOS_API_HOST_INSECURE"] = os.Getenv("ARVADOS_API_HOST_INSECURE") + + // destConfig + destConfig = make(map[string]string) + destConfig["ARVADOS_API_HOST"] = os.Getenv("ARVADOS_API_HOST") + destConfig["ARVADOS_API_TOKEN"] = os.Getenv("ARVADOS_API_TOKEN") + destConfig["ARVADOS_API_HOST_INSECURE"] = os.Getenv("ARVADOS_API_HOST_INSECURE") + + arvadostest.StartAPI() + arvadostest.StartKeep() + + // initialize keep-rsync + err := initializeKeepRsync() + c.Assert(err, Equals, nil) + + // Create two more keep servers to be used as destination + arvadostest.StartKeepAdditional(true) + + // load kcDest + kcDest, err = keepclient.MakeKeepClient(&arvDest) + c.Assert(err, Equals, nil) +} + +// Test readConfigFromFile method +func (s *ServerRequiredSuite) TestReadConfigFromFile(c *C) { + // Setup a test config file + file, err := ioutil.TempFile(os.TempDir(), "config") + c.Assert(err, Equals, nil) + defer os.Remove(file.Name()) + + fileContent := "ARVADOS_API_HOST=testhost\n" + fileContent += "ARVADOS_API_TOKEN=testtoken\n" + fileContent += "ARVADOS_API_HOST_INSECURE=true" + + _, err = file.Write([]byte(fileContent)) + + // Invoke readConfigFromFile method with this test filename + config, err := readConfigFromFile(file.Name()) + c.Assert(err, Equals, nil) + c.Assert(config["ARVADOS_API_HOST"], Equals, "testhost") + c.Assert(config["ARVADOS_API_TOKEN"], Equals, "testtoken") + c.Assert(config["ARVADOS_API_HOST_INSECURE"], Equals, "true") + c.Assert(config["EXTERNAL_CLIENT"], Equals, "") +} + +// Test keep-rsync initialization, with src and dest keep servers. +// Do a Put and Get in src, both of which should succeed. +// Do a Get in dest for the same hash, which should raise block not found error. +func (s *ServerRequiredSuite) TestRsyncPutInSrc_GetFromDestShouldFail(c *C) { + setupRsync(c) + + // Put a block in src using kcSrc and Get it + data := []byte("test-data") + hash := fmt.Sprintf("%x", md5.Sum(data)) + + hash2, rep, err := kcSrc.PutB(data) + c.Check(hash2, Matches, fmt.Sprintf(`^%s\+9(\+.+)?$`, hash)) + c.Check(rep, Equals, 2) + c.Check(err, Equals, nil) + + reader, blocklen, _, err := kcSrc.Get(hash) + c.Assert(err, Equals, nil) + c.Check(blocklen, Equals, int64(9)) + all, err := ioutil.ReadAll(reader) + c.Check(all, DeepEquals, data) + + // Get using kcDest should fail with NotFound error + _, _, _, err = kcDest.Get(hash) + c.Assert(err.Error(), Equals, "Block not found") +} -- 2.30.2