2853: Use rendezvous hashing to select probe order.

[arvados.git] / sdk / go / keepclient / keepclient_test.go

diff --git a/sdk/go/keepclient/keepclient_test.go b/sdk/go/keepclient/keepclient_test.go
index 015b24896ebf72ed756509057a7f622f464a4438..608e714432dc885b94f0d77a0dccb0594cc333b9 100644 (file)
--- a/sdk/go/keepclient/keepclient_test.go
+++ b/sdk/go/keepclient/keepclient_test.go
@@ -76,7 +76,7 @@ func (s *ServerRequiredSuite) TearDownSuite(c *C) {
 }
 
 func (s *ServerRequiredSuite) TestMakeKeepClient(c *C) {
-       os.Setenv("ARVADOS_API_HOST", "localhost:3001")
+       os.Setenv("ARVADOS_API_HOST", "localhost:3000")
        os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
        os.Setenv("ARVADOS_API_HOST_INSECURE", "true")
 
@@ -87,20 +87,27 @@ func (s *ServerRequiredSuite) TestMakeKeepClient(c *C) {
 
        c.Assert(err, Equals, nil)
        c.Check(len(kc.ServiceRoots()), Equals, 2)
-       c.Check(kc.ServiceRoots()[0], Equals, "http://localhost:25107")
-       c.Check(kc.ServiceRoots()[1], Equals, "http://localhost:25108")
+       for _, root := range kc.ServiceRoots() {
+               c.Check(root, Matches, "http://localhost:2510[\\d]")
+       }
 }
 
 func (s *StandaloneSuite) TestShuffleServiceRoots(c *C) {
+       roots := map[string]string{
+               "zzzzz-bi6l4-2q7dq8becevdqfb": "http://localhost:1",
+               "zzzzz-bi6l4-4gbhck2w7lq0d96": "http://localhost:2",
+               "zzzzz-bi6l4-4bt69dsk0quh7ae": "http://localhost:3",
+               "zzzzz-bi6l4-62w1fgd0ud2krxl": "http://localhost:4",
+       }
        kc := KeepClient{}
-       kc.SetServiceRoots([]string{"http://localhost:25107", "http://localhost:25108", "http://localhost:25109", "http://localhost:25110", "http://localhost:25111", "http://localhost:25112", "http://localhost:25113", "http://localhost:25114", "http://localhost:25115", "http://localhost:25116", "http://localhost:25117", "http://localhost:25118", "http://localhost:25119", "http://localhost:25120", "http://localhost:25121", "http://localhost:25122", "http://localhost:25123"})
+       kc.SetServiceRoots(roots)
 
        // "foo" acbd18db4cc2f85cedef654fccc4a4d8
-       foo_shuffle := []string{"http://localhost:25116", "http://localhost:25120", "http://localhost:25119", "http://localhost:25122", "http://localhost:25108", "http://localhost:25114", "http://localhost:25112", "http://localhost:25107", "http://localhost:25118", "http://localhost:25111", "http://localhost:25113", "http://localhost:25121", "http://localhost:25110", "http://localhost:25117", "http://localhost:25109", "http://localhost:25115", "http://localhost:25123"}
+       foo_shuffle := []string{"http://localhost:4", "http://localhost:1", "http://localhost:3", "http://localhost:2"}
        c.Check(kc.shuffledServiceRoots("acbd18db4cc2f85cedef654fccc4a4d8"), DeepEquals, foo_shuffle)
 
        // "bar" 37b51d194a7513e45b56f6524f2d51f2
-       bar_shuffle := []string{"http://localhost:25108", "http://localhost:25112", "http://localhost:25119", "http://localhost:25107", "http://localhost:25110", "http://localhost:25116", "http://localhost:25122", "http://localhost:25120", "http://localhost:25121", "http://localhost:25117", "http://localhost:25111", "http://localhost:25123", "http://localhost:25118", "http://localhost:25113", "http://localhost:25114", "http://localhost:25115", "http://localhost:25109"}
+       bar_shuffle := []string{"http://localhost:3", "http://localhost:2", "http://localhost:4", "http://localhost:1"}
        c.Check(kc.shuffledServiceRoots("37b51d194a7513e45b56f6524f2d51f2"), DeepEquals, bar_shuffle)
 }
 
@@ -273,19 +280,19 @@ func (s *StandaloneSuite) TestPutB(c *C) {
                hash,
                "abc123",
                "foo",
-               make(chan string, 2)}
+               make(chan string, 5)}
 
        arv, _ := arvadosclient.MakeArvadosClient()
        kc, _ := MakeKeepClient(&arv)
 
        kc.Want_replicas = 2
        arv.ApiToken = "abc123"
-       service_roots := make([]string, 5)
+       service_roots := make(map[string]string)
 
        ks := RunSomeFakeKeepServers(st, 5, 2990)
 
        for i := 0; i < len(ks); i += 1 {
-               service_roots[i] = ks[i].url
+               service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = ks[i].url
                defer ks[i].listener.Close()
        }
 
@@ -315,19 +322,19 @@ func (s *StandaloneSuite) TestPutHR(c *C) {
                hash,
                "abc123",
                "foo",
-               make(chan string, 2)}
+               make(chan string, 5)}
 
        arv, _ := arvadosclient.MakeArvadosClient()
        kc, _ := MakeKeepClient(&arv)
 
        kc.Want_replicas = 2
        arv.ApiToken = "abc123"
-       service_roots := make([]string, 5)
+       service_roots := make(map[string]string)
 
        ks := RunSomeFakeKeepServers(st, 5, 2990)
 
        for i := 0; i < len(ks); i += 1 {
-               service_roots[i] = ks[i].url
+               service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = ks[i].url
                defer ks[i].listener.Close()
        }
 
@@ -366,7 +373,7 @@ func (s *StandaloneSuite) TestPutWithFail(c *C) {
                hash,
                "abc123",
                "foo",
-               make(chan string, 2)}
+               make(chan string, 4)}
 
        fh := FailHandler{
                make(chan string, 1)}
@@ -376,17 +383,17 @@ func (s *StandaloneSuite) TestPutWithFail(c *C) {
 
        kc.Want_replicas = 2
        arv.ApiToken = "abc123"
-       service_roots := make([]string, 5)
+       service_roots := make(map[string]string)
 
        ks1 := RunSomeFakeKeepServers(st, 4, 2990)
        ks2 := RunSomeFakeKeepServers(fh, 1, 2995)
 
        for i, k := range ks1 {
-               service_roots[i] = k.url
+               service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
                defer k.listener.Close()
        }
        for i, k := range ks2 {
-               service_roots[len(ks1)+i] = k.url
+               service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
                defer k.listener.Close()
        }
 
@@ -425,29 +432,27 @@ func (s *StandaloneSuite) TestPutWithTooManyFail(c *C) {
 
        kc.Want_replicas = 2
        arv.ApiToken = "abc123"
-       service_roots := make([]string, 5)
+       service_roots := make(map[string]string)
 
        ks1 := RunSomeFakeKeepServers(st, 1, 2990)
        ks2 := RunSomeFakeKeepServers(fh, 4, 2991)
 
        for i, k := range ks1 {
-               service_roots[i] = k.url
+               service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
                defer k.listener.Close()
        }
        for i, k := range ks2 {
-               service_roots[len(ks1)+i] = k.url
+               service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
                defer k.listener.Close()
        }
 
        kc.SetServiceRoots(service_roots)
 
-       shuff := kc.shuffledServiceRoots(fmt.Sprintf("%x", md5.Sum([]byte("foo"))))
-
        _, replicas, err := kc.PutB([]byte("foo"))
 
        c.Check(err, Equals, InsufficientReplicasError)
        c.Check(replicas, Equals, 1)
-       c.Check(<-st.handled, Equals, shuff[1])
+       c.Check(<-st.handled, Matches, ".*2990")
 
        log.Printf("TestPutWithTooManyFail done")
 }
@@ -483,7 +488,7 @@ func (s *StandaloneSuite) TestGet(c *C) {
        arv, err := arvadosclient.MakeArvadosClient()
        kc, _ := MakeKeepClient(&arv)
        arv.ApiToken = "abc123"
-       kc.SetServiceRoots([]string{url})
+       kc.SetServiceRoots(map[string]string{"x":url})
 
        r, n, url2, err := kc.Get(hash)
        defer r.Close()
@@ -509,7 +514,7 @@ func (s *StandaloneSuite) TestGetFail(c *C) {
        arv, err := arvadosclient.MakeArvadosClient()
        kc, _ := MakeKeepClient(&arv)
        arv.ApiToken = "abc123"
-       kc.SetServiceRoots([]string{url})
+       kc.SetServiceRoots(map[string]string{"x":url})
 
        r, n, url2, err := kc.Get(hash)
        c.Check(err, Equals, BlockNotFound)
@@ -539,7 +544,7 @@ func (s *StandaloneSuite) TestChecksum(c *C) {
        arv, err := arvadosclient.MakeArvadosClient()
        kc, _ := MakeKeepClient(&arv)
        arv.ApiToken = "abc123"
-       kc.SetServiceRoots([]string{url})
+       kc.SetServiceRoots(map[string]string{"x":url})
 
        r, n, _, err := kc.Get(barhash)
        _, err = ioutil.ReadAll(r)
@@ -557,50 +562,59 @@ func (s *StandaloneSuite) TestChecksum(c *C) {
 }
 
 func (s *StandaloneSuite) TestGetWithFailures(c *C) {
-
-       hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+       content := []byte("waz")
+       hash := fmt.Sprintf("%x", md5.Sum(content))
 
        fh := FailHandler{
-               make(chan string, 1)}
+               make(chan string, 4)}
 
        st := StubGetHandler{
                c,
                hash,
                "abc123",
-               []byte("foo")}
+               content}
 
        arv, err := arvadosclient.MakeArvadosClient()
        kc, _ := MakeKeepClient(&arv)
        arv.ApiToken = "abc123"
-       service_roots := make([]string, 5)
+       service_roots := make(map[string]string)
 
        ks1 := RunSomeFakeKeepServers(st, 1, 2990)
        ks2 := RunSomeFakeKeepServers(fh, 4, 2991)
 
        for i, k := range ks1 {
-               service_roots[i] = k.url
+               service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
                defer k.listener.Close()
        }
        for i, k := range ks2 {
-               service_roots[len(ks1)+i] = k.url
+               service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
                defer k.listener.Close()
        }
 
        kc.SetServiceRoots(service_roots)
 
+       // This test works only if one of the failing services is
+       // attempted before the succeeding service. Otherwise,
+       // <-fh.handled below will just hang! (Probe order depends on
+       // the choice of block content "waz" and the UUIDs of the fake
+       // servers, so we just tried different strings until we found
+       // an example that passes this Assert.)
+       c.Assert(NewRootSorter(service_roots, hash).GetSortedRoots()[0], Matches, ".*299[1-4]")
+
        r, n, url2, err := kc.Get(hash)
+
        <-fh.handled
        c.Check(err, Equals, nil)
        c.Check(n, Equals, int64(3))
        c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks1[0].url, hash))
 
-       content, err2 := ioutil.ReadAll(r)
+       read_content, err2 := ioutil.ReadAll(r)
        c.Check(err2, Equals, nil)
-       c.Check(content, DeepEquals, []byte("foo"))
+       c.Check(read_content, DeepEquals, content)
 }
 
 func (s *ServerRequiredSuite) TestPutGetHead(c *C) {
-       os.Setenv("ARVADOS_API_HOST", "localhost:3001")
+       os.Setenv("ARVADOS_API_HOST", "localhost:3000")
        os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
        os.Setenv("ARVADOS_API_HOST_INSECURE", "true")
 
@@ -659,12 +673,12 @@ func (s *StandaloneSuite) TestPutProxy(c *C) {
        kc.Want_replicas = 2
        kc.Using_proxy = true
        arv.ApiToken = "abc123"
-       service_roots := make([]string, 1)
+       service_roots := make(map[string]string)
 
        ks1 := RunSomeFakeKeepServers(st, 1, 2990)
 
        for i, k := range ks1 {
-               service_roots[i] = k.url
+               service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
                defer k.listener.Close()
        }
 
@@ -690,12 +704,12 @@ func (s *StandaloneSuite) TestPutProxyInsufficientReplicas(c *C) {
        kc.Want_replicas = 3
        kc.Using_proxy = true
        arv.ApiToken = "abc123"
-       service_roots := make([]string, 1)
+       service_roots := make(map[string]string)
 
        ks1 := RunSomeFakeKeepServers(st, 1, 2990)
 
        for i, k := range ks1 {
-               service_roots[i] = k.url
+               service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
                defer k.listener.Close()
        }
        kc.SetServiceRoots(service_roots)