X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/0e5dfbbcefcb462617ad89ef116a5b18f9e3ebc8..c3d4f8a585202ec58df5506934b698039c200b68:/sdk/go/keepclient/keepclient_test.go diff --git a/sdk/go/keepclient/keepclient_test.go b/sdk/go/keepclient/keepclient_test.go index 5f9915d95c..f0da600c24 100644 --- a/sdk/go/keepclient/keepclient_test.go +++ b/sdk/go/keepclient/keepclient_test.go @@ -1,11 +1,11 @@ package keepclient import ( - "git.curoverse.com/arvados.git/sdk/go/arvadosclient" - "git.curoverse.com/arvados.git/sdk/go/streamer" "crypto/md5" - "flag" "fmt" + "git.curoverse.com/arvados.git/sdk/go/arvadosclient" + "git.curoverse.com/arvados.git/sdk/go/arvadostest" + "git.curoverse.com/arvados.git/sdk/go/streamer" . "gopkg.in/check.v1" "io" "io/ioutil" @@ -13,8 +13,9 @@ import ( "net" "net/http" "os" - "os/exec" + "strings" "testing" + "time" ) // Gocheck boilerplate @@ -26,8 +27,6 @@ func Test(t *testing.T) { var _ = Suite(&ServerRequiredSuite{}) var _ = Suite(&StandaloneSuite{}) -var no_server = flag.Bool("no-server", false, "Skip 'ServerRequireSuite'") - // Tests that require the Keep server running type ServerRequiredSuite struct{} @@ -40,58 +39,44 @@ func pythonDir() string { } func (s *ServerRequiredSuite) SetUpSuite(c *C) { - if *no_server { - c.Skip("Skipping tests that require server") - return - } - os.Chdir(pythonDir()) - { - cmd := exec.Command("python", "run_test_server.py", "start") - stderr, err := cmd.StderrPipe() - if err != nil { - log.Fatalf("Setting up stderr pipe: %s", err) - } - go io.Copy(os.Stderr, stderr) - if err := cmd.Run(); err != nil { - panic(fmt.Sprintf("'python run_test_server.py start' returned error %s", err)) - } - } - { - cmd := exec.Command("python", "run_test_server.py", "start_keep") - stderr, err := cmd.StderrPipe() - if err != nil { - log.Fatalf("Setting up stderr pipe: %s", err) - } - go io.Copy(os.Stderr, stderr) - if err := cmd.Run(); err != nil { - panic(fmt.Sprintf("'python run_test_server.py start_keep' returned error %s", err)) - } - } + arvadostest.StartAPI() + arvadostest.StartKeep(2, false) } func (s *ServerRequiredSuite) TearDownSuite(c *C) { - os.Chdir(pythonDir()) - exec.Command("python", "run_test_server.py", "stop_keep").Run() - exec.Command("python", "run_test_server.py", "stop").Run() + arvadostest.StopKeep(2) + arvadostest.StopAPI() } func (s *ServerRequiredSuite) TestMakeKeepClient(c *C) { - os.Setenv("ARVADOS_API_HOST", "localhost:3000") - os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h") - os.Setenv("ARVADOS_API_HOST_INSECURE", "true") - arv, err := arvadosclient.MakeArvadosClient() c.Assert(err, Equals, nil) - kc, err := MakeKeepClient(&arv) + kc, err := MakeKeepClient(arv) c.Assert(err, Equals, nil) - c.Check(len(kc.ServiceRoots()), Equals, 2) - for _, root := range kc.ServiceRoots() { - c.Check(root, Matches, "http://localhost:2510[\\d]") + c.Check(len(kc.LocalRoots()), Equals, 2) + for _, root := range kc.LocalRoots() { + c.Check(root, Matches, "http://localhost:\\d+") } } +func (s *ServerRequiredSuite) TestDefaultReplications(c *C) { + arv, err := arvadosclient.MakeArvadosClient() + c.Assert(err, Equals, nil) + + kc, err := MakeKeepClient(arv) + c.Assert(kc.Want_replicas, Equals, 2) + + arv.DiscoveryDoc["defaultCollectionReplication"] = 3.0 + kc, err = MakeKeepClient(arv) + c.Assert(kc.Want_replicas, Equals, 3) + + arv.DiscoveryDoc["defaultCollectionReplication"] = 1.0 + kc, err = MakeKeepClient(arv) + c.Assert(kc.Want_replicas, Equals, 1) +} + type StubPutHandler struct { c *C expectPath string @@ -100,44 +85,42 @@ type StubPutHandler struct { handled chan string } -func (this StubPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { - this.c.Check(req.URL.Path, Equals, "/"+this.expectPath) - this.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", this.expectApiToken)) +func (sph StubPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { + sph.c.Check(req.URL.Path, Equals, "/"+sph.expectPath) + sph.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sph.expectApiToken)) body, err := ioutil.ReadAll(req.Body) - this.c.Check(err, Equals, nil) - this.c.Check(body, DeepEquals, []byte(this.expectBody)) + sph.c.Check(err, Equals, nil) + sph.c.Check(body, DeepEquals, []byte(sph.expectBody)) resp.WriteHeader(200) - this.handled <- fmt.Sprintf("http://%s", req.Host) + sph.handled <- fmt.Sprintf("http://%s", req.Host) } -func RunBogusKeepServer(st http.Handler, port int) (listener net.Listener, url string) { +func RunFakeKeepServer(st http.Handler) (ks KeepServer) { var err error - listener, err = net.ListenTCP("tcp", &net.TCPAddr{Port: port}) + ks.listener, err = net.ListenTCP("tcp", &net.TCPAddr{Port: 0}) if err != nil { - panic(fmt.Sprintf("Could not listen on tcp port %v", port)) + panic(fmt.Sprintf("Could not listen on any port")) } - - url = fmt.Sprintf("http://localhost:%d", port) - - go http.Serve(listener, st) - return listener, url + ks.url = fmt.Sprintf("http://%s", ks.listener.Addr().String()) + go http.Serve(ks.listener, st) + return } -func UploadToStubHelper(c *C, st http.Handler, f func(KeepClient, string, +func UploadToStubHelper(c *C, st http.Handler, f func(*KeepClient, string, io.ReadCloser, io.WriteCloser, chan uploadStatus)) { - listener, url := RunBogusKeepServer(st, 2990) - defer listener.Close() + ks := RunFakeKeepServer(st) + defer ks.listener.Close() arv, _ := arvadosclient.MakeArvadosClient() arv.ApiToken = "abc123" - kc, _ := MakeKeepClient(&arv) + kc, _ := MakeKeepClient(arv) reader, writer := io.Pipe() upload_status := make(chan uploadStatus) - f(kc, url, reader, writer, upload_status) + f(kc, ks.url, reader, writer, upload_status) } func (s *StandaloneSuite) TestUploadToStubKeepServer(c *C) { @@ -151,10 +134,9 @@ func (s *StandaloneSuite) TestUploadToStubKeepServer(c *C) { make(chan string)} UploadToStubHelper(c, st, - func(kc KeepClient, url string, reader io.ReadCloser, - writer io.WriteCloser, upload_status chan uploadStatus) { + func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, upload_status chan uploadStatus) { - go kc.uploadToKeepServer(url, st.expectPath, reader, upload_status, int64(len("foo"))) + go kc.uploadToKeepServer(url, st.expectPath, reader, upload_status, int64(len("foo")), 0) writer.Write([]byte("foo")) writer.Close() @@ -163,13 +145,9 @@ func (s *StandaloneSuite) TestUploadToStubKeepServer(c *C) { status := <-upload_status c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, ""}) }) - - log.Printf("TestUploadToStubKeepServer done") } func (s *StandaloneSuite) TestUploadToStubKeepServerBufferReader(c *C) { - log.Printf("TestUploadToStubKeepServerBufferReader") - st := StubPutHandler{ c, "acbd18db4cc2f85cedef654fccc4a4d8", @@ -178,7 +156,7 @@ func (s *StandaloneSuite) TestUploadToStubKeepServerBufferReader(c *C) { make(chan string)} UploadToStubHelper(c, st, - func(kc KeepClient, url string, reader io.ReadCloser, + func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, upload_status chan uploadStatus) { tr := streamer.AsyncStreamFromReader(512, reader) @@ -186,7 +164,7 @@ func (s *StandaloneSuite) TestUploadToStubKeepServerBufferReader(c *C) { br1 := tr.MakeStreamReader() - go kc.uploadToKeepServer(url, st.expectPath, br1, upload_status, 3) + go kc.uploadToKeepServer(url, st.expectPath, br1, upload_status, 3, 0) writer.Write([]byte("foo")) writer.Close() @@ -196,32 +174,53 @@ func (s *StandaloneSuite) TestUploadToStubKeepServerBufferReader(c *C) { status := <-upload_status c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, ""}) }) - - log.Printf("TestUploadToStubKeepServerBufferReader done") } type FailHandler struct { handled chan string } -func (this FailHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { +func (fh FailHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { resp.WriteHeader(500) - this.handled <- fmt.Sprintf("http://%s", req.Host) + fh.handled <- fmt.Sprintf("http://%s", req.Host) } -func (s *StandaloneSuite) TestFailedUploadToStubKeepServer(c *C) { - log.Printf("TestFailedUploadToStubKeepServer") +type FailThenSucceedHandler struct { + handled chan string + count int + successhandler StubGetHandler +} + +func (fh *FailThenSucceedHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { + if fh.count == 0 { + resp.WriteHeader(500) + fh.count += 1 + fh.handled <- fmt.Sprintf("http://%s", req.Host) + } else { + fh.successhandler.ServeHTTP(resp, req) + } +} + +type Error404Handler struct { + handled chan string +} +func (fh Error404Handler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { + resp.WriteHeader(404) + fh.handled <- fmt.Sprintf("http://%s", req.Host) +} + +func (s *StandaloneSuite) TestFailedUploadToStubKeepServer(c *C) { st := FailHandler{ make(chan string)} hash := "acbd18db4cc2f85cedef654fccc4a4d8" UploadToStubHelper(c, st, - func(kc KeepClient, url string, reader io.ReadCloser, + func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, upload_status chan uploadStatus) { - go kc.uploadToKeepServer(url, hash, reader, upload_status, 3) + go kc.uploadToKeepServer(url, hash, reader, upload_status, 3, 0) writer.Write([]byte("foo")) writer.Close() @@ -232,7 +231,6 @@ func (s *StandaloneSuite) TestFailedUploadToStubKeepServer(c *C) { c.Check(status.url, Equals, fmt.Sprintf("%s/%s", url, hash)) c.Check(status.statusCode, Equals, 500) }) - log.Printf("TestFailedUploadToStubKeepServer done") } type KeepServer struct { @@ -240,20 +238,17 @@ type KeepServer struct { url string } -func RunSomeFakeKeepServers(st http.Handler, n int, port int) (ks []KeepServer) { +func RunSomeFakeKeepServers(st http.Handler, n int) (ks []KeepServer) { ks = make([]KeepServer, n) for i := 0; i < n; i += 1 { - boguslistener, bogusurl := RunBogusKeepServer(st, port+i) - ks[i] = KeepServer{boguslistener, bogusurl} + ks[i] = RunFakeKeepServer(st) } return ks } func (s *StandaloneSuite) TestPutB(c *C) { - log.Printf("TestPutB") - hash := Md5String("foo") st := StubPutHandler{ @@ -264,25 +259,27 @@ func (s *StandaloneSuite) TestPutB(c *C) { make(chan string, 5)} arv, _ := arvadosclient.MakeArvadosClient() - kc, _ := MakeKeepClient(&arv) + kc, _ := MakeKeepClient(arv) kc.Want_replicas = 2 arv.ApiToken = "abc123" - service_roots := make(map[string]string) + localRoots := make(map[string]string) + writableLocalRoots := make(map[string]string) - ks := RunSomeFakeKeepServers(st, 5, 2990) + ks := RunSomeFakeKeepServers(st, 5) - for i := 0; i < len(ks); i += 1 { - service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = ks[i].url - defer ks[i].listener.Close() + for i, k := range ks { + localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url + writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url + defer k.listener.Close() } - kc.SetServiceRoots(service_roots) + kc.SetServiceRoots(localRoots, writableLocalRoots, nil) kc.PutB([]byte("foo")) shuff := NewRootSorter( - kc.ServiceRoots(), Md5String("foo")).GetSortedRoots() + kc.LocalRoots(), Md5String("foo")).GetSortedRoots() s1 := <-st.handled s2 := <-st.handled @@ -290,13 +287,9 @@ func (s *StandaloneSuite) TestPutB(c *C) { (s1 == shuff[1] && s2 == shuff[0]), Equals, true) - - log.Printf("TestPutB done") } func (s *StandaloneSuite) TestPutHR(c *C) { - log.Printf("TestPutHR") - hash := fmt.Sprintf("%x", md5.Sum([]byte("foo"))) st := StubPutHandler{ @@ -307,20 +300,22 @@ func (s *StandaloneSuite) TestPutHR(c *C) { make(chan string, 5)} arv, _ := arvadosclient.MakeArvadosClient() - kc, _ := MakeKeepClient(&arv) + kc, _ := MakeKeepClient(arv) kc.Want_replicas = 2 arv.ApiToken = "abc123" - service_roots := make(map[string]string) + localRoots := make(map[string]string) + writableLocalRoots := make(map[string]string) - ks := RunSomeFakeKeepServers(st, 5, 2990) + ks := RunSomeFakeKeepServers(st, 5) - for i := 0; i < len(ks); i += 1 { - service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = ks[i].url - defer ks[i].listener.Close() + for i, k := range ks { + localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url + writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url + defer k.listener.Close() } - kc.SetServiceRoots(service_roots) + kc.SetServiceRoots(localRoots, writableLocalRoots, nil) reader, writer := io.Pipe() @@ -331,8 +326,7 @@ func (s *StandaloneSuite) TestPutHR(c *C) { kc.PutHR(hash, reader, 3) - shuff := NewRootSorter(kc.ServiceRoots(), hash).GetSortedRoots() - log.Print(shuff) + shuff := NewRootSorter(kc.LocalRoots(), hash).GetSortedRoots() s1 := <-st.handled s2 := <-st.handled @@ -341,13 +335,9 @@ func (s *StandaloneSuite) TestPutHR(c *C) { (s1 == shuff[1] && s2 == shuff[0]), Equals, true) - - log.Printf("TestPutHR done") } func (s *StandaloneSuite) TestPutWithFail(c *C) { - log.Printf("TestPutWithFail") - hash := fmt.Sprintf("%x", md5.Sum([]byte("foo"))) st := StubPutHandler{ @@ -361,28 +351,32 @@ func (s *StandaloneSuite) TestPutWithFail(c *C) { make(chan string, 1)} arv, err := arvadosclient.MakeArvadosClient() - kc, _ := MakeKeepClient(&arv) + kc, _ := MakeKeepClient(arv) kc.Want_replicas = 2 arv.ApiToken = "abc123" - service_roots := make(map[string]string) + localRoots := make(map[string]string) + writableLocalRoots := make(map[string]string) - ks1 := RunSomeFakeKeepServers(st, 4, 2990) - ks2 := RunSomeFakeKeepServers(fh, 1, 2995) + ks1 := RunSomeFakeKeepServers(st, 4) + ks2 := RunSomeFakeKeepServers(fh, 1) for i, k := range ks1 { - service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url + localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url + writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url defer k.listener.Close() } for i, k := range ks2 { - service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url + localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url + writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url defer k.listener.Close() } - kc.SetServiceRoots(service_roots) + kc.SetServiceRoots(localRoots, writableLocalRoots, nil) shuff := NewRootSorter( - kc.ServiceRoots(), Md5String("foo")).GetSortedRoots() + kc.LocalRoots(), Md5String("foo")).GetSortedRoots() + c.Logf("%+v", shuff) phash, replicas, err := kc.PutB([]byte("foo")) @@ -402,8 +396,6 @@ func (s *StandaloneSuite) TestPutWithFail(c *C) { } func (s *StandaloneSuite) TestPutWithTooManyFail(c *C) { - log.Printf("TestPutWithTooManyFail") - hash := fmt.Sprintf("%x", md5.Sum([]byte("foo"))) st := StubPutHandler{ @@ -417,79 +409,121 @@ func (s *StandaloneSuite) TestPutWithTooManyFail(c *C) { make(chan string, 4)} arv, err := arvadosclient.MakeArvadosClient() - kc, _ := MakeKeepClient(&arv) + kc, _ := MakeKeepClient(arv) kc.Want_replicas = 2 + kc.Retries = 0 arv.ApiToken = "abc123" - service_roots := make(map[string]string) + localRoots := make(map[string]string) + writableLocalRoots := make(map[string]string) - ks1 := RunSomeFakeKeepServers(st, 1, 2990) - ks2 := RunSomeFakeKeepServers(fh, 4, 2991) + ks1 := RunSomeFakeKeepServers(st, 1) + ks2 := RunSomeFakeKeepServers(fh, 4) for i, k := range ks1 { - service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url + localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url + writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url defer k.listener.Close() } for i, k := range ks2 { - service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url + localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url + writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url defer k.listener.Close() } - kc.SetServiceRoots(service_roots) + kc.SetServiceRoots(localRoots, writableLocalRoots, nil) _, replicas, err := kc.PutB([]byte("foo")) c.Check(err, Equals, InsufficientReplicasError) c.Check(replicas, Equals, 1) - c.Check(<-st.handled, Matches, ".*2990") - - log.Printf("TestPutWithTooManyFail done") + c.Check(<-st.handled, Equals, ks1[0].url) } type StubGetHandler struct { c *C expectPath string expectApiToken string - returnBody []byte + httpStatus int + body []byte } -func (this StubGetHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { - this.c.Check(req.URL.Path, Equals, "/"+this.expectPath) - this.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", this.expectApiToken)) - resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(this.returnBody))) - resp.Write(this.returnBody) +func (sgh StubGetHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { + sgh.c.Check(req.URL.Path, Equals, "/"+sgh.expectPath) + sgh.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sgh.expectApiToken)) + resp.WriteHeader(sgh.httpStatus) + resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(sgh.body))) + resp.Write(sgh.body) } func (s *StandaloneSuite) TestGet(c *C) { - log.Printf("TestGet") - hash := fmt.Sprintf("%x", md5.Sum([]byte("foo"))) st := StubGetHandler{ c, hash, "abc123", + http.StatusOK, []byte("foo")} - listener, url := RunBogusKeepServer(st, 2990) - defer listener.Close() + ks := RunFakeKeepServer(st) + defer ks.listener.Close() arv, err := arvadosclient.MakeArvadosClient() - kc, _ := MakeKeepClient(&arv) + kc, _ := MakeKeepClient(arv) arv.ApiToken = "abc123" - kc.SetServiceRoots(map[string]string{"x":url}) + kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) r, n, url2, err := kc.Get(hash) defer r.Close() c.Check(err, Equals, nil) c.Check(n, Equals, int64(3)) - c.Check(url2, Equals, fmt.Sprintf("%s/%s", url, hash)) + c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks.url, hash)) content, err2 := ioutil.ReadAll(r) c.Check(err2, Equals, nil) c.Check(content, DeepEquals, []byte("foo")) +} + +func (s *StandaloneSuite) TestGet404(c *C) { + hash := fmt.Sprintf("%x", md5.Sum([]byte("foo"))) + + st := Error404Handler{make(chan string, 1)} + + ks := RunFakeKeepServer(st) + defer ks.listener.Close() + + arv, err := arvadosclient.MakeArvadosClient() + kc, _ := MakeKeepClient(arv) + arv.ApiToken = "abc123" + kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) + + r, n, url2, err := kc.Get(hash) + c.Check(err, Equals, BlockNotFound) + c.Check(n, Equals, int64(0)) + c.Check(url2, Equals, "") + c.Check(r, Equals, nil) +} + +func (s *StandaloneSuite) TestGetEmptyBlock(c *C) { + st := Error404Handler{make(chan string, 1)} + + ks := RunFakeKeepServer(st) + defer ks.listener.Close() - log.Printf("TestGet done") + arv, err := arvadosclient.MakeArvadosClient() + kc, _ := MakeKeepClient(arv) + arv.ApiToken = "abc123" + kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) + + r, n, url2, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e+0") + c.Check(err, IsNil) + c.Check(n, Equals, int64(0)) + c.Check(url2, Equals, "") + c.Assert(r, NotNil) + buf, err := ioutil.ReadAll(r) + c.Check(err, IsNil) + c.Check(buf, DeepEquals, []byte{}) } func (s *StandaloneSuite) TestGetFail(c *C) { @@ -497,21 +531,203 @@ func (s *StandaloneSuite) TestGetFail(c *C) { st := FailHandler{make(chan string, 1)} - listener, url := RunBogusKeepServer(st, 2990) - defer listener.Close() + ks := RunFakeKeepServer(st) + defer ks.listener.Close() arv, err := arvadosclient.MakeArvadosClient() - kc, _ := MakeKeepClient(&arv) + kc, _ := MakeKeepClient(arv) arv.ApiToken = "abc123" - kc.SetServiceRoots(map[string]string{"x":url}) + kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) + kc.Retries = 0 r, n, url2, err := kc.Get(hash) - c.Check(err, Equals, BlockNotFound) + errNotFound, _ := err.(*ErrNotFound) + c.Check(errNotFound, NotNil) + c.Check(strings.Contains(errNotFound.Error(), "HTTP 500"), Equals, true) + c.Check(errNotFound.Temporary(), Equals, true) + c.Check(n, Equals, int64(0)) + c.Check(url2, Equals, "") + c.Check(r, Equals, nil) +} + +func (s *StandaloneSuite) TestGetFailRetry(c *C) { + hash := fmt.Sprintf("%x", md5.Sum([]byte("foo"))) + + st := &FailThenSucceedHandler{make(chan string, 1), 0, + StubGetHandler{ + c, + hash, + "abc123", + http.StatusOK, + []byte("foo")}} + + ks := RunFakeKeepServer(st) + defer ks.listener.Close() + + arv, err := arvadosclient.MakeArvadosClient() + kc, _ := MakeKeepClient(arv) + arv.ApiToken = "abc123" + kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) + + r, n, url2, err := kc.Get(hash) + defer r.Close() + c.Check(err, Equals, nil) + c.Check(n, Equals, int64(3)) + c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks.url, hash)) + + content, err2 := ioutil.ReadAll(r) + c.Check(err2, Equals, nil) + c.Check(content, DeepEquals, []byte("foo")) +} + +func (s *StandaloneSuite) TestGetNetError(c *C) { + hash := fmt.Sprintf("%x", md5.Sum([]byte("foo"))) + + arv, err := arvadosclient.MakeArvadosClient() + kc, _ := MakeKeepClient(arv) + arv.ApiToken = "abc123" + kc.SetServiceRoots(map[string]string{"x": "http://localhost:62222"}, nil, nil) + + r, n, url2, err := kc.Get(hash) + errNotFound, _ := err.(*ErrNotFound) + c.Check(errNotFound, NotNil) + c.Check(strings.Contains(errNotFound.Error(), "connection refused"), Equals, true) + c.Check(errNotFound.Temporary(), Equals, true) c.Check(n, Equals, int64(0)) c.Check(url2, Equals, "") c.Check(r, Equals, nil) } +func (s *StandaloneSuite) TestGetWithServiceHint(c *C) { + uuid := "zzzzz-bi6l4-123451234512345" + hash := fmt.Sprintf("%x", md5.Sum([]byte("foo"))) + + // This one shouldn't be used: + ks0 := RunFakeKeepServer(StubGetHandler{ + c, + "error if used", + "abc123", + http.StatusOK, + []byte("foo")}) + defer ks0.listener.Close() + // This one should be used: + ks := RunFakeKeepServer(StubGetHandler{ + c, + hash + "+K@" + uuid, + "abc123", + http.StatusOK, + []byte("foo")}) + defer ks.listener.Close() + + arv, err := arvadosclient.MakeArvadosClient() + kc, _ := MakeKeepClient(arv) + arv.ApiToken = "abc123" + kc.SetServiceRoots( + map[string]string{"x": ks0.url}, + nil, + map[string]string{uuid: ks.url}) + + r, n, uri, err := kc.Get(hash + "+K@" + uuid) + defer r.Close() + c.Check(err, Equals, nil) + c.Check(n, Equals, int64(3)) + c.Check(uri, Equals, fmt.Sprintf("%s/%s", ks.url, hash+"+K@"+uuid)) + + content, err := ioutil.ReadAll(r) + c.Check(err, Equals, nil) + c.Check(content, DeepEquals, []byte("foo")) +} + +// Use a service hint to fetch from a local disk service, overriding +// rendezvous probe order. +func (s *StandaloneSuite) TestGetWithLocalServiceHint(c *C) { + uuid := "zzzzz-bi6l4-zzzzzzzzzzzzzzz" + hash := fmt.Sprintf("%x", md5.Sum([]byte("foo"))) + + // This one shouldn't be used, although it appears first in + // rendezvous probe order: + ks0 := RunFakeKeepServer(StubGetHandler{ + c, + "error if used", + "abc123", + http.StatusOK, + []byte("foo")}) + defer ks0.listener.Close() + // This one should be used: + ks := RunFakeKeepServer(StubGetHandler{ + c, + hash + "+K@" + uuid, + "abc123", + http.StatusOK, + []byte("foo")}) + defer ks.listener.Close() + + arv, err := arvadosclient.MakeArvadosClient() + kc, _ := MakeKeepClient(arv) + arv.ApiToken = "abc123" + kc.SetServiceRoots( + map[string]string{ + "zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url, + "zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url, + "zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url, + uuid: ks.url}, + nil, + map[string]string{ + "zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url, + "zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url, + "zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url, + uuid: ks.url}, + ) + + r, n, uri, err := kc.Get(hash + "+K@" + uuid) + defer r.Close() + c.Check(err, Equals, nil) + c.Check(n, Equals, int64(3)) + c.Check(uri, Equals, fmt.Sprintf("%s/%s", ks.url, hash+"+K@"+uuid)) + + content, err := ioutil.ReadAll(r) + c.Check(err, Equals, nil) + c.Check(content, DeepEquals, []byte("foo")) +} + +func (s *StandaloneSuite) TestGetWithServiceHintFailoverToLocals(c *C) { + uuid := "zzzzz-bi6l4-123451234512345" + hash := fmt.Sprintf("%x", md5.Sum([]byte("foo"))) + + ksLocal := RunFakeKeepServer(StubGetHandler{ + c, + hash + "+K@" + uuid, + "abc123", + http.StatusOK, + []byte("foo")}) + defer ksLocal.listener.Close() + ksGateway := RunFakeKeepServer(StubGetHandler{ + c, + hash + "+K@" + uuid, + "abc123", + http.StatusInternalServerError, + []byte("Error")}) + defer ksGateway.listener.Close() + + arv, err := arvadosclient.MakeArvadosClient() + kc, _ := MakeKeepClient(arv) + arv.ApiToken = "abc123" + kc.SetServiceRoots( + map[string]string{"zzzzz-bi6l4-keepdisk0000000": ksLocal.url}, + nil, + map[string]string{uuid: ksGateway.url}) + + r, n, uri, err := kc.Get(hash + "+K@" + uuid) + c.Assert(err, Equals, nil) + defer r.Close() + c.Check(n, Equals, int64(3)) + c.Check(uri, Equals, fmt.Sprintf("%s/%s", ksLocal.url, hash+"+K@"+uuid)) + + content, err := ioutil.ReadAll(r) + c.Check(err, Equals, nil) + c.Check(content, DeepEquals, []byte("foo")) +} + type BarHandler struct { handled chan string } @@ -527,13 +743,13 @@ func (s *StandaloneSuite) TestChecksum(c *C) { st := BarHandler{make(chan string, 1)} - listener, url := RunBogusKeepServer(st, 2990) - defer listener.Close() + ks := RunFakeKeepServer(st) + defer ks.listener.Close() arv, err := arvadosclient.MakeArvadosClient() - kc, _ := MakeKeepClient(&arv) + kc, _ := MakeKeepClient(arv) arv.ApiToken = "abc123" - kc.SetServiceRoots(map[string]string{"x":url}) + kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) r, n, _, err := kc.Get(barhash) _, err = ioutil.ReadAll(r) @@ -554,33 +770,38 @@ func (s *StandaloneSuite) TestGetWithFailures(c *C) { content := []byte("waz") hash := fmt.Sprintf("%x", md5.Sum(content)) - fh := FailHandler{ + fh := Error404Handler{ make(chan string, 4)} st := StubGetHandler{ c, hash, "abc123", + http.StatusOK, content} arv, err := arvadosclient.MakeArvadosClient() - kc, _ := MakeKeepClient(&arv) + kc, _ := MakeKeepClient(arv) arv.ApiToken = "abc123" - service_roots := make(map[string]string) + localRoots := make(map[string]string) + writableLocalRoots := make(map[string]string) - ks1 := RunSomeFakeKeepServers(st, 1, 2990) - ks2 := RunSomeFakeKeepServers(fh, 4, 2991) + ks1 := RunSomeFakeKeepServers(st, 1) + ks2 := RunSomeFakeKeepServers(fh, 4) for i, k := range ks1 { - service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url + localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url + writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url defer k.listener.Close() } for i, k := range ks2 { - service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url + localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url + writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url defer k.listener.Close() } - kc.SetServiceRoots(service_roots) + kc.SetServiceRoots(localRoots, writableLocalRoots, nil) + kc.Retries = 0 // This test works only if one of the failing services is // attempted before the succeeding service. Otherwise, @@ -588,7 +809,7 @@ func (s *StandaloneSuite) TestGetWithFailures(c *C) { // the choice of block content "waz" and the UUIDs of the fake // servers, so we just tried different strings until we found // an example that passes this Assert.) - c.Assert(NewRootSorter(service_roots, hash).GetSortedRoots()[0], Matches, ".*299[1-4]") + c.Assert(NewRootSorter(localRoots, hash).GetSortedRoots()[0], Not(Equals), ks1[0].url) r, n, url2, err := kc.Get(hash) @@ -603,13 +824,10 @@ func (s *StandaloneSuite) TestGetWithFailures(c *C) { } func (s *ServerRequiredSuite) TestPutGetHead(c *C) { - os.Setenv("ARVADOS_API_HOST", "localhost:3000") - os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h") - os.Setenv("ARVADOS_API_HOST_INSECURE", "true") content := []byte("TestPutGetHead") arv, err := arvadosclient.MakeArvadosClient() - kc, err := MakeKeepClient(&arv) + kc, err := MakeKeepClient(arv) c.Assert(err, Equals, nil) hash := fmt.Sprintf("%x", md5.Sum(content)) @@ -621,7 +839,7 @@ func (s *ServerRequiredSuite) TestPutGetHead(c *C) { } { hash2, replicas, err := kc.PutB(content) - c.Check(hash2, Equals, fmt.Sprintf("%s+%d", hash, len(content))) + c.Check(hash2, Matches, fmt.Sprintf(`%s\+%d\b.*`, hash, len(content))) c.Check(replicas, Equals, 2) c.Check(err, Equals, nil) } @@ -629,7 +847,7 @@ func (s *ServerRequiredSuite) TestPutGetHead(c *C) { r, n, url2, err := kc.Get(hash) c.Check(err, Equals, nil) c.Check(n, Equals, int64(len(content))) - c.Check(url2, Equals, fmt.Sprintf("http://localhost:25108/%s", hash)) + c.Check(url2, Matches, fmt.Sprintf("http://localhost:\\d+/%s", hash)) read_content, err2 := ioutil.ReadAll(r) c.Check(err2, Equals, nil) @@ -639,7 +857,7 @@ func (s *ServerRequiredSuite) TestPutGetHead(c *C) { n, url2, err := kc.Ask(hash) c.Check(err, Equals, nil) c.Check(n, Equals, int64(len(content))) - c.Check(url2, Equals, fmt.Sprintf("http://localhost:25108/%s", hash)) + c.Check(url2, Matches, fmt.Sprintf("http://localhost:\\d+/%s", hash)) } } @@ -653,71 +871,397 @@ func (this StubProxyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Reque } func (s *StandaloneSuite) TestPutProxy(c *C) { - log.Printf("TestPutProxy") - st := StubProxyHandler{make(chan string, 1)} arv, err := arvadosclient.MakeArvadosClient() - kc, _ := MakeKeepClient(&arv) + kc, _ := MakeKeepClient(arv) kc.Want_replicas = 2 - kc.Using_proxy = true arv.ApiToken = "abc123" - service_roots := make(map[string]string) + localRoots := make(map[string]string) + writableLocalRoots := make(map[string]string) - ks1 := RunSomeFakeKeepServers(st, 1, 2990) + ks1 := RunSomeFakeKeepServers(st, 1) for i, k := range ks1 { - service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url + localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url + writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url defer k.listener.Close() } - kc.SetServiceRoots(service_roots) + kc.SetServiceRoots(localRoots, writableLocalRoots, nil) _, replicas, err := kc.PutB([]byte("foo")) <-st.handled c.Check(err, Equals, nil) c.Check(replicas, Equals, 2) - - log.Printf("TestPutProxy done") } func (s *StandaloneSuite) TestPutProxyInsufficientReplicas(c *C) { - log.Printf("TestPutProxy") - st := StubProxyHandler{make(chan string, 1)} arv, err := arvadosclient.MakeArvadosClient() - kc, _ := MakeKeepClient(&arv) + kc, _ := MakeKeepClient(arv) kc.Want_replicas = 3 - kc.Using_proxy = true arv.ApiToken = "abc123" - service_roots := make(map[string]string) + localRoots := make(map[string]string) + writableLocalRoots := make(map[string]string) - ks1 := RunSomeFakeKeepServers(st, 1, 2990) + ks1 := RunSomeFakeKeepServers(st, 1) for i, k := range ks1 { - service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url + localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url + writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url defer k.listener.Close() } - kc.SetServiceRoots(service_roots) + kc.SetServiceRoots(localRoots, writableLocalRoots, nil) _, replicas, err := kc.PutB([]byte("foo")) <-st.handled c.Check(err, Equals, InsufficientReplicasError) c.Check(replicas, Equals, 2) - - log.Printf("TestPutProxy done") } func (s *StandaloneSuite) TestMakeLocator(c *C) { - l := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce+3+Aabcde@12345678") + l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce+3+Aabcde@12345678") + c.Check(err, Equals, nil) + c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce") + c.Check(l.Size, Equals, 3) + c.Check(l.Hints, DeepEquals, []string{"3", "Aabcde@12345678"}) +} +func (s *StandaloneSuite) TestMakeLocatorNoHints(c *C) { + l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce") + c.Check(err, Equals, nil) + c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce") + c.Check(l.Size, Equals, -1) + c.Check(l.Hints, DeepEquals, []string{}) +} + +func (s *StandaloneSuite) TestMakeLocatorNoSizeHint(c *C) { + l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce+Aabcde@12345678") + c.Check(err, Equals, nil) + c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce") + c.Check(l.Size, Equals, -1) + c.Check(l.Hints, DeepEquals, []string{"Aabcde@12345678"}) +} + +func (s *StandaloneSuite) TestMakeLocatorPreservesUnrecognizedHints(c *C) { + str := "91f372a266fe2bf2823cb8ec7fda31ce+3+Unknown+Kzzzzz+Afoobar" + l, err := MakeLocator(str) + c.Check(err, Equals, nil) c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce") c.Check(l.Size, Equals, 3) - c.Check(l.Signature, Equals, "abcde") - c.Check(l.Timestamp, Equals, "12345678") + c.Check(l.Hints, DeepEquals, []string{"3", "Unknown", "Kzzzzz", "Afoobar"}) + c.Check(l.String(), Equals, str) +} + +func (s *StandaloneSuite) TestMakeLocatorInvalidInput(c *C) { + _, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31c") + c.Check(err, Equals, InvalidLocatorError) +} + +func (s *StandaloneSuite) TestPutBWant2ReplicasWithOnlyOneWritableLocalRoot(c *C) { + hash := Md5String("foo") + + st := StubPutHandler{ + c, + hash, + "abc123", + "foo", + make(chan string, 5)} + + arv, _ := arvadosclient.MakeArvadosClient() + kc, _ := MakeKeepClient(arv) + + kc.Want_replicas = 2 + arv.ApiToken = "abc123" + localRoots := make(map[string]string) + writableLocalRoots := make(map[string]string) + + ks := RunSomeFakeKeepServers(st, 5) + + for i, k := range ks { + localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url + if i == 0 { + writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url + } + defer k.listener.Close() + } + + kc.SetServiceRoots(localRoots, writableLocalRoots, nil) + + _, replicas, err := kc.PutB([]byte("foo")) + + c.Check(err, Equals, InsufficientReplicasError) + c.Check(replicas, Equals, 1) + + c.Check(<-st.handled, Equals, localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", 0)]) +} + +func (s *StandaloneSuite) TestPutBWithNoWritableLocalRoots(c *C) { + hash := Md5String("foo") + + st := StubPutHandler{ + c, + hash, + "abc123", + "foo", + make(chan string, 5)} + + arv, _ := arvadosclient.MakeArvadosClient() + kc, _ := MakeKeepClient(arv) + + kc.Want_replicas = 2 + arv.ApiToken = "abc123" + localRoots := make(map[string]string) + writableLocalRoots := make(map[string]string) + + ks := RunSomeFakeKeepServers(st, 5) + + for i, k := range ks { + localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url + defer k.listener.Close() + } + + kc.SetServiceRoots(localRoots, writableLocalRoots, nil) + + _, replicas, err := kc.PutB([]byte("foo")) + + c.Check(err, Equals, InsufficientReplicasError) + c.Check(replicas, Equals, 0) +} + +type StubGetIndexHandler struct { + c *C + expectPath string + expectAPIToken string + httpStatus int + body []byte +} + +func (h StubGetIndexHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { + h.c.Check(req.URL.Path, Equals, h.expectPath) + h.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", h.expectAPIToken)) + resp.WriteHeader(h.httpStatus) + resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(h.body))) + resp.Write(h.body) +} + +func (s *StandaloneSuite) TestGetIndexWithNoPrefix(c *C) { + hash := fmt.Sprintf("%x", md5.Sum([]byte("foo"))) + + st := StubGetIndexHandler{ + c, + "/index", + "abc123", + http.StatusOK, + []byte(hash + "+3 1443559274\n\n")} + + ks := RunFakeKeepServer(st) + defer ks.listener.Close() + + arv, err := arvadosclient.MakeArvadosClient() + kc, _ := MakeKeepClient(arv) + arv.ApiToken = "abc123" + kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) + + r, err := kc.GetIndex("x", "") + c.Check(err, Equals, nil) + + content, err2 := ioutil.ReadAll(r) + c.Check(err2, Equals, nil) + c.Check(content, DeepEquals, st.body[0:len(st.body)-1]) +} + +func (s *StandaloneSuite) TestGetIndexWithPrefix(c *C) { + hash := fmt.Sprintf("%x", md5.Sum([]byte("foo"))) + + st := StubGetIndexHandler{ + c, + "/index/" + hash[0:3], + "abc123", + http.StatusOK, + []byte(hash + "+3 1443559274\n\n")} + + ks := RunFakeKeepServer(st) + defer ks.listener.Close() + + arv, err := arvadosclient.MakeArvadosClient() + kc, _ := MakeKeepClient(arv) + arv.ApiToken = "abc123" + kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) + + r, err := kc.GetIndex("x", hash[0:3]) + c.Check(err, Equals, nil) + + content, err2 := ioutil.ReadAll(r) + c.Check(err2, Equals, nil) + c.Check(content, DeepEquals, st.body[0:len(st.body)-1]) +} + +func (s *StandaloneSuite) TestGetIndexIncomplete(c *C) { + hash := fmt.Sprintf("%x", md5.Sum([]byte("foo"))) + + st := StubGetIndexHandler{ + c, + "/index/" + hash[0:3], + "abc123", + http.StatusOK, + []byte(hash)} + + ks := RunFakeKeepServer(st) + defer ks.listener.Close() + + arv, err := arvadosclient.MakeArvadosClient() + kc, _ := MakeKeepClient(arv) + arv.ApiToken = "abc123" + kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) + + _, err = kc.GetIndex("x", hash[0:3]) + c.Check(err, Equals, ErrIncompleteIndex) +} + +func (s *StandaloneSuite) TestGetIndexWithNoSuchServer(c *C) { + hash := fmt.Sprintf("%x", md5.Sum([]byte("foo"))) + + st := StubGetIndexHandler{ + c, + "/index/" + hash[0:3], + "abc123", + http.StatusOK, + []byte(hash)} + + ks := RunFakeKeepServer(st) + defer ks.listener.Close() + + arv, err := arvadosclient.MakeArvadosClient() + kc, _ := MakeKeepClient(arv) + arv.ApiToken = "abc123" + kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) + + _, err = kc.GetIndex("y", hash[0:3]) + c.Check(err, Equals, ErrNoSuchKeepServer) +} + +func (s *StandaloneSuite) TestGetIndexWithNoSuchPrefix(c *C) { + st := StubGetIndexHandler{ + c, + "/index/abcd", + "abc123", + http.StatusOK, + []byte("\n")} + + ks := RunFakeKeepServer(st) + defer ks.listener.Close() + + arv, err := arvadosclient.MakeArvadosClient() + kc, _ := MakeKeepClient(arv) + arv.ApiToken = "abc123" + kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) + + r, err := kc.GetIndex("x", "abcd") + c.Check(err, Equals, nil) + + content, err2 := ioutil.ReadAll(r) + c.Check(err2, Equals, nil) + c.Check(content, DeepEquals, st.body[0:len(st.body)-1]) +} + +type FailThenSucceedPutHandler struct { + handled chan string + count int + successhandler StubPutHandler +} + +func (h *FailThenSucceedPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { + if h.count == 0 { + resp.WriteHeader(500) + h.count += 1 + h.handled <- fmt.Sprintf("http://%s", req.Host) + } else { + h.successhandler.ServeHTTP(resp, req) + } +} + +func (s *StandaloneSuite) TestPutBRetry(c *C) { + st := &FailThenSucceedPutHandler{make(chan string, 1), 0, + StubPutHandler{ + c, + Md5String("foo"), + "abc123", + "foo", + make(chan string, 5)}} + + arv, _ := arvadosclient.MakeArvadosClient() + kc, _ := MakeKeepClient(arv) + + kc.Want_replicas = 2 + arv.ApiToken = "abc123" + localRoots := make(map[string]string) + writableLocalRoots := make(map[string]string) + + ks := RunSomeFakeKeepServers(st, 2) + + for i, k := range ks { + localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url + writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url + defer k.listener.Close() + } + + kc.SetServiceRoots(localRoots, writableLocalRoots, nil) + + hash, replicas, err := kc.PutB([]byte("foo")) + + c.Check(err, Equals, nil) + c.Check(hash, Equals, "") + c.Check(replicas, Equals, 2) +} + +func (s *ServerRequiredSuite) TestMakeKeepClientWithNonDiskTypeService(c *C) { + arv, err := arvadosclient.MakeArvadosClient() + c.Assert(err, Equals, nil) + + // Add an additional "testblobstore" keepservice + blobKeepService := make(arvadosclient.Dict) + err = arv.Create("keep_services", + arvadosclient.Dict{"keep_service": arvadosclient.Dict{ + "service_host": "localhost", + "service_port": "21321", + "service_type": "testblobstore"}}, + &blobKeepService) + c.Assert(err, Equals, nil) + defer func() { arv.Delete("keep_services", blobKeepService["uuid"].(string), nil, nil) }() + + // Make a keepclient and ensure that the testblobstore is included + kc, err := MakeKeepClient(arv) + c.Assert(err, Equals, nil) + + // verify kc.LocalRoots + c.Check(len(kc.LocalRoots()), Equals, 3) + for _, root := range kc.LocalRoots() { + c.Check(root, Matches, "http://localhost:\\d+") + } + c.Assert(kc.LocalRoots()[blobKeepService["uuid"].(string)], Not(Equals), "") + + // verify kc.GatewayRoots + c.Check(len(kc.GatewayRoots()), Equals, 3) + for _, root := range kc.GatewayRoots() { + c.Check(root, Matches, "http://localhost:\\d+") + } + c.Assert(kc.GatewayRoots()[blobKeepService["uuid"].(string)], Not(Equals), "") + + // verify kc.WritableLocalRoots + c.Check(len(kc.WritableLocalRoots()), Equals, 3) + for _, root := range kc.WritableLocalRoots() { + c.Check(root, Matches, "http://localhost:\\d+") + } + c.Assert(kc.WritableLocalRoots()[blobKeepService["uuid"].(string)], Not(Equals), "") + + c.Assert(kc.replicasPerService, Equals, 0) + c.Assert(kc.foundNonDiskSvc, Equals, true) + c.Assert(kc.Client.Timeout, Equals, 300*time.Second) }