X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/03a016b1edff72e698474cd31a887530ea89b530..9e0399b7fcff3ea83ee986b07f60b8b27659d5c9:/sdk/go/keepclient/keepclient_test.go diff --git a/sdk/go/keepclient/keepclient_test.go b/sdk/go/keepclient/keepclient_test.go index 055141cbe8..59c4127240 100644 --- a/sdk/go/keepclient/keepclient_test.go +++ b/sdk/go/keepclient/keepclient_test.go @@ -19,8 +19,8 @@ import ( "testing" "time" - "git.curoverse.com/arvados.git/sdk/go/arvadosclient" - "git.curoverse.com/arvados.git/sdk/go/arvadostest" + "git.arvados.org/arvados.git/sdk/go/arvadosclient" + "git.arvados.org/arvados.git/sdk/go/arvadostest" . "gopkg.in/check.v1" ) @@ -80,28 +80,33 @@ func (s *ServerRequiredSuite) TestDefaultReplications(c *C) { c.Assert(err, Equals, nil) kc, err := MakeKeepClient(arv) + c.Check(err, IsNil) c.Assert(kc.Want_replicas, Equals, 2) arv.DiscoveryDoc["defaultCollectionReplication"] = 3.0 kc, err = MakeKeepClient(arv) + c.Check(err, IsNil) c.Assert(kc.Want_replicas, Equals, 3) arv.DiscoveryDoc["defaultCollectionReplication"] = 1.0 kc, err = MakeKeepClient(arv) + c.Check(err, IsNil) c.Assert(kc.Want_replicas, Equals, 1) } type StubPutHandler struct { - c *C - expectPath string - expectApiToken string - expectBody string - handled chan string + c *C + expectPath string + expectApiToken string + expectBody string + expectStorageClass string + handled chan string } func (sph StubPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { sph.c.Check(req.URL.Path, Equals, "/"+sph.expectPath) sph.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sph.expectApiToken)) + sph.c.Check(req.Header.Get("X-Keep-Storage-Classes"), Equals, sph.expectStorageClass) body, err := ioutil.ReadAll(req.Body) sph.c.Check(err, Equals, nil) sph.c.Check(body, DeepEquals, []byte(sph.expectBody)) @@ -134,9 +139,9 @@ func UploadToStubHelper(c *C, st http.Handler, f func(*KeepClient, string, kc, _ := MakeKeepClient(arv) reader, writer := io.Pipe() - upload_status := make(chan uploadStatus) + uploadStatusChan := make(chan uploadStatus) - f(kc, ks.url, reader, writer, upload_status) + f(kc, ks.url, reader, writer, uploadStatusChan) } func (s *StandaloneSuite) TestUploadToStubKeepServer(c *C) { @@ -147,18 +152,19 @@ func (s *StandaloneSuite) TestUploadToStubKeepServer(c *C) { "acbd18db4cc2f85cedef654fccc4a4d8", "abc123", "foo", + "hot", make(chan string)} UploadToStubHelper(c, st, - func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, upload_status chan uploadStatus) { - - go kc.uploadToKeepServer(url, st.expectPath, reader, upload_status, int64(len("foo")), 0) + func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, uploadStatusChan chan uploadStatus) { + kc.StorageClasses = []string{"hot"} + go kc.uploadToKeepServer(url, st.expectPath, reader, uploadStatusChan, int64(len("foo")), kc.getRequestID()) writer.Write([]byte("foo")) writer.Close() <-st.handled - status := <-upload_status + status := <-uploadStatusChan c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, ""}) }) } @@ -169,15 +175,16 @@ func (s *StandaloneSuite) TestUploadToStubKeepServerBufferReader(c *C) { "acbd18db4cc2f85cedef654fccc4a4d8", "abc123", "foo", + "", make(chan string)} UploadToStubHelper(c, st, - func(kc *KeepClient, url string, _ io.ReadCloser, _ io.WriteCloser, upload_status chan uploadStatus) { - go kc.uploadToKeepServer(url, st.expectPath, bytes.NewBuffer([]byte("foo")), upload_status, 3, 0) + func(kc *KeepClient, url string, _ io.ReadCloser, _ io.WriteCloser, uploadStatusChan chan uploadStatus) { + go kc.uploadToKeepServer(url, st.expectPath, bytes.NewBuffer([]byte("foo")), uploadStatusChan, 3, kc.getRequestID()) <-st.handled - status := <-upload_status + status := <-uploadStatusChan c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, ""}) }) } @@ -194,13 +201,15 @@ func (fh FailHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { type FailThenSucceedHandler struct { handled chan string count int - successhandler StubGetHandler + successhandler http.Handler + reqIDs []string } func (fh *FailThenSucceedHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { + fh.reqIDs = append(fh.reqIDs, req.Header.Get("X-Request-Id")) if fh.count == 0 { resp.WriteHeader(500) - fh.count += 1 + fh.count++ fh.handled <- fmt.Sprintf("http://%s", req.Host) } else { fh.successhandler.ServeHTTP(resp, req) @@ -224,16 +233,16 @@ func (s *StandaloneSuite) TestFailedUploadToStubKeepServer(c *C) { UploadToStubHelper(c, st, func(kc *KeepClient, url string, reader io.ReadCloser, - writer io.WriteCloser, upload_status chan uploadStatus) { + writer io.WriteCloser, uploadStatusChan chan uploadStatus) { - go kc.uploadToKeepServer(url, hash, reader, upload_status, 3, 0) + go kc.uploadToKeepServer(url, hash, reader, uploadStatusChan, 3, kc.getRequestID()) writer.Write([]byte("foo")) writer.Close() <-st.handled - status := <-upload_status + status := <-uploadStatusChan c.Check(status.url, Equals, fmt.Sprintf("%s/%s", url, hash)) c.Check(status.statusCode, Equals, 500) }) @@ -262,6 +271,7 @@ func (s *StandaloneSuite) TestPutB(c *C) { hash, "abc123", "foo", + "", make(chan string, 5)} arv, _ := arvadosclient.MakeArvadosClient() @@ -303,6 +313,7 @@ func (s *StandaloneSuite) TestPutHR(c *C) { hash, "abc123", "foo", + "", make(chan string, 5)} arv, _ := arvadosclient.MakeArvadosClient() @@ -351,12 +362,14 @@ func (s *StandaloneSuite) TestPutWithFail(c *C) { hash, "abc123", "foo", + "", make(chan string, 4)} fh := FailHandler{ make(chan string, 1)} arv, err := arvadosclient.MakeArvadosClient() + c.Check(err, IsNil) kc, _ := MakeKeepClient(arv) kc.Want_replicas = 2 @@ -409,12 +422,14 @@ func (s *StandaloneSuite) TestPutWithTooManyFail(c *C) { hash, "abc123", "foo", + "", make(chan string, 1)} fh := FailHandler{ make(chan string, 4)} arv, err := arvadosclient.MakeArvadosClient() + c.Check(err, IsNil) kc, _ := MakeKeepClient(arv) kc.Want_replicas = 2 @@ -476,6 +491,7 @@ func (s *StandaloneSuite) TestGet(c *C) { defer ks.listener.Close() arv, err := arvadosclient.MakeArvadosClient() + c.Check(err, IsNil) kc, _ := MakeKeepClient(arv) arv.ApiToken = "abc123" kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) @@ -500,6 +516,7 @@ func (s *StandaloneSuite) TestGet404(c *C) { defer ks.listener.Close() arv, err := arvadosclient.MakeArvadosClient() + c.Check(err, IsNil) kc, _ := MakeKeepClient(arv) arv.ApiToken = "abc123" kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) @@ -518,6 +535,7 @@ func (s *StandaloneSuite) TestGetEmptyBlock(c *C) { defer ks.listener.Close() arv, err := arvadosclient.MakeArvadosClient() + c.Check(err, IsNil) kc, _ := MakeKeepClient(arv) arv.ApiToken = "abc123" kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) @@ -541,6 +559,7 @@ func (s *StandaloneSuite) TestGetFail(c *C) { defer ks.listener.Close() arv, err := arvadosclient.MakeArvadosClient() + c.Check(err, IsNil) kc, _ := MakeKeepClient(arv) arv.ApiToken = "abc123" kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) @@ -559,8 +578,9 @@ func (s *StandaloneSuite) TestGetFail(c *C) { func (s *StandaloneSuite) TestGetFailRetry(c *C) { hash := fmt.Sprintf("%x", md5.Sum([]byte("foo"))) - st := &FailThenSucceedHandler{make(chan string, 1), 0, - StubGetHandler{ + st := &FailThenSucceedHandler{ + handled: make(chan string, 1), + successhandler: StubGetHandler{ c, hash, "abc123", @@ -571,6 +591,7 @@ func (s *StandaloneSuite) TestGetFailRetry(c *C) { defer ks.listener.Close() arv, err := arvadosclient.MakeArvadosClient() + c.Check(err, IsNil) kc, _ := MakeKeepClient(arv) arv.ApiToken = "abc123" kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) @@ -584,12 +605,20 @@ func (s *StandaloneSuite) TestGetFailRetry(c *C) { content, err2 := ioutil.ReadAll(r) c.Check(err2, Equals, nil) c.Check(content, DeepEquals, []byte("foo")) + + c.Logf("%q", st.reqIDs) + c.Assert(len(st.reqIDs) > 1, Equals, true) + for _, reqid := range st.reqIDs { + c.Check(reqid, Not(Equals), "") + c.Check(reqid, Equals, st.reqIDs[0]) + } } func (s *StandaloneSuite) TestGetNetError(c *C) { hash := fmt.Sprintf("%x", md5.Sum([]byte("foo"))) arv, err := arvadosclient.MakeArvadosClient() + c.Check(err, IsNil) kc, _ := MakeKeepClient(arv) arv.ApiToken = "abc123" kc.SetServiceRoots(map[string]string{"x": "http://localhost:62222"}, nil, nil) @@ -626,6 +655,7 @@ func (s *StandaloneSuite) TestGetWithServiceHint(c *C) { defer ks.listener.Close() arv, err := arvadosclient.MakeArvadosClient() + c.Check(err, IsNil) kc, _ := MakeKeepClient(arv) arv.ApiToken = "abc123" kc.SetServiceRoots( @@ -669,6 +699,7 @@ func (s *StandaloneSuite) TestGetWithLocalServiceHint(c *C) { defer ks.listener.Close() arv, err := arvadosclient.MakeArvadosClient() + c.Check(err, IsNil) kc, _ := MakeKeepClient(arv) arv.ApiToken = "abc123" kc.SetServiceRoots( @@ -676,13 +707,13 @@ func (s *StandaloneSuite) TestGetWithLocalServiceHint(c *C) { "zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url, "zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url, "zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url, - uuid: ks.url}, + uuid: ks.url}, nil, map[string]string{ "zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url, "zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url, "zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url, - uuid: ks.url}, + uuid: ks.url}, ) r, n, uri, err := kc.Get(hash + "+K@" + uuid) @@ -716,6 +747,7 @@ func (s *StandaloneSuite) TestGetWithServiceHintFailoverToLocals(c *C) { defer ksGateway.listener.Close() arv, err := arvadosclient.MakeArvadosClient() + c.Check(err, IsNil) kc, _ := MakeKeepClient(arv) arv.ApiToken = "abc123" kc.SetServiceRoots( @@ -738,9 +770,9 @@ type BarHandler struct { handled chan string } -func (this BarHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { +func (h BarHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { resp.Write([]byte("bar")) - this.handled <- fmt.Sprintf("http://%s", req.Host) + h.handled <- fmt.Sprintf("http://%s", req.Host) } func (s *StandaloneSuite) TestChecksum(c *C) { @@ -753,11 +785,13 @@ func (s *StandaloneSuite) TestChecksum(c *C) { defer ks.listener.Close() arv, err := arvadosclient.MakeArvadosClient() + c.Check(err, IsNil) kc, _ := MakeKeepClient(arv) arv.ApiToken = "abc123" kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) r, n, _, err := kc.Get(barhash) + c.Check(err, IsNil) _, err = ioutil.ReadAll(r) c.Check(n, Equals, int64(3)) c.Check(err, Equals, nil) @@ -765,6 +799,7 @@ func (s *StandaloneSuite) TestChecksum(c *C) { <-st.handled r, n, _, err = kc.Get(foohash) + c.Check(err, IsNil) _, err = ioutil.ReadAll(r) c.Check(n, Equals, int64(3)) c.Check(err, Equals, BadChecksum) @@ -787,6 +822,7 @@ func (s *StandaloneSuite) TestGetWithFailures(c *C) { content} arv, err := arvadosclient.MakeArvadosClient() + c.Check(err, IsNil) kc, _ := MakeKeepClient(arv) arv.ApiToken = "abc123" localRoots := make(map[string]string) @@ -824,15 +860,16 @@ func (s *StandaloneSuite) TestGetWithFailures(c *C) { c.Check(n, Equals, int64(3)) c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks1[0].url, hash)) - read_content, err2 := ioutil.ReadAll(r) + readContent, err2 := ioutil.ReadAll(r) c.Check(err2, Equals, nil) - c.Check(read_content, DeepEquals, content) + c.Check(readContent, DeepEquals, content) } func (s *ServerRequiredSuite) TestPutGetHead(c *C) { content := []byte("TestPutGetHead") arv, err := arvadosclient.MakeArvadosClient() + c.Check(err, IsNil) kc, err := MakeKeepClient(arv) c.Assert(err, Equals, nil) @@ -855,9 +892,9 @@ func (s *ServerRequiredSuite) TestPutGetHead(c *C) { c.Check(n, Equals, int64(len(content))) c.Check(url2, Matches, fmt.Sprintf("http://localhost:\\d+/%s", hash)) - read_content, err2 := ioutil.ReadAll(r) + readContent, err2 := ioutil.ReadAll(r) c.Check(err2, Equals, nil) - c.Check(read_content, DeepEquals, content) + c.Check(readContent, DeepEquals, content) } { n, url2, err := kc.Ask(hash) @@ -865,21 +902,35 @@ func (s *ServerRequiredSuite) TestPutGetHead(c *C) { c.Check(n, Equals, int64(len(content))) c.Check(url2, Matches, fmt.Sprintf("http://localhost:\\d+/%s", hash)) } + { + loc, err := kc.LocalLocator(hash) + c.Check(err, Equals, nil) + c.Assert(len(loc) >= 32, Equals, true) + c.Check(loc[:32], Equals, hash[:32]) + } + { + content := []byte("the perth county conspiracy") + loc, err := kc.LocalLocator(fmt.Sprintf("%x+%d+Rzaaaa-abcde@12345", md5.Sum(content), len(content))) + c.Check(loc, Equals, "") + c.Check(err, ErrorMatches, `.*HEAD .*\+R.*`) + c.Check(err, ErrorMatches, `.*HTTP 400.*`) + } } type StubProxyHandler struct { handled chan string } -func (this StubProxyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { +func (h StubProxyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { resp.Header().Set("X-Keep-Replicas-Stored", "2") - this.handled <- fmt.Sprintf("http://%s", req.Host) + h.handled <- fmt.Sprintf("http://%s", req.Host) } func (s *StandaloneSuite) TestPutProxy(c *C) { st := StubProxyHandler{make(chan string, 1)} arv, err := arvadosclient.MakeArvadosClient() + c.Check(err, IsNil) kc, _ := MakeKeepClient(arv) kc.Want_replicas = 2 @@ -908,6 +959,7 @@ func (s *StandaloneSuite) TestPutProxyInsufficientReplicas(c *C) { st := StubProxyHandler{make(chan string, 1)} arv, err := arvadosclient.MakeArvadosClient() + c.Check(err, IsNil) kc, _ := MakeKeepClient(arv) kc.Want_replicas = 3 @@ -978,6 +1030,7 @@ func (s *StandaloneSuite) TestPutBWant2ReplicasWithOnlyOneWritableLocalRoot(c *C hash, "abc123", "foo", + "", make(chan string, 5)} arv, _ := arvadosclient.MakeArvadosClient() @@ -1016,6 +1069,7 @@ func (s *StandaloneSuite) TestPutBWithNoWritableLocalRoots(c *C) { hash, "abc123", "foo", + "", make(chan string, 5)} arv, _ := arvadosclient.MakeArvadosClient() @@ -1099,6 +1153,7 @@ func (s *StandaloneSuite) TestGetIndexWithPrefix(c *C) { defer ks.listener.Close() arv, err := arvadosclient.MakeArvadosClient() + c.Check(err, IsNil) kc, _ := MakeKeepClient(arv) arv.ApiToken = "abc123" kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) @@ -1125,6 +1180,7 @@ func (s *StandaloneSuite) TestGetIndexIncomplete(c *C) { defer ks.listener.Close() arv, err := arvadosclient.MakeArvadosClient() + c.Check(err, IsNil) kc, _ := MakeKeepClient(arv) arv.ApiToken = "abc123" kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) @@ -1147,6 +1203,7 @@ func (s *StandaloneSuite) TestGetIndexWithNoSuchServer(c *C) { defer ks.listener.Close() arv, err := arvadosclient.MakeArvadosClient() + c.Check(err, IsNil) kc, _ := MakeKeepClient(arv) arv.ApiToken = "abc123" kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) @@ -1167,6 +1224,7 @@ func (s *StandaloneSuite) TestGetIndexWithNoSuchPrefix(c *C) { defer ks.listener.Close() arv, err := arvadosclient.MakeArvadosClient() + c.Check(err, IsNil) kc, _ := MakeKeepClient(arv) arv.ApiToken = "abc123" kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil) @@ -1179,29 +1237,15 @@ func (s *StandaloneSuite) TestGetIndexWithNoSuchPrefix(c *C) { c.Check(content, DeepEquals, st.body[0:len(st.body)-1]) } -type FailThenSucceedPutHandler struct { - handled chan string - count int - successhandler StubPutHandler -} - -func (h *FailThenSucceedPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { - if h.count == 0 { - resp.WriteHeader(500) - h.count += 1 - h.handled <- fmt.Sprintf("http://%s", req.Host) - } else { - h.successhandler.ServeHTTP(resp, req) - } -} - func (s *StandaloneSuite) TestPutBRetry(c *C) { - st := &FailThenSucceedPutHandler{make(chan string, 1), 0, - StubPutHandler{ + st := &FailThenSucceedHandler{ + handled: make(chan string, 1), + successhandler: StubPutHandler{ c, Md5String("foo"), "abc123", "foo", + "", make(chan string, 5)}} arv, _ := arvadosclient.MakeArvadosClient()