7 "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
8 "git.curoverse.com/arvados.git/sdk/go/arvadostest"
9 "git.curoverse.com/arvados.git/sdk/go/streamer"
22 // Gocheck boilerplate
23 func Test(t *testing.T) {
27 // Gocheck boilerplate
28 var _ = Suite(&ServerRequiredSuite{})
29 var _ = Suite(&StandaloneSuite{})
31 var no_server = flag.Bool("no-server", false, "Skip 'ServerRequireSuite'")
33 // Tests that require the Keep server running
34 type ServerRequiredSuite struct{}
37 type StandaloneSuite struct{}
39 func pythonDir() string {
41 return fmt.Sprintf("%s/../../python/tests", cwd)
44 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
46 c.Skip("Skipping tests that require server")
49 arvadostest.StartAPI()
50 arvadostest.StartKeep(2, false)
53 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
57 arvadostest.StopKeep(2)
61 func (s *ServerRequiredSuite) TestMakeKeepClient(c *C) {
62 arv, err := arvadosclient.MakeArvadosClient()
63 c.Assert(err, Equals, nil)
65 kc, err := MakeKeepClient(&arv)
67 c.Assert(err, Equals, nil)
68 c.Check(len(kc.LocalRoots()), Equals, 2)
69 for _, root := range kc.LocalRoots() {
70 c.Check(root, Matches, "http://localhost:\\d+")
74 func (s *ServerRequiredSuite) TestDefaultReplications(c *C) {
75 arv, err := arvadosclient.MakeArvadosClient()
76 c.Assert(err, Equals, nil)
78 kc, err := MakeKeepClient(&arv)
79 c.Assert(kc.Want_replicas, Equals, 2)
81 arv.DiscoveryDoc["defaultCollectionReplication"] = 3.0
82 kc, err = MakeKeepClient(&arv)
83 c.Assert(kc.Want_replicas, Equals, 3)
85 arv.DiscoveryDoc["defaultCollectionReplication"] = 1.0
86 kc, err = MakeKeepClient(&arv)
87 c.Assert(kc.Want_replicas, Equals, 1)
90 type StubPutHandler struct {
98 func (sph StubPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
99 sph.c.Check(req.URL.Path, Equals, "/"+sph.expectPath)
100 sph.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sph.expectApiToken))
101 body, err := ioutil.ReadAll(req.Body)
102 sph.c.Check(err, Equals, nil)
103 sph.c.Check(body, DeepEquals, []byte(sph.expectBody))
104 resp.WriteHeader(200)
105 sph.handled <- fmt.Sprintf("http://%s", req.Host)
108 func RunFakeKeepServer(st http.Handler) (ks KeepServer) {
110 ks.listener, err = net.ListenTCP("tcp", &net.TCPAddr{Port: 0})
112 panic(fmt.Sprintf("Could not listen on any port"))
114 ks.url = fmt.Sprintf("http://%s", ks.listener.Addr().String())
115 go http.Serve(ks.listener, st)
119 func UploadToStubHelper(c *C, st http.Handler, f func(*KeepClient, string,
120 io.ReadCloser, io.WriteCloser, chan uploadStatus)) {
122 ks := RunFakeKeepServer(st)
123 defer ks.listener.Close()
125 arv, _ := arvadosclient.MakeArvadosClient()
126 arv.ApiToken = "abc123"
128 kc, _ := MakeKeepClient(&arv)
130 reader, writer := io.Pipe()
131 upload_status := make(chan uploadStatus)
133 f(kc, ks.url, reader, writer, upload_status)
136 func (s *StandaloneSuite) TestUploadToStubKeepServer(c *C) {
137 log.Printf("TestUploadToStubKeepServer")
139 st := StubPutHandler{
141 "acbd18db4cc2f85cedef654fccc4a4d8",
146 UploadToStubHelper(c, st,
147 func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, upload_status chan uploadStatus) {
149 go kc.uploadToKeepServer(url, st.expectPath, reader, upload_status, int64(len("foo")), 0)
151 writer.Write([]byte("foo"))
155 status := <-upload_status
156 c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, ""})
160 func (s *StandaloneSuite) TestUploadToStubKeepServerBufferReader(c *C) {
161 st := StubPutHandler{
163 "acbd18db4cc2f85cedef654fccc4a4d8",
168 UploadToStubHelper(c, st,
169 func(kc *KeepClient, url string, reader io.ReadCloser,
170 writer io.WriteCloser, upload_status chan uploadStatus) {
172 tr := streamer.AsyncStreamFromReader(512, reader)
175 br1 := tr.MakeStreamReader()
177 go kc.uploadToKeepServer(url, st.expectPath, br1, upload_status, 3, 0)
179 writer.Write([]byte("foo"))
184 status := <-upload_status
185 c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, ""})
189 type FailHandler struct {
193 func (fh FailHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
194 resp.WriteHeader(500)
195 fh.handled <- fmt.Sprintf("http://%s", req.Host)
198 type FailThenSucceedHandler struct {
201 successhandler StubGetHandler
204 func (fh *FailThenSucceedHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
206 resp.WriteHeader(500)
208 fh.handled <- fmt.Sprintf("http://%s", req.Host)
210 fh.successhandler.ServeHTTP(resp, req)
214 type Error404Handler struct {
218 func (fh Error404Handler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
219 resp.WriteHeader(404)
220 fh.handled <- fmt.Sprintf("http://%s", req.Host)
223 func (s *StandaloneSuite) TestFailedUploadToStubKeepServer(c *C) {
227 hash := "acbd18db4cc2f85cedef654fccc4a4d8"
229 UploadToStubHelper(c, st,
230 func(kc *KeepClient, url string, reader io.ReadCloser,
231 writer io.WriteCloser, upload_status chan uploadStatus) {
233 go kc.uploadToKeepServer(url, hash, reader, upload_status, 3, 0)
235 writer.Write([]byte("foo"))
240 status := <-upload_status
241 c.Check(status.url, Equals, fmt.Sprintf("%s/%s", url, hash))
242 c.Check(status.statusCode, Equals, 500)
246 type KeepServer struct {
247 listener net.Listener
251 func RunSomeFakeKeepServers(st http.Handler, n int) (ks []KeepServer) {
252 ks = make([]KeepServer, n)
254 for i := 0; i < n; i += 1 {
255 ks[i] = RunFakeKeepServer(st)
261 func (s *StandaloneSuite) TestPutB(c *C) {
262 hash := Md5String("foo")
264 st := StubPutHandler{
269 make(chan string, 5)}
271 arv, _ := arvadosclient.MakeArvadosClient()
272 kc, _ := MakeKeepClient(&arv)
275 arv.ApiToken = "abc123"
276 localRoots := make(map[string]string)
277 writableLocalRoots := make(map[string]string)
279 ks := RunSomeFakeKeepServers(st, 5)
281 for i, k := range ks {
282 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
283 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
284 defer k.listener.Close()
287 kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
289 kc.PutB([]byte("foo"))
291 shuff := NewRootSorter(
292 kc.LocalRoots(), Md5String("foo")).GetSortedRoots()
296 c.Check((s1 == shuff[0] && s2 == shuff[1]) ||
297 (s1 == shuff[1] && s2 == shuff[0]),
302 func (s *StandaloneSuite) TestPutHR(c *C) {
303 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
305 st := StubPutHandler{
310 make(chan string, 5)}
312 arv, _ := arvadosclient.MakeArvadosClient()
313 kc, _ := MakeKeepClient(&arv)
316 arv.ApiToken = "abc123"
317 localRoots := make(map[string]string)
318 writableLocalRoots := make(map[string]string)
320 ks := RunSomeFakeKeepServers(st, 5)
322 for i, k := range ks {
323 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
324 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
325 defer k.listener.Close()
328 kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
330 reader, writer := io.Pipe()
333 writer.Write([]byte("foo"))
337 kc.PutHR(hash, reader, 3)
339 shuff := NewRootSorter(kc.LocalRoots(), hash).GetSortedRoots()
344 c.Check((s1 == shuff[0] && s2 == shuff[1]) ||
345 (s1 == shuff[1] && s2 == shuff[0]),
350 func (s *StandaloneSuite) TestPutWithFail(c *C) {
351 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
353 st := StubPutHandler{
358 make(chan string, 4)}
361 make(chan string, 1)}
363 arv, err := arvadosclient.MakeArvadosClient()
364 kc, _ := MakeKeepClient(&arv)
367 arv.ApiToken = "abc123"
368 localRoots := make(map[string]string)
369 writableLocalRoots := make(map[string]string)
371 ks1 := RunSomeFakeKeepServers(st, 4)
372 ks2 := RunSomeFakeKeepServers(fh, 1)
374 for i, k := range ks1 {
375 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
376 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
377 defer k.listener.Close()
379 for i, k := range ks2 {
380 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
381 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
382 defer k.listener.Close()
385 kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
387 shuff := NewRootSorter(
388 kc.LocalRoots(), Md5String("foo")).GetSortedRoots()
391 phash, replicas, err := kc.PutB([]byte("foo"))
395 c.Check(err, Equals, nil)
396 c.Check(phash, Equals, "")
397 c.Check(replicas, Equals, 2)
402 c.Check((s1 == shuff[1] && s2 == shuff[2]) ||
403 (s1 == shuff[2] && s2 == shuff[1]),
408 func (s *StandaloneSuite) TestPutWithTooManyFail(c *C) {
409 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
411 st := StubPutHandler{
416 make(chan string, 1)}
419 make(chan string, 4)}
421 arv, err := arvadosclient.MakeArvadosClient()
422 kc, _ := MakeKeepClient(&arv)
426 arv.ApiToken = "abc123"
427 localRoots := make(map[string]string)
428 writableLocalRoots := make(map[string]string)
430 ks1 := RunSomeFakeKeepServers(st, 1)
431 ks2 := RunSomeFakeKeepServers(fh, 4)
433 for i, k := range ks1 {
434 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
435 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
436 defer k.listener.Close()
438 for i, k := range ks2 {
439 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
440 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
441 defer k.listener.Close()
444 kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
446 _, replicas, err := kc.PutB([]byte("foo"))
448 c.Check(err, Equals, InsufficientReplicasError)
449 c.Check(replicas, Equals, 1)
450 c.Check(<-st.handled, Equals, ks1[0].url)
453 type StubGetHandler struct {
456 expectApiToken string
461 func (sgh StubGetHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
462 sgh.c.Check(req.URL.Path, Equals, "/"+sgh.expectPath)
463 sgh.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sgh.expectApiToken))
464 resp.WriteHeader(sgh.httpStatus)
465 resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(sgh.body)))
469 func (s *StandaloneSuite) TestGet(c *C) {
470 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
472 st := StubGetHandler{
479 ks := RunFakeKeepServer(st)
480 defer ks.listener.Close()
482 arv, err := arvadosclient.MakeArvadosClient()
483 kc, _ := MakeKeepClient(&arv)
484 arv.ApiToken = "abc123"
485 kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
487 r, n, url2, err := kc.Get(hash)
489 c.Check(err, Equals, nil)
490 c.Check(n, Equals, int64(3))
491 c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks.url, hash))
493 content, err2 := ioutil.ReadAll(r)
494 c.Check(err2, Equals, nil)
495 c.Check(content, DeepEquals, []byte("foo"))
498 func (s *StandaloneSuite) TestGet404(c *C) {
499 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
501 st := Error404Handler{make(chan string, 1)}
503 ks := RunFakeKeepServer(st)
504 defer ks.listener.Close()
506 arv, err := arvadosclient.MakeArvadosClient()
507 kc, _ := MakeKeepClient(&arv)
508 arv.ApiToken = "abc123"
509 kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
511 r, n, url2, err := kc.Get(hash)
512 c.Check(err, Equals, BlockNotFound)
513 c.Check(n, Equals, int64(0))
514 c.Check(url2, Equals, "")
515 c.Check(r, Equals, nil)
518 func (s *StandaloneSuite) TestGetFail(c *C) {
519 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
521 st := FailHandler{make(chan string, 1)}
523 ks := RunFakeKeepServer(st)
524 defer ks.listener.Close()
526 arv, err := arvadosclient.MakeArvadosClient()
527 kc, _ := MakeKeepClient(&arv)
528 arv.ApiToken = "abc123"
529 kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
532 r, n, url2, err := kc.Get(hash)
533 errNotFound, _ := err.(*ErrNotFound)
534 c.Check(errNotFound, NotNil)
535 c.Check(strings.Contains(errNotFound.Error(), "HTTP 500"), Equals, true)
536 c.Check(errNotFound.Temporary(), Equals, true)
537 c.Check(n, Equals, int64(0))
538 c.Check(url2, Equals, "")
539 c.Check(r, Equals, nil)
542 func (s *StandaloneSuite) TestGetFailRetry(c *C) {
543 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
545 st := &FailThenSucceedHandler{make(chan string, 1), 0,
553 ks := RunFakeKeepServer(st)
554 defer ks.listener.Close()
556 arv, err := arvadosclient.MakeArvadosClient()
557 kc, _ := MakeKeepClient(&arv)
558 arv.ApiToken = "abc123"
559 kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
561 r, n, url2, err := kc.Get(hash)
563 c.Check(err, Equals, nil)
564 c.Check(n, Equals, int64(3))
565 c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks.url, hash))
567 content, err2 := ioutil.ReadAll(r)
568 c.Check(err2, Equals, nil)
569 c.Check(content, DeepEquals, []byte("foo"))
572 func (s *StandaloneSuite) TestGetNetError(c *C) {
573 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
575 arv, err := arvadosclient.MakeArvadosClient()
576 kc, _ := MakeKeepClient(&arv)
577 arv.ApiToken = "abc123"
578 kc.SetServiceRoots(map[string]string{"x": "http://localhost:62222"}, nil, nil)
580 r, n, url2, err := kc.Get(hash)
581 errNotFound, _ := err.(*ErrNotFound)
582 c.Check(errNotFound, NotNil)
583 c.Check(strings.Contains(errNotFound.Error(), "connection refused"), Equals, true)
584 c.Check(errNotFound.Temporary(), Equals, true)
585 c.Check(n, Equals, int64(0))
586 c.Check(url2, Equals, "")
587 c.Check(r, Equals, nil)
590 func (s *StandaloneSuite) TestGetWithServiceHint(c *C) {
591 uuid := "zzzzz-bi6l4-123451234512345"
592 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
594 // This one shouldn't be used:
595 ks0 := RunFakeKeepServer(StubGetHandler{
601 defer ks0.listener.Close()
602 // This one should be used:
603 ks := RunFakeKeepServer(StubGetHandler{
609 defer ks.listener.Close()
611 arv, err := arvadosclient.MakeArvadosClient()
612 kc, _ := MakeKeepClient(&arv)
613 arv.ApiToken = "abc123"
615 map[string]string{"x": ks0.url},
617 map[string]string{uuid: ks.url})
619 r, n, uri, err := kc.Get(hash + "+K@" + uuid)
621 c.Check(err, Equals, nil)
622 c.Check(n, Equals, int64(3))
623 c.Check(uri, Equals, fmt.Sprintf("%s/%s", ks.url, hash+"+K@"+uuid))
625 content, err := ioutil.ReadAll(r)
626 c.Check(err, Equals, nil)
627 c.Check(content, DeepEquals, []byte("foo"))
630 // Use a service hint to fetch from a local disk service, overriding
631 // rendezvous probe order.
632 func (s *StandaloneSuite) TestGetWithLocalServiceHint(c *C) {
633 uuid := "zzzzz-bi6l4-zzzzzzzzzzzzzzz"
634 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
636 // This one shouldn't be used, although it appears first in
637 // rendezvous probe order:
638 ks0 := RunFakeKeepServer(StubGetHandler{
644 defer ks0.listener.Close()
645 // This one should be used:
646 ks := RunFakeKeepServer(StubGetHandler{
652 defer ks.listener.Close()
654 arv, err := arvadosclient.MakeArvadosClient()
655 kc, _ := MakeKeepClient(&arv)
656 arv.ApiToken = "abc123"
659 "zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url,
660 "zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url,
661 "zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url,
665 "zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url,
666 "zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url,
667 "zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url,
671 r, n, uri, err := kc.Get(hash + "+K@" + uuid)
673 c.Check(err, Equals, nil)
674 c.Check(n, Equals, int64(3))
675 c.Check(uri, Equals, fmt.Sprintf("%s/%s", ks.url, hash+"+K@"+uuid))
677 content, err := ioutil.ReadAll(r)
678 c.Check(err, Equals, nil)
679 c.Check(content, DeepEquals, []byte("foo"))
682 func (s *StandaloneSuite) TestGetWithServiceHintFailoverToLocals(c *C) {
683 uuid := "zzzzz-bi6l4-123451234512345"
684 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
686 ksLocal := RunFakeKeepServer(StubGetHandler{
692 defer ksLocal.listener.Close()
693 ksGateway := RunFakeKeepServer(StubGetHandler{
697 http.StatusInternalServerError,
699 defer ksGateway.listener.Close()
701 arv, err := arvadosclient.MakeArvadosClient()
702 kc, _ := MakeKeepClient(&arv)
703 arv.ApiToken = "abc123"
705 map[string]string{"zzzzz-bi6l4-keepdisk0000000": ksLocal.url},
707 map[string]string{uuid: ksGateway.url})
709 r, n, uri, err := kc.Get(hash + "+K@" + uuid)
710 c.Assert(err, Equals, nil)
712 c.Check(n, Equals, int64(3))
713 c.Check(uri, Equals, fmt.Sprintf("%s/%s", ksLocal.url, hash+"+K@"+uuid))
715 content, err := ioutil.ReadAll(r)
716 c.Check(err, Equals, nil)
717 c.Check(content, DeepEquals, []byte("foo"))
720 type BarHandler struct {
724 func (this BarHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
725 resp.Write([]byte("bar"))
726 this.handled <- fmt.Sprintf("http://%s", req.Host)
729 func (s *StandaloneSuite) TestChecksum(c *C) {
730 foohash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
731 barhash := fmt.Sprintf("%x", md5.Sum([]byte("bar")))
733 st := BarHandler{make(chan string, 1)}
735 ks := RunFakeKeepServer(st)
736 defer ks.listener.Close()
738 arv, err := arvadosclient.MakeArvadosClient()
739 kc, _ := MakeKeepClient(&arv)
740 arv.ApiToken = "abc123"
741 kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
743 r, n, _, err := kc.Get(barhash)
744 _, err = ioutil.ReadAll(r)
745 c.Check(n, Equals, int64(3))
746 c.Check(err, Equals, nil)
750 r, n, _, err = kc.Get(foohash)
751 _, err = ioutil.ReadAll(r)
752 c.Check(n, Equals, int64(3))
753 c.Check(err, Equals, BadChecksum)
758 func (s *StandaloneSuite) TestGetWithFailures(c *C) {
759 content := []byte("waz")
760 hash := fmt.Sprintf("%x", md5.Sum(content))
762 fh := Error404Handler{
763 make(chan string, 4)}
765 st := StubGetHandler{
772 arv, err := arvadosclient.MakeArvadosClient()
773 kc, _ := MakeKeepClient(&arv)
774 arv.ApiToken = "abc123"
775 localRoots := make(map[string]string)
776 writableLocalRoots := make(map[string]string)
778 ks1 := RunSomeFakeKeepServers(st, 1)
779 ks2 := RunSomeFakeKeepServers(fh, 4)
781 for i, k := range ks1 {
782 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
783 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
784 defer k.listener.Close()
786 for i, k := range ks2 {
787 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
788 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
789 defer k.listener.Close()
792 kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
795 // This test works only if one of the failing services is
796 // attempted before the succeeding service. Otherwise,
797 // <-fh.handled below will just hang! (Probe order depends on
798 // the choice of block content "waz" and the UUIDs of the fake
799 // servers, so we just tried different strings until we found
800 // an example that passes this Assert.)
801 c.Assert(NewRootSorter(localRoots, hash).GetSortedRoots()[0], Not(Equals), ks1[0].url)
803 r, n, url2, err := kc.Get(hash)
806 c.Check(err, Equals, nil)
807 c.Check(n, Equals, int64(3))
808 c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks1[0].url, hash))
810 read_content, err2 := ioutil.ReadAll(r)
811 c.Check(err2, Equals, nil)
812 c.Check(read_content, DeepEquals, content)
815 func (s *ServerRequiredSuite) TestPutGetHead(c *C) {
816 content := []byte("TestPutGetHead")
818 arv, err := arvadosclient.MakeArvadosClient()
819 kc, err := MakeKeepClient(&arv)
820 c.Assert(err, Equals, nil)
822 hash := fmt.Sprintf("%x", md5.Sum(content))
825 n, _, err := kc.Ask(hash)
826 c.Check(err, Equals, BlockNotFound)
827 c.Check(n, Equals, int64(0))
830 hash2, replicas, err := kc.PutB(content)
831 c.Check(hash2, Matches, fmt.Sprintf(`%s\+%d\b.*`, hash, len(content)))
832 c.Check(replicas, Equals, 2)
833 c.Check(err, Equals, nil)
836 r, n, url2, err := kc.Get(hash)
837 c.Check(err, Equals, nil)
838 c.Check(n, Equals, int64(len(content)))
839 c.Check(url2, Matches, fmt.Sprintf("http://localhost:\\d+/%s", hash))
841 read_content, err2 := ioutil.ReadAll(r)
842 c.Check(err2, Equals, nil)
843 c.Check(read_content, DeepEquals, content)
846 n, url2, err := kc.Ask(hash)
847 c.Check(err, Equals, nil)
848 c.Check(n, Equals, int64(len(content)))
849 c.Check(url2, Matches, fmt.Sprintf("http://localhost:\\d+/%s", hash))
853 type StubProxyHandler struct {
857 func (this StubProxyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
858 resp.Header().Set("X-Keep-Replicas-Stored", "2")
859 this.handled <- fmt.Sprintf("http://%s", req.Host)
862 func (s *StandaloneSuite) TestPutProxy(c *C) {
863 st := StubProxyHandler{make(chan string, 1)}
865 arv, err := arvadosclient.MakeArvadosClient()
866 kc, _ := MakeKeepClient(&arv)
869 arv.ApiToken = "abc123"
870 localRoots := make(map[string]string)
871 writableLocalRoots := make(map[string]string)
873 ks1 := RunSomeFakeKeepServers(st, 1)
875 for i, k := range ks1 {
876 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
877 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
878 defer k.listener.Close()
881 kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
883 _, replicas, err := kc.PutB([]byte("foo"))
886 c.Check(err, Equals, nil)
887 c.Check(replicas, Equals, 2)
890 func (s *StandaloneSuite) TestPutProxyInsufficientReplicas(c *C) {
891 st := StubProxyHandler{make(chan string, 1)}
893 arv, err := arvadosclient.MakeArvadosClient()
894 kc, _ := MakeKeepClient(&arv)
897 arv.ApiToken = "abc123"
898 localRoots := make(map[string]string)
899 writableLocalRoots := make(map[string]string)
901 ks1 := RunSomeFakeKeepServers(st, 1)
903 for i, k := range ks1 {
904 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
905 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
906 defer k.listener.Close()
908 kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
910 _, replicas, err := kc.PutB([]byte("foo"))
913 c.Check(err, Equals, InsufficientReplicasError)
914 c.Check(replicas, Equals, 2)
917 func (s *StandaloneSuite) TestMakeLocator(c *C) {
918 l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce+3+Aabcde@12345678")
919 c.Check(err, Equals, nil)
920 c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
921 c.Check(l.Size, Equals, 3)
922 c.Check(l.Hints, DeepEquals, []string{"3", "Aabcde@12345678"})
925 func (s *StandaloneSuite) TestMakeLocatorNoHints(c *C) {
926 l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce")
927 c.Check(err, Equals, nil)
928 c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
929 c.Check(l.Size, Equals, -1)
930 c.Check(l.Hints, DeepEquals, []string{})
933 func (s *StandaloneSuite) TestMakeLocatorNoSizeHint(c *C) {
934 l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce+Aabcde@12345678")
935 c.Check(err, Equals, nil)
936 c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
937 c.Check(l.Size, Equals, -1)
938 c.Check(l.Hints, DeepEquals, []string{"Aabcde@12345678"})
941 func (s *StandaloneSuite) TestMakeLocatorPreservesUnrecognizedHints(c *C) {
942 str := "91f372a266fe2bf2823cb8ec7fda31ce+3+Unknown+Kzzzzz+Afoobar"
943 l, err := MakeLocator(str)
944 c.Check(err, Equals, nil)
945 c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
946 c.Check(l.Size, Equals, 3)
947 c.Check(l.Hints, DeepEquals, []string{"3", "Unknown", "Kzzzzz", "Afoobar"})
948 c.Check(l.String(), Equals, str)
951 func (s *StandaloneSuite) TestMakeLocatorInvalidInput(c *C) {
952 _, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31c")
953 c.Check(err, Equals, InvalidLocatorError)
956 func (s *StandaloneSuite) TestPutBWant2ReplicasWithOnlyOneWritableLocalRoot(c *C) {
957 hash := Md5String("foo")
959 st := StubPutHandler{
964 make(chan string, 5)}
966 arv, _ := arvadosclient.MakeArvadosClient()
967 kc, _ := MakeKeepClient(&arv)
970 arv.ApiToken = "abc123"
971 localRoots := make(map[string]string)
972 writableLocalRoots := make(map[string]string)
974 ks := RunSomeFakeKeepServers(st, 5)
976 for i, k := range ks {
977 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
979 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
981 defer k.listener.Close()
984 kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
986 _, replicas, err := kc.PutB([]byte("foo"))
988 c.Check(err, Equals, InsufficientReplicasError)
989 c.Check(replicas, Equals, 1)
991 c.Check(<-st.handled, Equals, localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", 0)])
994 func (s *StandaloneSuite) TestPutBWithNoWritableLocalRoots(c *C) {
995 hash := Md5String("foo")
997 st := StubPutHandler{
1002 make(chan string, 5)}
1004 arv, _ := arvadosclient.MakeArvadosClient()
1005 kc, _ := MakeKeepClient(&arv)
1007 kc.Want_replicas = 2
1008 arv.ApiToken = "abc123"
1009 localRoots := make(map[string]string)
1010 writableLocalRoots := make(map[string]string)
1012 ks := RunSomeFakeKeepServers(st, 5)
1014 for i, k := range ks {
1015 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
1016 defer k.listener.Close()
1019 kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
1021 _, replicas, err := kc.PutB([]byte("foo"))
1023 c.Check(err, Equals, InsufficientReplicasError)
1024 c.Check(replicas, Equals, 0)
1027 type StubGetIndexHandler struct {
1030 expectAPIToken string
1035 func (h StubGetIndexHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
1036 h.c.Check(req.URL.Path, Equals, h.expectPath)
1037 h.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", h.expectAPIToken))
1038 resp.WriteHeader(h.httpStatus)
1039 resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(h.body)))
1043 func (s *StandaloneSuite) TestGetIndexWithNoPrefix(c *C) {
1044 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
1046 st := StubGetIndexHandler{
1051 []byte(hash + "+3 1443559274\n\n")}
1053 ks := RunFakeKeepServer(st)
1054 defer ks.listener.Close()
1056 arv, err := arvadosclient.MakeArvadosClient()
1057 kc, _ := MakeKeepClient(&arv)
1058 arv.ApiToken = "abc123"
1059 kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
1061 r, err := kc.GetIndex("x", "")
1062 c.Check(err, Equals, nil)
1064 content, err2 := ioutil.ReadAll(r)
1065 c.Check(err2, Equals, nil)
1066 c.Check(content, DeepEquals, st.body[0:len(st.body)-1])
1069 func (s *StandaloneSuite) TestGetIndexWithPrefix(c *C) {
1070 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
1072 st := StubGetIndexHandler{
1074 "/index/" + hash[0:3],
1077 []byte(hash + "+3 1443559274\n\n")}
1079 ks := RunFakeKeepServer(st)
1080 defer ks.listener.Close()
1082 arv, err := arvadosclient.MakeArvadosClient()
1083 kc, _ := MakeKeepClient(&arv)
1084 arv.ApiToken = "abc123"
1085 kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
1087 r, err := kc.GetIndex("x", hash[0:3])
1088 c.Check(err, Equals, nil)
1090 content, err2 := ioutil.ReadAll(r)
1091 c.Check(err2, Equals, nil)
1092 c.Check(content, DeepEquals, st.body[0:len(st.body)-1])
1095 func (s *StandaloneSuite) TestGetIndexIncomplete(c *C) {
1096 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
1098 st := StubGetIndexHandler{
1100 "/index/" + hash[0:3],
1105 ks := RunFakeKeepServer(st)
1106 defer ks.listener.Close()
1108 arv, err := arvadosclient.MakeArvadosClient()
1109 kc, _ := MakeKeepClient(&arv)
1110 arv.ApiToken = "abc123"
1111 kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
1113 _, err = kc.GetIndex("x", hash[0:3])
1114 c.Check(err, Equals, ErrIncompleteIndex)
1117 func (s *StandaloneSuite) TestGetIndexWithNoSuchServer(c *C) {
1118 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
1120 st := StubGetIndexHandler{
1122 "/index/" + hash[0:3],
1127 ks := RunFakeKeepServer(st)
1128 defer ks.listener.Close()
1130 arv, err := arvadosclient.MakeArvadosClient()
1131 kc, _ := MakeKeepClient(&arv)
1132 arv.ApiToken = "abc123"
1133 kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
1135 _, err = kc.GetIndex("y", hash[0:3])
1136 c.Check(err, Equals, ErrNoSuchKeepServer)
1139 func (s *StandaloneSuite) TestGetIndexWithNoSuchPrefix(c *C) {
1140 st := StubGetIndexHandler{
1147 ks := RunFakeKeepServer(st)
1148 defer ks.listener.Close()
1150 arv, err := arvadosclient.MakeArvadosClient()
1151 kc, _ := MakeKeepClient(&arv)
1152 arv.ApiToken = "abc123"
1153 kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
1155 r, err := kc.GetIndex("x", "abcd")
1156 c.Check(err, Equals, nil)
1158 content, err2 := ioutil.ReadAll(r)
1159 c.Check(err2, Equals, nil)
1160 c.Check(content, DeepEquals, st.body[0:len(st.body)-1])
1163 type FailThenSucceedPutHandler struct {
1166 successhandler StubPutHandler
1169 func (h *FailThenSucceedPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
1171 resp.WriteHeader(500)
1173 h.handled <- fmt.Sprintf("http://%s", req.Host)
1175 h.successhandler.ServeHTTP(resp, req)
1179 func (s *StandaloneSuite) TestPutBRetry(c *C) {
1180 st := &FailThenSucceedPutHandler{make(chan string, 1), 0,
1186 make(chan string, 5)}}
1188 arv, _ := arvadosclient.MakeArvadosClient()
1189 kc, _ := MakeKeepClient(&arv)
1191 kc.Want_replicas = 2
1192 arv.ApiToken = "abc123"
1193 localRoots := make(map[string]string)
1194 writableLocalRoots := make(map[string]string)
1196 ks := RunSomeFakeKeepServers(st, 2)
1198 for i, k := range ks {
1199 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
1200 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
1201 defer k.listener.Close()
1204 kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
1206 hash, replicas, err := kc.PutB([]byte("foo"))
1208 c.Check(err, Equals, nil)
1209 c.Check(hash, Equals, "")
1210 c.Check(replicas, Equals, 2)
1213 func (s *ServerRequiredSuite) TestMakeKeepClientWithNonDiskTypeService(c *C) {
1214 arv, err := arvadosclient.MakeArvadosClient()
1215 c.Assert(err, Equals, nil)
1217 // Add an additional "testblobstore" keepservice
1218 blobKeepService := make(arvadosclient.Dict)
1219 err = arv.Create("keep_services",
1220 arvadosclient.Dict{"keep_service": arvadosclient.Dict{
1221 "service_host": "localhost",
1222 "service_port": "21321",
1223 "service_type": "testblobstore"}},
1225 c.Assert(err, Equals, nil)
1226 defer func() { arv.Delete("keep_services", blobKeepService["uuid"].(string), nil, nil) }()
1228 // Make a keepclient and ensure that the testblobstore is included
1229 kc, err := MakeKeepClient(&arv)
1230 c.Assert(err, Equals, nil)
1232 // verify kc.LocalRoots
1233 c.Check(len(kc.LocalRoots()), Equals, 3)
1234 for _, root := range kc.LocalRoots() {
1235 c.Check(root, Matches, "http://localhost:\\d+")
1237 c.Assert(kc.LocalRoots()[blobKeepService["uuid"].(string)], Not(Equals), "")
1239 // verify kc.GatewayRoots
1240 c.Check(len(kc.GatewayRoots()), Equals, 3)
1241 for _, root := range kc.GatewayRoots() {
1242 c.Check(root, Matches, "http://localhost:\\d+")
1244 c.Assert(kc.GatewayRoots()[blobKeepService["uuid"].(string)], Not(Equals), "")
1246 // verify kc.WritableLocalRoots
1247 c.Check(len(kc.WritableLocalRoots()), Equals, 3)
1248 for _, root := range kc.WritableLocalRoots() {
1249 c.Check(root, Matches, "http://localhost:\\d+")
1251 c.Assert(kc.WritableLocalRoots()[blobKeepService["uuid"].(string)], Not(Equals), "")
1253 c.Assert(kc.replicasPerService, Equals, 0)
1254 c.Assert(kc.foundNonDiskSvc, Equals, true)
1255 c.Assert(kc.Client.Timeout, Equals, 300*time.Second)