1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: Apache-2.0
22 "git.arvados.org/arvados.git/sdk/go/arvadosclient"
23 "git.arvados.org/arvados.git/sdk/go/arvadostest"
27 // Gocheck boilerplate
28 func Test(t *testing.T) {
32 // Gocheck boilerplate
33 var _ = Suite(&ServerRequiredSuite{})
34 var _ = Suite(&StandaloneSuite{})
36 // Tests that require the Keep server running
37 type ServerRequiredSuite struct{}
40 type StandaloneSuite struct{}
42 func (s *StandaloneSuite) SetUpTest(c *C) {
43 RefreshServiceDiscovery()
46 func pythonDir() string {
48 return fmt.Sprintf("%s/../../python/tests", cwd)
51 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
52 arvadostest.StartAPI()
53 arvadostest.StartKeep(2, false)
56 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
57 arvadostest.StopKeep(2)
61 func (s *ServerRequiredSuite) SetUpTest(c *C) {
62 RefreshServiceDiscovery()
65 func (s *ServerRequiredSuite) TestMakeKeepClient(c *C) {
66 arv, err := arvadosclient.MakeArvadosClient()
67 c.Assert(err, Equals, nil)
69 kc, err := MakeKeepClient(arv)
71 c.Assert(err, Equals, nil)
72 c.Check(len(kc.LocalRoots()), Equals, 2)
73 for _, root := range kc.LocalRoots() {
74 c.Check(root, Matches, "http://localhost:\\d+")
78 func (s *ServerRequiredSuite) TestDefaultReplications(c *C) {
79 arv, err := arvadosclient.MakeArvadosClient()
80 c.Assert(err, Equals, nil)
82 kc, err := MakeKeepClient(arv)
84 c.Assert(kc.Want_replicas, Equals, 2)
86 arv.DiscoveryDoc["defaultCollectionReplication"] = 3.0
87 kc, err = MakeKeepClient(arv)
89 c.Assert(kc.Want_replicas, Equals, 3)
91 arv.DiscoveryDoc["defaultCollectionReplication"] = 1.0
92 kc, err = MakeKeepClient(arv)
94 c.Assert(kc.Want_replicas, Equals, 1)
97 type StubPutHandler struct {
100 expectApiToken string
102 expectStorageClass string
106 func (sph StubPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
107 sph.c.Check(req.URL.Path, Equals, "/"+sph.expectPath)
108 sph.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sph.expectApiToken))
109 sph.c.Check(req.Header.Get("X-Keep-Storage-Classes"), Equals, sph.expectStorageClass)
110 body, err := ioutil.ReadAll(req.Body)
111 sph.c.Check(err, Equals, nil)
112 sph.c.Check(body, DeepEquals, []byte(sph.expectBody))
113 resp.WriteHeader(200)
114 sph.handled <- fmt.Sprintf("http://%s", req.Host)
117 func RunFakeKeepServer(st http.Handler) (ks KeepServer) {
119 // If we don't explicitly bind it to localhost, ks.listener.Addr() will
120 // bind to 0.0.0.0 or [::] which is not a valid address for Dial()
121 ks.listener, err = net.ListenTCP("tcp", &net.TCPAddr{IP: []byte{127, 0, 0, 1}, Port: 0})
123 panic(fmt.Sprintf("Could not listen on any port"))
125 ks.url = fmt.Sprintf("http://%s", ks.listener.Addr().String())
126 go http.Serve(ks.listener, st)
130 func UploadToStubHelper(c *C, st http.Handler, f func(*KeepClient, string,
131 io.ReadCloser, io.WriteCloser, chan uploadStatus)) {
133 ks := RunFakeKeepServer(st)
134 defer ks.listener.Close()
136 arv, _ := arvadosclient.MakeArvadosClient()
137 arv.ApiToken = "abc123"
139 kc, _ := MakeKeepClient(arv)
141 reader, writer := io.Pipe()
142 uploadStatusChan := make(chan uploadStatus)
144 f(kc, ks.url, reader, writer, uploadStatusChan)
147 func (s *StandaloneSuite) TestUploadToStubKeepServer(c *C) {
148 log.Printf("TestUploadToStubKeepServer")
150 st := StubPutHandler{
152 "acbd18db4cc2f85cedef654fccc4a4d8",
158 UploadToStubHelper(c, st,
159 func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, uploadStatusChan chan uploadStatus) {
160 kc.StorageClasses = []string{"hot"}
161 go kc.uploadToKeepServer(url, st.expectPath, reader, uploadStatusChan, int64(len("foo")), kc.getRequestID())
163 writer.Write([]byte("foo"))
167 status := <-uploadStatusChan
168 c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, ""})
172 func (s *StandaloneSuite) TestUploadToStubKeepServerBufferReader(c *C) {
173 st := StubPutHandler{
175 "acbd18db4cc2f85cedef654fccc4a4d8",
181 UploadToStubHelper(c, st,
182 func(kc *KeepClient, url string, _ io.ReadCloser, _ io.WriteCloser, uploadStatusChan chan uploadStatus) {
183 go kc.uploadToKeepServer(url, st.expectPath, bytes.NewBuffer([]byte("foo")), uploadStatusChan, 3, kc.getRequestID())
187 status := <-uploadStatusChan
188 c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, ""})
192 type FailHandler struct {
196 func (fh FailHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
197 resp.WriteHeader(500)
198 fh.handled <- fmt.Sprintf("http://%s", req.Host)
201 type FailThenSucceedHandler struct {
204 successhandler http.Handler
208 func (fh *FailThenSucceedHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
209 fh.reqIDs = append(fh.reqIDs, req.Header.Get("X-Request-Id"))
211 resp.WriteHeader(500)
213 fh.handled <- fmt.Sprintf("http://%s", req.Host)
215 fh.successhandler.ServeHTTP(resp, req)
219 type Error404Handler struct {
223 func (fh Error404Handler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
224 resp.WriteHeader(404)
225 fh.handled <- fmt.Sprintf("http://%s", req.Host)
228 func (s *StandaloneSuite) TestFailedUploadToStubKeepServer(c *C) {
232 hash := "acbd18db4cc2f85cedef654fccc4a4d8"
234 UploadToStubHelper(c, st,
235 func(kc *KeepClient, url string, reader io.ReadCloser,
236 writer io.WriteCloser, uploadStatusChan chan uploadStatus) {
238 go kc.uploadToKeepServer(url, hash, reader, uploadStatusChan, 3, kc.getRequestID())
240 writer.Write([]byte("foo"))
245 status := <-uploadStatusChan
246 c.Check(status.url, Equals, fmt.Sprintf("%s/%s", url, hash))
247 c.Check(status.statusCode, Equals, 500)
251 type KeepServer struct {
252 listener net.Listener
256 func RunSomeFakeKeepServers(st http.Handler, n int) (ks []KeepServer) {
257 ks = make([]KeepServer, n)
259 for i := 0; i < n; i += 1 {
260 ks[i] = RunFakeKeepServer(st)
266 func (s *StandaloneSuite) TestPutB(c *C) {
267 hash := Md5String("foo")
269 st := StubPutHandler{
275 make(chan string, 5)}
277 arv, _ := arvadosclient.MakeArvadosClient()
278 kc, _ := MakeKeepClient(arv)
281 arv.ApiToken = "abc123"
282 localRoots := make(map[string]string)
283 writableLocalRoots := make(map[string]string)
285 ks := RunSomeFakeKeepServers(st, 5)
287 for i, k := range ks {
288 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
289 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
290 defer k.listener.Close()
293 kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
295 kc.PutB([]byte("foo"))
297 shuff := NewRootSorter(
298 kc.LocalRoots(), Md5String("foo")).GetSortedRoots()
302 c.Check((s1 == shuff[0] && s2 == shuff[1]) ||
303 (s1 == shuff[1] && s2 == shuff[0]),
308 func (s *StandaloneSuite) TestPutHR(c *C) {
309 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
311 st := StubPutHandler{
317 make(chan string, 5)}
319 arv, _ := arvadosclient.MakeArvadosClient()
320 kc, _ := MakeKeepClient(arv)
323 arv.ApiToken = "abc123"
324 localRoots := make(map[string]string)
325 writableLocalRoots := make(map[string]string)
327 ks := RunSomeFakeKeepServers(st, 5)
329 for i, k := range ks {
330 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
331 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
332 defer k.listener.Close()
335 kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
337 reader, writer := io.Pipe()
340 writer.Write([]byte("foo"))
344 kc.PutHR(hash, reader, 3)
346 shuff := NewRootSorter(kc.LocalRoots(), hash).GetSortedRoots()
351 c.Check((s1 == shuff[0] && s2 == shuff[1]) ||
352 (s1 == shuff[1] && s2 == shuff[0]),
357 func (s *StandaloneSuite) TestPutWithFail(c *C) {
358 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
360 st := StubPutHandler{
366 make(chan string, 4)}
369 make(chan string, 1)}
371 arv, err := arvadosclient.MakeArvadosClient()
373 kc, _ := MakeKeepClient(arv)
376 arv.ApiToken = "abc123"
377 localRoots := make(map[string]string)
378 writableLocalRoots := make(map[string]string)
380 ks1 := RunSomeFakeKeepServers(st, 4)
381 ks2 := RunSomeFakeKeepServers(fh, 1)
383 for i, k := range ks1 {
384 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
385 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
386 defer k.listener.Close()
388 for i, k := range ks2 {
389 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
390 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
391 defer k.listener.Close()
394 kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
396 shuff := NewRootSorter(
397 kc.LocalRoots(), Md5String("foo")).GetSortedRoots()
400 phash, replicas, err := kc.PutB([]byte("foo"))
404 c.Check(err, Equals, nil)
405 c.Check(phash, Equals, "")
406 c.Check(replicas, Equals, 2)
411 c.Check((s1 == shuff[1] && s2 == shuff[2]) ||
412 (s1 == shuff[2] && s2 == shuff[1]),
417 func (s *StandaloneSuite) TestPutWithTooManyFail(c *C) {
418 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
420 st := StubPutHandler{
426 make(chan string, 1)}
429 make(chan string, 4)}
431 arv, err := arvadosclient.MakeArvadosClient()
433 kc, _ := MakeKeepClient(arv)
437 arv.ApiToken = "abc123"
438 localRoots := make(map[string]string)
439 writableLocalRoots := make(map[string]string)
441 ks1 := RunSomeFakeKeepServers(st, 1)
442 ks2 := RunSomeFakeKeepServers(fh, 4)
444 for i, k := range ks1 {
445 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
446 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
447 defer k.listener.Close()
449 for i, k := range ks2 {
450 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
451 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
452 defer k.listener.Close()
455 kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
457 _, replicas, err := kc.PutB([]byte("foo"))
459 c.Check(err, FitsTypeOf, InsufficientReplicasError(errors.New("")))
460 c.Check(replicas, Equals, 1)
461 c.Check(<-st.handled, Equals, ks1[0].url)
464 type StubGetHandler struct {
467 expectApiToken string
472 func (sgh StubGetHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
473 sgh.c.Check(req.URL.Path, Equals, "/"+sgh.expectPath)
474 sgh.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sgh.expectApiToken))
475 resp.WriteHeader(sgh.httpStatus)
476 resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(sgh.body)))
480 func (s *StandaloneSuite) TestGet(c *C) {
481 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
483 st := StubGetHandler{
490 ks := RunFakeKeepServer(st)
491 defer ks.listener.Close()
493 arv, err := arvadosclient.MakeArvadosClient()
495 kc, _ := MakeKeepClient(arv)
496 arv.ApiToken = "abc123"
497 kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
499 r, n, url2, err := kc.Get(hash)
501 c.Check(err, Equals, nil)
502 c.Check(n, Equals, int64(3))
503 c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks.url, hash))
505 content, err2 := ioutil.ReadAll(r)
506 c.Check(err2, Equals, nil)
507 c.Check(content, DeepEquals, []byte("foo"))
510 func (s *StandaloneSuite) TestGet404(c *C) {
511 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
513 st := Error404Handler{make(chan string, 1)}
515 ks := RunFakeKeepServer(st)
516 defer ks.listener.Close()
518 arv, err := arvadosclient.MakeArvadosClient()
520 kc, _ := MakeKeepClient(arv)
521 arv.ApiToken = "abc123"
522 kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
524 r, n, url2, err := kc.Get(hash)
525 c.Check(err, Equals, BlockNotFound)
526 c.Check(n, Equals, int64(0))
527 c.Check(url2, Equals, "")
528 c.Check(r, Equals, nil)
531 func (s *StandaloneSuite) TestGetEmptyBlock(c *C) {
532 st := Error404Handler{make(chan string, 1)}
534 ks := RunFakeKeepServer(st)
535 defer ks.listener.Close()
537 arv, err := arvadosclient.MakeArvadosClient()
539 kc, _ := MakeKeepClient(arv)
540 arv.ApiToken = "abc123"
541 kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
543 r, n, url2, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e+0")
545 c.Check(n, Equals, int64(0))
546 c.Check(url2, Equals, "")
548 buf, err := ioutil.ReadAll(r)
550 c.Check(buf, DeepEquals, []byte{})
553 func (s *StandaloneSuite) TestGetFail(c *C) {
554 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
556 st := FailHandler{make(chan string, 1)}
558 ks := RunFakeKeepServer(st)
559 defer ks.listener.Close()
561 arv, err := arvadosclient.MakeArvadosClient()
563 kc, _ := MakeKeepClient(arv)
564 arv.ApiToken = "abc123"
565 kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
568 r, n, url2, err := kc.Get(hash)
569 errNotFound, _ := err.(*ErrNotFound)
570 c.Check(errNotFound, NotNil)
571 c.Check(strings.Contains(errNotFound.Error(), "HTTP 500"), Equals, true)
572 c.Check(errNotFound.Temporary(), Equals, true)
573 c.Check(n, Equals, int64(0))
574 c.Check(url2, Equals, "")
575 c.Check(r, Equals, nil)
578 func (s *StandaloneSuite) TestGetFailRetry(c *C) {
579 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
581 st := &FailThenSucceedHandler{
582 handled: make(chan string, 1),
583 successhandler: StubGetHandler{
590 ks := RunFakeKeepServer(st)
591 defer ks.listener.Close()
593 arv, err := arvadosclient.MakeArvadosClient()
595 kc, _ := MakeKeepClient(arv)
596 arv.ApiToken = "abc123"
597 kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
599 r, n, url2, err := kc.Get(hash)
601 c.Check(err, Equals, nil)
602 c.Check(n, Equals, int64(3))
603 c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks.url, hash))
605 content, err2 := ioutil.ReadAll(r)
606 c.Check(err2, Equals, nil)
607 c.Check(content, DeepEquals, []byte("foo"))
609 c.Logf("%q", st.reqIDs)
610 c.Assert(len(st.reqIDs) > 1, Equals, true)
611 for _, reqid := range st.reqIDs {
612 c.Check(reqid, Not(Equals), "")
613 c.Check(reqid, Equals, st.reqIDs[0])
617 func (s *StandaloneSuite) TestGetNetError(c *C) {
618 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
620 arv, err := arvadosclient.MakeArvadosClient()
622 kc, _ := MakeKeepClient(arv)
623 arv.ApiToken = "abc123"
624 kc.SetServiceRoots(map[string]string{"x": "http://localhost:62222"}, nil, nil)
626 r, n, url2, err := kc.Get(hash)
627 errNotFound, _ := err.(*ErrNotFound)
628 c.Check(errNotFound, NotNil)
629 c.Check(strings.Contains(errNotFound.Error(), "connection refused"), Equals, true)
630 c.Check(errNotFound.Temporary(), Equals, true)
631 c.Check(n, Equals, int64(0))
632 c.Check(url2, Equals, "")
633 c.Check(r, Equals, nil)
636 func (s *StandaloneSuite) TestGetWithServiceHint(c *C) {
637 uuid := "zzzzz-bi6l4-123451234512345"
638 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
640 // This one shouldn't be used:
641 ks0 := RunFakeKeepServer(StubGetHandler{
647 defer ks0.listener.Close()
648 // This one should be used:
649 ks := RunFakeKeepServer(StubGetHandler{
655 defer ks.listener.Close()
657 arv, err := arvadosclient.MakeArvadosClient()
659 kc, _ := MakeKeepClient(arv)
660 arv.ApiToken = "abc123"
662 map[string]string{"x": ks0.url},
664 map[string]string{uuid: ks.url})
666 r, n, uri, err := kc.Get(hash + "+K@" + uuid)
668 c.Check(err, Equals, nil)
669 c.Check(n, Equals, int64(3))
670 c.Check(uri, Equals, fmt.Sprintf("%s/%s", ks.url, hash+"+K@"+uuid))
672 content, err := ioutil.ReadAll(r)
673 c.Check(err, Equals, nil)
674 c.Check(content, DeepEquals, []byte("foo"))
677 // Use a service hint to fetch from a local disk service, overriding
678 // rendezvous probe order.
679 func (s *StandaloneSuite) TestGetWithLocalServiceHint(c *C) {
680 uuid := "zzzzz-bi6l4-zzzzzzzzzzzzzzz"
681 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
683 // This one shouldn't be used, although it appears first in
684 // rendezvous probe order:
685 ks0 := RunFakeKeepServer(StubGetHandler{
691 defer ks0.listener.Close()
692 // This one should be used:
693 ks := RunFakeKeepServer(StubGetHandler{
699 defer ks.listener.Close()
701 arv, err := arvadosclient.MakeArvadosClient()
703 kc, _ := MakeKeepClient(arv)
704 arv.ApiToken = "abc123"
707 "zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url,
708 "zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url,
709 "zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url,
713 "zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url,
714 "zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url,
715 "zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url,
719 r, n, uri, err := kc.Get(hash + "+K@" + uuid)
721 c.Check(err, Equals, nil)
722 c.Check(n, Equals, int64(3))
723 c.Check(uri, Equals, fmt.Sprintf("%s/%s", ks.url, hash+"+K@"+uuid))
725 content, err := ioutil.ReadAll(r)
726 c.Check(err, Equals, nil)
727 c.Check(content, DeepEquals, []byte("foo"))
730 func (s *StandaloneSuite) TestGetWithServiceHintFailoverToLocals(c *C) {
731 uuid := "zzzzz-bi6l4-123451234512345"
732 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
734 ksLocal := RunFakeKeepServer(StubGetHandler{
740 defer ksLocal.listener.Close()
741 ksGateway := RunFakeKeepServer(StubGetHandler{
745 http.StatusInternalServerError,
747 defer ksGateway.listener.Close()
749 arv, err := arvadosclient.MakeArvadosClient()
751 kc, _ := MakeKeepClient(arv)
752 arv.ApiToken = "abc123"
754 map[string]string{"zzzzz-bi6l4-keepdisk0000000": ksLocal.url},
756 map[string]string{uuid: ksGateway.url})
758 r, n, uri, err := kc.Get(hash + "+K@" + uuid)
759 c.Assert(err, Equals, nil)
761 c.Check(n, Equals, int64(3))
762 c.Check(uri, Equals, fmt.Sprintf("%s/%s", ksLocal.url, hash+"+K@"+uuid))
764 content, err := ioutil.ReadAll(r)
765 c.Check(err, Equals, nil)
766 c.Check(content, DeepEquals, []byte("foo"))
769 type BarHandler struct {
773 func (h BarHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
774 resp.Write([]byte("bar"))
775 h.handled <- fmt.Sprintf("http://%s", req.Host)
778 func (s *StandaloneSuite) TestChecksum(c *C) {
779 foohash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
780 barhash := fmt.Sprintf("%x", md5.Sum([]byte("bar")))
782 st := BarHandler{make(chan string, 1)}
784 ks := RunFakeKeepServer(st)
785 defer ks.listener.Close()
787 arv, err := arvadosclient.MakeArvadosClient()
789 kc, _ := MakeKeepClient(arv)
790 arv.ApiToken = "abc123"
791 kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
793 r, n, _, err := kc.Get(barhash)
795 _, err = ioutil.ReadAll(r)
796 c.Check(n, Equals, int64(3))
797 c.Check(err, Equals, nil)
801 r, n, _, err = kc.Get(foohash)
803 _, err = ioutil.ReadAll(r)
804 c.Check(n, Equals, int64(3))
805 c.Check(err, Equals, BadChecksum)
810 func (s *StandaloneSuite) TestGetWithFailures(c *C) {
811 content := []byte("waz")
812 hash := fmt.Sprintf("%x", md5.Sum(content))
814 fh := Error404Handler{
815 make(chan string, 4)}
817 st := StubGetHandler{
824 arv, err := arvadosclient.MakeArvadosClient()
826 kc, _ := MakeKeepClient(arv)
827 arv.ApiToken = "abc123"
828 localRoots := make(map[string]string)
829 writableLocalRoots := make(map[string]string)
831 ks1 := RunSomeFakeKeepServers(st, 1)
832 ks2 := RunSomeFakeKeepServers(fh, 4)
834 for i, k := range ks1 {
835 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
836 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
837 defer k.listener.Close()
839 for i, k := range ks2 {
840 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
841 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
842 defer k.listener.Close()
845 kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
848 // This test works only if one of the failing services is
849 // attempted before the succeeding service. Otherwise,
850 // <-fh.handled below will just hang! (Probe order depends on
851 // the choice of block content "waz" and the UUIDs of the fake
852 // servers, so we just tried different strings until we found
853 // an example that passes this Assert.)
854 c.Assert(NewRootSorter(localRoots, hash).GetSortedRoots()[0], Not(Equals), ks1[0].url)
856 r, n, url2, err := kc.Get(hash)
859 c.Check(err, Equals, nil)
860 c.Check(n, Equals, int64(3))
861 c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks1[0].url, hash))
863 readContent, err2 := ioutil.ReadAll(r)
864 c.Check(err2, Equals, nil)
865 c.Check(readContent, DeepEquals, content)
868 func (s *ServerRequiredSuite) TestPutGetHead(c *C) {
869 content := []byte("TestPutGetHead")
871 arv, err := arvadosclient.MakeArvadosClient()
873 kc, err := MakeKeepClient(arv)
874 c.Assert(err, Equals, nil)
876 hash := fmt.Sprintf("%x", md5.Sum(content))
879 n, _, err := kc.Ask(hash)
880 c.Check(err, Equals, BlockNotFound)
881 c.Check(n, Equals, int64(0))
884 hash2, replicas, err := kc.PutB(content)
885 c.Check(hash2, Matches, fmt.Sprintf(`%s\+%d\b.*`, hash, len(content)))
886 c.Check(replicas, Equals, 2)
887 c.Check(err, Equals, nil)
890 r, n, url2, err := kc.Get(hash)
891 c.Check(err, Equals, nil)
892 c.Check(n, Equals, int64(len(content)))
893 c.Check(url2, Matches, fmt.Sprintf("http://localhost:\\d+/%s", hash))
895 readContent, err2 := ioutil.ReadAll(r)
896 c.Check(err2, Equals, nil)
897 c.Check(readContent, DeepEquals, content)
900 n, url2, err := kc.Ask(hash)
901 c.Check(err, Equals, nil)
902 c.Check(n, Equals, int64(len(content)))
903 c.Check(url2, Matches, fmt.Sprintf("http://localhost:\\d+/%s", hash))
906 loc, err := kc.LocalLocator(hash)
907 c.Check(err, Equals, nil)
908 c.Assert(len(loc) >= 32, Equals, true)
909 c.Check(loc[:32], Equals, hash[:32])
912 content := []byte("the perth county conspiracy")
913 loc, err := kc.LocalLocator(fmt.Sprintf("%x+%d+Rzaaaa-abcde@12345", md5.Sum(content), len(content)))
914 c.Check(loc, Equals, "")
915 c.Check(err, ErrorMatches, `.*HEAD .*\+R.*`)
916 c.Check(err, ErrorMatches, `.*HTTP 400.*`)
920 type StubProxyHandler struct {
924 func (h StubProxyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
925 resp.Header().Set("X-Keep-Replicas-Stored", "2")
926 h.handled <- fmt.Sprintf("http://%s", req.Host)
929 func (s *StandaloneSuite) TestPutProxy(c *C) {
930 st := StubProxyHandler{make(chan string, 1)}
932 arv, err := arvadosclient.MakeArvadosClient()
934 kc, _ := MakeKeepClient(arv)
937 arv.ApiToken = "abc123"
938 localRoots := make(map[string]string)
939 writableLocalRoots := make(map[string]string)
941 ks1 := RunSomeFakeKeepServers(st, 1)
943 for i, k := range ks1 {
944 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
945 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
946 defer k.listener.Close()
949 kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
951 _, replicas, err := kc.PutB([]byte("foo"))
954 c.Check(err, Equals, nil)
955 c.Check(replicas, Equals, 2)
958 func (s *StandaloneSuite) TestPutProxyInsufficientReplicas(c *C) {
959 st := StubProxyHandler{make(chan string, 1)}
961 arv, err := arvadosclient.MakeArvadosClient()
963 kc, _ := MakeKeepClient(arv)
966 arv.ApiToken = "abc123"
967 localRoots := make(map[string]string)
968 writableLocalRoots := make(map[string]string)
970 ks1 := RunSomeFakeKeepServers(st, 1)
972 for i, k := range ks1 {
973 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
974 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
975 defer k.listener.Close()
977 kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
979 _, replicas, err := kc.PutB([]byte("foo"))
982 c.Check(err, FitsTypeOf, InsufficientReplicasError(errors.New("")))
983 c.Check(replicas, Equals, 2)
986 func (s *StandaloneSuite) TestMakeLocator(c *C) {
987 l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce+3+Aabcde@12345678")
988 c.Check(err, Equals, nil)
989 c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
990 c.Check(l.Size, Equals, 3)
991 c.Check(l.Hints, DeepEquals, []string{"3", "Aabcde@12345678"})
994 func (s *StandaloneSuite) TestMakeLocatorNoHints(c *C) {
995 l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce")
996 c.Check(err, Equals, nil)
997 c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
998 c.Check(l.Size, Equals, -1)
999 c.Check(l.Hints, DeepEquals, []string{})
1002 func (s *StandaloneSuite) TestMakeLocatorNoSizeHint(c *C) {
1003 l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce+Aabcde@12345678")
1004 c.Check(err, Equals, nil)
1005 c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
1006 c.Check(l.Size, Equals, -1)
1007 c.Check(l.Hints, DeepEquals, []string{"Aabcde@12345678"})
1010 func (s *StandaloneSuite) TestMakeLocatorPreservesUnrecognizedHints(c *C) {
1011 str := "91f372a266fe2bf2823cb8ec7fda31ce+3+Unknown+Kzzzzz+Afoobar"
1012 l, err := MakeLocator(str)
1013 c.Check(err, Equals, nil)
1014 c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
1015 c.Check(l.Size, Equals, 3)
1016 c.Check(l.Hints, DeepEquals, []string{"3", "Unknown", "Kzzzzz", "Afoobar"})
1017 c.Check(l.String(), Equals, str)
1020 func (s *StandaloneSuite) TestMakeLocatorInvalidInput(c *C) {
1021 _, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31c")
1022 c.Check(err, Equals, InvalidLocatorError)
1025 func (s *StandaloneSuite) TestPutBWant2ReplicasWithOnlyOneWritableLocalRoot(c *C) {
1026 hash := Md5String("foo")
1028 st := StubPutHandler{
1034 make(chan string, 5)}
1036 arv, _ := arvadosclient.MakeArvadosClient()
1037 kc, _ := MakeKeepClient(arv)
1039 kc.Want_replicas = 2
1040 arv.ApiToken = "abc123"
1041 localRoots := make(map[string]string)
1042 writableLocalRoots := make(map[string]string)
1044 ks := RunSomeFakeKeepServers(st, 5)
1046 for i, k := range ks {
1047 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
1049 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
1051 defer k.listener.Close()
1054 kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
1056 _, replicas, err := kc.PutB([]byte("foo"))
1058 c.Check(err, FitsTypeOf, InsufficientReplicasError(errors.New("")))
1059 c.Check(replicas, Equals, 1)
1061 c.Check(<-st.handled, Equals, localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", 0)])
1064 func (s *StandaloneSuite) TestPutBWithNoWritableLocalRoots(c *C) {
1065 hash := Md5String("foo")
1067 st := StubPutHandler{
1073 make(chan string, 5)}
1075 arv, _ := arvadosclient.MakeArvadosClient()
1076 kc, _ := MakeKeepClient(arv)
1078 kc.Want_replicas = 2
1079 arv.ApiToken = "abc123"
1080 localRoots := make(map[string]string)
1081 writableLocalRoots := make(map[string]string)
1083 ks := RunSomeFakeKeepServers(st, 5)
1085 for i, k := range ks {
1086 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
1087 defer k.listener.Close()
1090 kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
1092 _, replicas, err := kc.PutB([]byte("foo"))
1094 c.Check(err, FitsTypeOf, InsufficientReplicasError(errors.New("")))
1095 c.Check(replicas, Equals, 0)
1098 type StubGetIndexHandler struct {
1101 expectAPIToken string
1106 func (h StubGetIndexHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
1107 h.c.Check(req.URL.Path, Equals, h.expectPath)
1108 h.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", h.expectAPIToken))
1109 resp.WriteHeader(h.httpStatus)
1110 resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(h.body)))
1114 func (s *StandaloneSuite) TestGetIndexWithNoPrefix(c *C) {
1115 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
1117 st := StubGetIndexHandler{
1122 []byte(hash + "+3 1443559274\n\n")}
1124 ks := RunFakeKeepServer(st)
1125 defer ks.listener.Close()
1127 arv, err := arvadosclient.MakeArvadosClient()
1128 c.Assert(err, IsNil)
1129 kc, err := MakeKeepClient(arv)
1130 c.Assert(err, IsNil)
1131 arv.ApiToken = "abc123"
1132 kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
1134 r, err := kc.GetIndex("x", "")
1137 content, err2 := ioutil.ReadAll(r)
1138 c.Check(err2, Equals, nil)
1139 c.Check(content, DeepEquals, st.body[0:len(st.body)-1])
1142 func (s *StandaloneSuite) TestGetIndexWithPrefix(c *C) {
1143 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
1145 st := StubGetIndexHandler{
1147 "/index/" + hash[0:3],
1150 []byte(hash + "+3 1443559274\n\n")}
1152 ks := RunFakeKeepServer(st)
1153 defer ks.listener.Close()
1155 arv, err := arvadosclient.MakeArvadosClient()
1157 kc, _ := MakeKeepClient(arv)
1158 arv.ApiToken = "abc123"
1159 kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
1161 r, err := kc.GetIndex("x", hash[0:3])
1162 c.Assert(err, Equals, nil)
1164 content, err2 := ioutil.ReadAll(r)
1165 c.Check(err2, Equals, nil)
1166 c.Check(content, DeepEquals, st.body[0:len(st.body)-1])
1169 func (s *StandaloneSuite) TestGetIndexIncomplete(c *C) {
1170 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
1172 st := StubGetIndexHandler{
1174 "/index/" + hash[0:3],
1179 ks := RunFakeKeepServer(st)
1180 defer ks.listener.Close()
1182 arv, err := arvadosclient.MakeArvadosClient()
1184 kc, _ := MakeKeepClient(arv)
1185 arv.ApiToken = "abc123"
1186 kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
1188 _, err = kc.GetIndex("x", hash[0:3])
1189 c.Check(err, Equals, ErrIncompleteIndex)
1192 func (s *StandaloneSuite) TestGetIndexWithNoSuchServer(c *C) {
1193 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
1195 st := StubGetIndexHandler{
1197 "/index/" + hash[0:3],
1202 ks := RunFakeKeepServer(st)
1203 defer ks.listener.Close()
1205 arv, err := arvadosclient.MakeArvadosClient()
1207 kc, _ := MakeKeepClient(arv)
1208 arv.ApiToken = "abc123"
1209 kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
1211 _, err = kc.GetIndex("y", hash[0:3])
1212 c.Check(err, Equals, ErrNoSuchKeepServer)
1215 func (s *StandaloneSuite) TestGetIndexWithNoSuchPrefix(c *C) {
1216 st := StubGetIndexHandler{
1223 ks := RunFakeKeepServer(st)
1224 defer ks.listener.Close()
1226 arv, err := arvadosclient.MakeArvadosClient()
1228 kc, _ := MakeKeepClient(arv)
1229 arv.ApiToken = "abc123"
1230 kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
1232 r, err := kc.GetIndex("x", "abcd")
1233 c.Check(err, Equals, nil)
1235 content, err2 := ioutil.ReadAll(r)
1236 c.Check(err2, Equals, nil)
1237 c.Check(content, DeepEquals, st.body[0:len(st.body)-1])
1240 func (s *StandaloneSuite) TestPutBRetry(c *C) {
1241 st := &FailThenSucceedHandler{
1242 handled: make(chan string, 1),
1243 successhandler: StubPutHandler{
1249 make(chan string, 5)}}
1251 arv, _ := arvadosclient.MakeArvadosClient()
1252 kc, _ := MakeKeepClient(arv)
1254 kc.Want_replicas = 2
1255 arv.ApiToken = "abc123"
1256 localRoots := make(map[string]string)
1257 writableLocalRoots := make(map[string]string)
1259 ks := RunSomeFakeKeepServers(st, 2)
1261 for i, k := range ks {
1262 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
1263 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
1264 defer k.listener.Close()
1267 kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
1269 hash, replicas, err := kc.PutB([]byte("foo"))
1271 c.Check(err, Equals, nil)
1272 c.Check(hash, Equals, "")
1273 c.Check(replicas, Equals, 2)
1276 func (s *ServerRequiredSuite) TestMakeKeepClientWithNonDiskTypeService(c *C) {
1277 arv, err := arvadosclient.MakeArvadosClient()
1278 c.Assert(err, Equals, nil)
1280 // Add an additional "testblobstore" keepservice
1281 blobKeepService := make(arvadosclient.Dict)
1282 err = arv.Create("keep_services",
1283 arvadosclient.Dict{"keep_service": arvadosclient.Dict{
1284 "service_host": "localhost",
1285 "service_port": "21321",
1286 "service_type": "testblobstore"}},
1288 c.Assert(err, Equals, nil)
1289 defer func() { arv.Delete("keep_services", blobKeepService["uuid"].(string), nil, nil) }()
1290 RefreshServiceDiscovery()
1292 // Make a keepclient and ensure that the testblobstore is included
1293 kc, err := MakeKeepClient(arv)
1294 c.Assert(err, Equals, nil)
1296 // verify kc.LocalRoots
1297 c.Check(len(kc.LocalRoots()), Equals, 3)
1298 for _, root := range kc.LocalRoots() {
1299 c.Check(root, Matches, "http://localhost:\\d+")
1301 c.Assert(kc.LocalRoots()[blobKeepService["uuid"].(string)], Not(Equals), "")
1303 // verify kc.GatewayRoots
1304 c.Check(len(kc.GatewayRoots()), Equals, 3)
1305 for _, root := range kc.GatewayRoots() {
1306 c.Check(root, Matches, "http://localhost:\\d+")
1308 c.Assert(kc.GatewayRoots()[blobKeepService["uuid"].(string)], Not(Equals), "")
1310 // verify kc.WritableLocalRoots
1311 c.Check(len(kc.WritableLocalRoots()), Equals, 3)
1312 for _, root := range kc.WritableLocalRoots() {
1313 c.Check(root, Matches, "http://localhost:\\d+")
1315 c.Assert(kc.WritableLocalRoots()[blobKeepService["uuid"].(string)], Not(Equals), "")
1317 c.Assert(kc.replicasPerService, Equals, 0)
1318 c.Assert(kc.foundNonDiskSvc, Equals, true)
1319 c.Assert(kc.httpClient().(*http.Client).Timeout, Equals, 300*time.Second)