1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: Apache-2.0
22 "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
23 "git.curoverse.com/arvados.git/sdk/go/arvadostest"
27 // Gocheck boilerplate
28 func Test(t *testing.T) {
32 // Gocheck boilerplate
33 var _ = Suite(&ServerRequiredSuite{})
34 var _ = Suite(&StandaloneSuite{})
36 // Tests that require the Keep server running
37 type ServerRequiredSuite struct{}
40 type StandaloneSuite struct{}
42 func (s *StandaloneSuite) SetUpTest(c *C) {
43 RefreshServiceDiscovery()
46 func pythonDir() string {
48 return fmt.Sprintf("%s/../../python/tests", cwd)
51 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
52 arvadostest.StartAPI()
53 arvadostest.StartKeep(2, false)
56 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
57 arvadostest.StopKeep(2)
61 func (s *ServerRequiredSuite) SetUpTest(c *C) {
62 RefreshServiceDiscovery()
65 func (s *ServerRequiredSuite) TestMakeKeepClient(c *C) {
66 arv, err := arvadosclient.MakeArvadosClient()
67 c.Assert(err, Equals, nil)
69 kc, err := MakeKeepClient(arv)
71 c.Assert(err, Equals, nil)
72 c.Check(len(kc.LocalRoots()), Equals, 2)
73 for _, root := range kc.LocalRoots() {
74 c.Check(root, Matches, "http://localhost:\\d+")
78 func (s *ServerRequiredSuite) TestDefaultReplications(c *C) {
79 arv, err := arvadosclient.MakeArvadosClient()
80 c.Assert(err, Equals, nil)
82 kc, err := MakeKeepClient(arv)
83 c.Assert(kc.Want_replicas, Equals, 2)
85 arv.DiscoveryDoc["defaultCollectionReplication"] = 3.0
86 kc, err = MakeKeepClient(arv)
87 c.Assert(kc.Want_replicas, Equals, 3)
89 arv.DiscoveryDoc["defaultCollectionReplication"] = 1.0
90 kc, err = MakeKeepClient(arv)
91 c.Assert(kc.Want_replicas, Equals, 1)
94 type StubPutHandler struct {
102 func (sph StubPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
103 sph.c.Check(req.URL.Path, Equals, "/"+sph.expectPath)
104 sph.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sph.expectApiToken))
105 body, err := ioutil.ReadAll(req.Body)
106 sph.c.Check(err, Equals, nil)
107 sph.c.Check(body, DeepEquals, []byte(sph.expectBody))
108 resp.WriteHeader(200)
109 sph.handled <- fmt.Sprintf("http://%s", req.Host)
112 func RunFakeKeepServer(st http.Handler) (ks KeepServer) {
114 // If we don't explicitly bind it to localhost, ks.listener.Addr() will
115 // bind to 0.0.0.0 or [::] which is not a valid address for Dial()
116 ks.listener, err = net.ListenTCP("tcp", &net.TCPAddr{IP: []byte{127, 0, 0, 1}, Port: 0})
118 panic(fmt.Sprintf("Could not listen on any port"))
120 ks.url = fmt.Sprintf("http://%s", ks.listener.Addr().String())
121 go http.Serve(ks.listener, st)
125 func UploadToStubHelper(c *C, st http.Handler, f func(*KeepClient, string,
126 io.ReadCloser, io.WriteCloser, chan uploadStatus)) {
128 ks := RunFakeKeepServer(st)
129 defer ks.listener.Close()
131 arv, _ := arvadosclient.MakeArvadosClient()
132 arv.ApiToken = "abc123"
134 kc, _ := MakeKeepClient(arv)
136 reader, writer := io.Pipe()
137 upload_status := make(chan uploadStatus)
139 f(kc, ks.url, reader, writer, upload_status)
142 func (s *StandaloneSuite) TestUploadToStubKeepServer(c *C) {
143 log.Printf("TestUploadToStubKeepServer")
145 st := StubPutHandler{
147 "acbd18db4cc2f85cedef654fccc4a4d8",
152 UploadToStubHelper(c, st,
153 func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, upload_status chan uploadStatus) {
155 go kc.uploadToKeepServer(url, st.expectPath, reader, upload_status, int64(len("foo")), 0)
157 writer.Write([]byte("foo"))
161 status := <-upload_status
162 c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, ""})
166 func (s *StandaloneSuite) TestUploadToStubKeepServerBufferReader(c *C) {
167 st := StubPutHandler{
169 "acbd18db4cc2f85cedef654fccc4a4d8",
174 UploadToStubHelper(c, st,
175 func(kc *KeepClient, url string, _ io.ReadCloser, _ io.WriteCloser, upload_status chan uploadStatus) {
176 go kc.uploadToKeepServer(url, st.expectPath, bytes.NewBuffer([]byte("foo")), upload_status, 3, 0)
180 status := <-upload_status
181 c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, ""})
185 type FailHandler struct {
189 func (fh FailHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
190 resp.WriteHeader(500)
191 fh.handled <- fmt.Sprintf("http://%s", req.Host)
194 type FailThenSucceedHandler struct {
197 successhandler StubGetHandler
200 func (fh *FailThenSucceedHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
202 resp.WriteHeader(500)
204 fh.handled <- fmt.Sprintf("http://%s", req.Host)
206 fh.successhandler.ServeHTTP(resp, req)
210 type Error404Handler struct {
214 func (fh Error404Handler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
215 resp.WriteHeader(404)
216 fh.handled <- fmt.Sprintf("http://%s", req.Host)
219 func (s *StandaloneSuite) TestFailedUploadToStubKeepServer(c *C) {
223 hash := "acbd18db4cc2f85cedef654fccc4a4d8"
225 UploadToStubHelper(c, st,
226 func(kc *KeepClient, url string, reader io.ReadCloser,
227 writer io.WriteCloser, upload_status chan uploadStatus) {
229 go kc.uploadToKeepServer(url, hash, reader, upload_status, 3, 0)
231 writer.Write([]byte("foo"))
236 status := <-upload_status
237 c.Check(status.url, Equals, fmt.Sprintf("%s/%s", url, hash))
238 c.Check(status.statusCode, Equals, 500)
242 type KeepServer struct {
243 listener net.Listener
247 func RunSomeFakeKeepServers(st http.Handler, n int) (ks []KeepServer) {
248 ks = make([]KeepServer, n)
250 for i := 0; i < n; i += 1 {
251 ks[i] = RunFakeKeepServer(st)
257 func (s *StandaloneSuite) TestPutB(c *C) {
258 hash := Md5String("foo")
260 st := StubPutHandler{
265 make(chan string, 5)}
267 arv, _ := arvadosclient.MakeArvadosClient()
268 kc, _ := MakeKeepClient(arv)
271 arv.ApiToken = "abc123"
272 localRoots := make(map[string]string)
273 writableLocalRoots := make(map[string]string)
275 ks := RunSomeFakeKeepServers(st, 5)
277 for i, k := range ks {
278 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
279 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
280 defer k.listener.Close()
283 kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
285 kc.PutB([]byte("foo"))
287 shuff := NewRootSorter(
288 kc.LocalRoots(), Md5String("foo")).GetSortedRoots()
292 c.Check((s1 == shuff[0] && s2 == shuff[1]) ||
293 (s1 == shuff[1] && s2 == shuff[0]),
298 func (s *StandaloneSuite) TestPutHR(c *C) {
299 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
301 st := StubPutHandler{
306 make(chan string, 5)}
308 arv, _ := arvadosclient.MakeArvadosClient()
309 kc, _ := MakeKeepClient(arv)
312 arv.ApiToken = "abc123"
313 localRoots := make(map[string]string)
314 writableLocalRoots := make(map[string]string)
316 ks := RunSomeFakeKeepServers(st, 5)
318 for i, k := range ks {
319 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
320 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
321 defer k.listener.Close()
324 kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
326 reader, writer := io.Pipe()
329 writer.Write([]byte("foo"))
333 kc.PutHR(hash, reader, 3)
335 shuff := NewRootSorter(kc.LocalRoots(), hash).GetSortedRoots()
340 c.Check((s1 == shuff[0] && s2 == shuff[1]) ||
341 (s1 == shuff[1] && s2 == shuff[0]),
346 func (s *StandaloneSuite) TestPutWithFail(c *C) {
347 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
349 st := StubPutHandler{
354 make(chan string, 4)}
357 make(chan string, 1)}
359 arv, err := arvadosclient.MakeArvadosClient()
360 kc, _ := MakeKeepClient(arv)
363 arv.ApiToken = "abc123"
364 localRoots := make(map[string]string)
365 writableLocalRoots := make(map[string]string)
367 ks1 := RunSomeFakeKeepServers(st, 4)
368 ks2 := RunSomeFakeKeepServers(fh, 1)
370 for i, k := range ks1 {
371 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
372 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
373 defer k.listener.Close()
375 for i, k := range ks2 {
376 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
377 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
378 defer k.listener.Close()
381 kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
383 shuff := NewRootSorter(
384 kc.LocalRoots(), Md5String("foo")).GetSortedRoots()
387 phash, replicas, err := kc.PutB([]byte("foo"))
391 c.Check(err, Equals, nil)
392 c.Check(phash, Equals, "")
393 c.Check(replicas, Equals, 2)
398 c.Check((s1 == shuff[1] && s2 == shuff[2]) ||
399 (s1 == shuff[2] && s2 == shuff[1]),
404 func (s *StandaloneSuite) TestPutWithTooManyFail(c *C) {
405 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
407 st := StubPutHandler{
412 make(chan string, 1)}
415 make(chan string, 4)}
417 arv, err := arvadosclient.MakeArvadosClient()
418 kc, _ := MakeKeepClient(arv)
422 arv.ApiToken = "abc123"
423 localRoots := make(map[string]string)
424 writableLocalRoots := make(map[string]string)
426 ks1 := RunSomeFakeKeepServers(st, 1)
427 ks2 := RunSomeFakeKeepServers(fh, 4)
429 for i, k := range ks1 {
430 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
431 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
432 defer k.listener.Close()
434 for i, k := range ks2 {
435 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
436 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
437 defer k.listener.Close()
440 kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
442 _, replicas, err := kc.PutB([]byte("foo"))
444 c.Check(err, FitsTypeOf, InsufficientReplicasError(errors.New("")))
445 c.Check(replicas, Equals, 1)
446 c.Check(<-st.handled, Equals, ks1[0].url)
449 type StubGetHandler struct {
452 expectApiToken string
457 func (sgh StubGetHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
458 sgh.c.Check(req.URL.Path, Equals, "/"+sgh.expectPath)
459 sgh.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sgh.expectApiToken))
460 resp.WriteHeader(sgh.httpStatus)
461 resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(sgh.body)))
465 func (s *StandaloneSuite) TestGet(c *C) {
466 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
468 st := StubGetHandler{
475 ks := RunFakeKeepServer(st)
476 defer ks.listener.Close()
478 arv, err := arvadosclient.MakeArvadosClient()
479 kc, _ := MakeKeepClient(arv)
480 arv.ApiToken = "abc123"
481 kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
483 r, n, url2, err := kc.Get(hash)
485 c.Check(err, Equals, nil)
486 c.Check(n, Equals, int64(3))
487 c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks.url, hash))
489 content, err2 := ioutil.ReadAll(r)
490 c.Check(err2, Equals, nil)
491 c.Check(content, DeepEquals, []byte("foo"))
494 func (s *StandaloneSuite) TestGet404(c *C) {
495 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
497 st := Error404Handler{make(chan string, 1)}
499 ks := RunFakeKeepServer(st)
500 defer ks.listener.Close()
502 arv, err := arvadosclient.MakeArvadosClient()
503 kc, _ := MakeKeepClient(arv)
504 arv.ApiToken = "abc123"
505 kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
507 r, n, url2, err := kc.Get(hash)
508 c.Check(err, Equals, BlockNotFound)
509 c.Check(n, Equals, int64(0))
510 c.Check(url2, Equals, "")
511 c.Check(r, Equals, nil)
514 func (s *StandaloneSuite) TestGetEmptyBlock(c *C) {
515 st := Error404Handler{make(chan string, 1)}
517 ks := RunFakeKeepServer(st)
518 defer ks.listener.Close()
520 arv, err := arvadosclient.MakeArvadosClient()
521 kc, _ := MakeKeepClient(arv)
522 arv.ApiToken = "abc123"
523 kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
525 r, n, url2, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e+0")
527 c.Check(n, Equals, int64(0))
528 c.Check(url2, Equals, "")
530 buf, err := ioutil.ReadAll(r)
532 c.Check(buf, DeepEquals, []byte{})
535 func (s *StandaloneSuite) TestGetFail(c *C) {
536 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
538 st := FailHandler{make(chan string, 1)}
540 ks := RunFakeKeepServer(st)
541 defer ks.listener.Close()
543 arv, err := arvadosclient.MakeArvadosClient()
544 kc, _ := MakeKeepClient(arv)
545 arv.ApiToken = "abc123"
546 kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
549 r, n, url2, err := kc.Get(hash)
550 errNotFound, _ := err.(*ErrNotFound)
551 c.Check(errNotFound, NotNil)
552 c.Check(strings.Contains(errNotFound.Error(), "HTTP 500"), Equals, true)
553 c.Check(errNotFound.Temporary(), Equals, true)
554 c.Check(n, Equals, int64(0))
555 c.Check(url2, Equals, "")
556 c.Check(r, Equals, nil)
559 func (s *StandaloneSuite) TestGetFailRetry(c *C) {
560 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
562 st := &FailThenSucceedHandler{make(chan string, 1), 0,
570 ks := RunFakeKeepServer(st)
571 defer ks.listener.Close()
573 arv, err := arvadosclient.MakeArvadosClient()
574 kc, _ := MakeKeepClient(arv)
575 arv.ApiToken = "abc123"
576 kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
578 r, n, url2, err := kc.Get(hash)
580 c.Check(err, Equals, nil)
581 c.Check(n, Equals, int64(3))
582 c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks.url, hash))
584 content, err2 := ioutil.ReadAll(r)
585 c.Check(err2, Equals, nil)
586 c.Check(content, DeepEquals, []byte("foo"))
589 func (s *StandaloneSuite) TestGetNetError(c *C) {
590 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
592 arv, err := arvadosclient.MakeArvadosClient()
593 kc, _ := MakeKeepClient(arv)
594 arv.ApiToken = "abc123"
595 kc.SetServiceRoots(map[string]string{"x": "http://localhost:62222"}, nil, nil)
597 r, n, url2, err := kc.Get(hash)
598 errNotFound, _ := err.(*ErrNotFound)
599 c.Check(errNotFound, NotNil)
600 c.Check(strings.Contains(errNotFound.Error(), "connection refused"), Equals, true)
601 c.Check(errNotFound.Temporary(), Equals, true)
602 c.Check(n, Equals, int64(0))
603 c.Check(url2, Equals, "")
604 c.Check(r, Equals, nil)
607 func (s *StandaloneSuite) TestGetWithServiceHint(c *C) {
608 uuid := "zzzzz-bi6l4-123451234512345"
609 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
611 // This one shouldn't be used:
612 ks0 := RunFakeKeepServer(StubGetHandler{
618 defer ks0.listener.Close()
619 // This one should be used:
620 ks := RunFakeKeepServer(StubGetHandler{
626 defer ks.listener.Close()
628 arv, err := arvadosclient.MakeArvadosClient()
629 kc, _ := MakeKeepClient(arv)
630 arv.ApiToken = "abc123"
632 map[string]string{"x": ks0.url},
634 map[string]string{uuid: ks.url})
636 r, n, uri, err := kc.Get(hash + "+K@" + uuid)
638 c.Check(err, Equals, nil)
639 c.Check(n, Equals, int64(3))
640 c.Check(uri, Equals, fmt.Sprintf("%s/%s", ks.url, hash+"+K@"+uuid))
642 content, err := ioutil.ReadAll(r)
643 c.Check(err, Equals, nil)
644 c.Check(content, DeepEquals, []byte("foo"))
647 // Use a service hint to fetch from a local disk service, overriding
648 // rendezvous probe order.
649 func (s *StandaloneSuite) TestGetWithLocalServiceHint(c *C) {
650 uuid := "zzzzz-bi6l4-zzzzzzzzzzzzzzz"
651 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
653 // This one shouldn't be used, although it appears first in
654 // rendezvous probe order:
655 ks0 := RunFakeKeepServer(StubGetHandler{
661 defer ks0.listener.Close()
662 // This one should be used:
663 ks := RunFakeKeepServer(StubGetHandler{
669 defer ks.listener.Close()
671 arv, err := arvadosclient.MakeArvadosClient()
672 kc, _ := MakeKeepClient(arv)
673 arv.ApiToken = "abc123"
676 "zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url,
677 "zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url,
678 "zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url,
682 "zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url,
683 "zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url,
684 "zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url,
688 r, n, uri, err := kc.Get(hash + "+K@" + uuid)
690 c.Check(err, Equals, nil)
691 c.Check(n, Equals, int64(3))
692 c.Check(uri, Equals, fmt.Sprintf("%s/%s", ks.url, hash+"+K@"+uuid))
694 content, err := ioutil.ReadAll(r)
695 c.Check(err, Equals, nil)
696 c.Check(content, DeepEquals, []byte("foo"))
699 func (s *StandaloneSuite) TestGetWithServiceHintFailoverToLocals(c *C) {
700 uuid := "zzzzz-bi6l4-123451234512345"
701 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
703 ksLocal := RunFakeKeepServer(StubGetHandler{
709 defer ksLocal.listener.Close()
710 ksGateway := RunFakeKeepServer(StubGetHandler{
714 http.StatusInternalServerError,
716 defer ksGateway.listener.Close()
718 arv, err := arvadosclient.MakeArvadosClient()
719 kc, _ := MakeKeepClient(arv)
720 arv.ApiToken = "abc123"
722 map[string]string{"zzzzz-bi6l4-keepdisk0000000": ksLocal.url},
724 map[string]string{uuid: ksGateway.url})
726 r, n, uri, err := kc.Get(hash + "+K@" + uuid)
727 c.Assert(err, Equals, nil)
729 c.Check(n, Equals, int64(3))
730 c.Check(uri, Equals, fmt.Sprintf("%s/%s", ksLocal.url, hash+"+K@"+uuid))
732 content, err := ioutil.ReadAll(r)
733 c.Check(err, Equals, nil)
734 c.Check(content, DeepEquals, []byte("foo"))
737 type BarHandler struct {
741 func (this BarHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
742 resp.Write([]byte("bar"))
743 this.handled <- fmt.Sprintf("http://%s", req.Host)
746 func (s *StandaloneSuite) TestChecksum(c *C) {
747 foohash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
748 barhash := fmt.Sprintf("%x", md5.Sum([]byte("bar")))
750 st := BarHandler{make(chan string, 1)}
752 ks := RunFakeKeepServer(st)
753 defer ks.listener.Close()
755 arv, err := arvadosclient.MakeArvadosClient()
756 kc, _ := MakeKeepClient(arv)
757 arv.ApiToken = "abc123"
758 kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
760 r, n, _, err := kc.Get(barhash)
761 _, err = ioutil.ReadAll(r)
762 c.Check(n, Equals, int64(3))
763 c.Check(err, Equals, nil)
767 r, n, _, err = kc.Get(foohash)
768 _, err = ioutil.ReadAll(r)
769 c.Check(n, Equals, int64(3))
770 c.Check(err, Equals, BadChecksum)
775 func (s *StandaloneSuite) TestGetWithFailures(c *C) {
776 content := []byte("waz")
777 hash := fmt.Sprintf("%x", md5.Sum(content))
779 fh := Error404Handler{
780 make(chan string, 4)}
782 st := StubGetHandler{
789 arv, err := arvadosclient.MakeArvadosClient()
790 kc, _ := MakeKeepClient(arv)
791 arv.ApiToken = "abc123"
792 localRoots := make(map[string]string)
793 writableLocalRoots := make(map[string]string)
795 ks1 := RunSomeFakeKeepServers(st, 1)
796 ks2 := RunSomeFakeKeepServers(fh, 4)
798 for i, k := range ks1 {
799 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
800 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
801 defer k.listener.Close()
803 for i, k := range ks2 {
804 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
805 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
806 defer k.listener.Close()
809 kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
812 // This test works only if one of the failing services is
813 // attempted before the succeeding service. Otherwise,
814 // <-fh.handled below will just hang! (Probe order depends on
815 // the choice of block content "waz" and the UUIDs of the fake
816 // servers, so we just tried different strings until we found
817 // an example that passes this Assert.)
818 c.Assert(NewRootSorter(localRoots, hash).GetSortedRoots()[0], Not(Equals), ks1[0].url)
820 r, n, url2, err := kc.Get(hash)
823 c.Check(err, Equals, nil)
824 c.Check(n, Equals, int64(3))
825 c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks1[0].url, hash))
827 read_content, err2 := ioutil.ReadAll(r)
828 c.Check(err2, Equals, nil)
829 c.Check(read_content, DeepEquals, content)
832 func (s *ServerRequiredSuite) TestPutGetHead(c *C) {
833 content := []byte("TestPutGetHead")
835 arv, err := arvadosclient.MakeArvadosClient()
836 kc, err := MakeKeepClient(arv)
837 c.Assert(err, Equals, nil)
839 hash := fmt.Sprintf("%x", md5.Sum(content))
842 n, _, err := kc.Ask(hash)
843 c.Check(err, Equals, BlockNotFound)
844 c.Check(n, Equals, int64(0))
847 hash2, replicas, err := kc.PutB(content)
848 c.Check(hash2, Matches, fmt.Sprintf(`%s\+%d\b.*`, hash, len(content)))
849 c.Check(replicas, Equals, 2)
850 c.Check(err, Equals, nil)
853 r, n, url2, err := kc.Get(hash)
854 c.Check(err, Equals, nil)
855 c.Check(n, Equals, int64(len(content)))
856 c.Check(url2, Matches, fmt.Sprintf("http://localhost:\\d+/%s", hash))
858 read_content, err2 := ioutil.ReadAll(r)
859 c.Check(err2, Equals, nil)
860 c.Check(read_content, DeepEquals, content)
863 n, url2, err := kc.Ask(hash)
864 c.Check(err, Equals, nil)
865 c.Check(n, Equals, int64(len(content)))
866 c.Check(url2, Matches, fmt.Sprintf("http://localhost:\\d+/%s", hash))
870 type StubProxyHandler struct {
874 func (this StubProxyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
875 resp.Header().Set("X-Keep-Replicas-Stored", "2")
876 this.handled <- fmt.Sprintf("http://%s", req.Host)
879 func (s *StandaloneSuite) TestPutProxy(c *C) {
880 st := StubProxyHandler{make(chan string, 1)}
882 arv, err := arvadosclient.MakeArvadosClient()
883 kc, _ := MakeKeepClient(arv)
886 arv.ApiToken = "abc123"
887 localRoots := make(map[string]string)
888 writableLocalRoots := make(map[string]string)
890 ks1 := RunSomeFakeKeepServers(st, 1)
892 for i, k := range ks1 {
893 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
894 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
895 defer k.listener.Close()
898 kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
900 _, replicas, err := kc.PutB([]byte("foo"))
903 c.Check(err, Equals, nil)
904 c.Check(replicas, Equals, 2)
907 func (s *StandaloneSuite) TestPutProxyInsufficientReplicas(c *C) {
908 st := StubProxyHandler{make(chan string, 1)}
910 arv, err := arvadosclient.MakeArvadosClient()
911 kc, _ := MakeKeepClient(arv)
914 arv.ApiToken = "abc123"
915 localRoots := make(map[string]string)
916 writableLocalRoots := make(map[string]string)
918 ks1 := RunSomeFakeKeepServers(st, 1)
920 for i, k := range ks1 {
921 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
922 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
923 defer k.listener.Close()
925 kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
927 _, replicas, err := kc.PutB([]byte("foo"))
930 c.Check(err, FitsTypeOf, InsufficientReplicasError(errors.New("")))
931 c.Check(replicas, Equals, 2)
934 func (s *StandaloneSuite) TestMakeLocator(c *C) {
935 l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce+3+Aabcde@12345678")
936 c.Check(err, Equals, nil)
937 c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
938 c.Check(l.Size, Equals, 3)
939 c.Check(l.Hints, DeepEquals, []string{"3", "Aabcde@12345678"})
942 func (s *StandaloneSuite) TestMakeLocatorNoHints(c *C) {
943 l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce")
944 c.Check(err, Equals, nil)
945 c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
946 c.Check(l.Size, Equals, -1)
947 c.Check(l.Hints, DeepEquals, []string{})
950 func (s *StandaloneSuite) TestMakeLocatorNoSizeHint(c *C) {
951 l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce+Aabcde@12345678")
952 c.Check(err, Equals, nil)
953 c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
954 c.Check(l.Size, Equals, -1)
955 c.Check(l.Hints, DeepEquals, []string{"Aabcde@12345678"})
958 func (s *StandaloneSuite) TestMakeLocatorPreservesUnrecognizedHints(c *C) {
959 str := "91f372a266fe2bf2823cb8ec7fda31ce+3+Unknown+Kzzzzz+Afoobar"
960 l, err := MakeLocator(str)
961 c.Check(err, Equals, nil)
962 c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
963 c.Check(l.Size, Equals, 3)
964 c.Check(l.Hints, DeepEquals, []string{"3", "Unknown", "Kzzzzz", "Afoobar"})
965 c.Check(l.String(), Equals, str)
968 func (s *StandaloneSuite) TestMakeLocatorInvalidInput(c *C) {
969 _, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31c")
970 c.Check(err, Equals, InvalidLocatorError)
973 func (s *StandaloneSuite) TestPutBWant2ReplicasWithOnlyOneWritableLocalRoot(c *C) {
974 hash := Md5String("foo")
976 st := StubPutHandler{
981 make(chan string, 5)}
983 arv, _ := arvadosclient.MakeArvadosClient()
984 kc, _ := MakeKeepClient(arv)
987 arv.ApiToken = "abc123"
988 localRoots := make(map[string]string)
989 writableLocalRoots := make(map[string]string)
991 ks := RunSomeFakeKeepServers(st, 5)
993 for i, k := range ks {
994 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
996 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
998 defer k.listener.Close()
1001 kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
1003 _, replicas, err := kc.PutB([]byte("foo"))
1005 c.Check(err, FitsTypeOf, InsufficientReplicasError(errors.New("")))
1006 c.Check(replicas, Equals, 1)
1008 c.Check(<-st.handled, Equals, localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", 0)])
1011 func (s *StandaloneSuite) TestPutBWithNoWritableLocalRoots(c *C) {
1012 hash := Md5String("foo")
1014 st := StubPutHandler{
1019 make(chan string, 5)}
1021 arv, _ := arvadosclient.MakeArvadosClient()
1022 kc, _ := MakeKeepClient(arv)
1024 kc.Want_replicas = 2
1025 arv.ApiToken = "abc123"
1026 localRoots := make(map[string]string)
1027 writableLocalRoots := make(map[string]string)
1029 ks := RunSomeFakeKeepServers(st, 5)
1031 for i, k := range ks {
1032 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
1033 defer k.listener.Close()
1036 kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
1038 _, replicas, err := kc.PutB([]byte("foo"))
1040 c.Check(err, FitsTypeOf, InsufficientReplicasError(errors.New("")))
1041 c.Check(replicas, Equals, 0)
1044 type StubGetIndexHandler struct {
1047 expectAPIToken string
1052 func (h StubGetIndexHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
1053 h.c.Check(req.URL.Path, Equals, h.expectPath)
1054 h.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", h.expectAPIToken))
1055 resp.WriteHeader(h.httpStatus)
1056 resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(h.body)))
1060 func (s *StandaloneSuite) TestGetIndexWithNoPrefix(c *C) {
1061 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
1063 st := StubGetIndexHandler{
1068 []byte(hash + "+3 1443559274\n\n")}
1070 ks := RunFakeKeepServer(st)
1071 defer ks.listener.Close()
1073 arv, err := arvadosclient.MakeArvadosClient()
1074 c.Assert(err, IsNil)
1075 kc, err := MakeKeepClient(arv)
1076 c.Assert(err, IsNil)
1077 arv.ApiToken = "abc123"
1078 kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
1080 r, err := kc.GetIndex("x", "")
1083 content, err2 := ioutil.ReadAll(r)
1084 c.Check(err2, Equals, nil)
1085 c.Check(content, DeepEquals, st.body[0:len(st.body)-1])
1088 func (s *StandaloneSuite) TestGetIndexWithPrefix(c *C) {
1089 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
1091 st := StubGetIndexHandler{
1093 "/index/" + hash[0:3],
1096 []byte(hash + "+3 1443559274\n\n")}
1098 ks := RunFakeKeepServer(st)
1099 defer ks.listener.Close()
1101 arv, err := arvadosclient.MakeArvadosClient()
1102 kc, _ := MakeKeepClient(arv)
1103 arv.ApiToken = "abc123"
1104 kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
1106 r, err := kc.GetIndex("x", hash[0:3])
1107 c.Assert(err, Equals, nil)
1109 content, err2 := ioutil.ReadAll(r)
1110 c.Check(err2, Equals, nil)
1111 c.Check(content, DeepEquals, st.body[0:len(st.body)-1])
1114 func (s *StandaloneSuite) TestGetIndexIncomplete(c *C) {
1115 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
1117 st := StubGetIndexHandler{
1119 "/index/" + hash[0:3],
1124 ks := RunFakeKeepServer(st)
1125 defer ks.listener.Close()
1127 arv, err := arvadosclient.MakeArvadosClient()
1128 kc, _ := MakeKeepClient(arv)
1129 arv.ApiToken = "abc123"
1130 kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
1132 _, err = kc.GetIndex("x", hash[0:3])
1133 c.Check(err, Equals, ErrIncompleteIndex)
1136 func (s *StandaloneSuite) TestGetIndexWithNoSuchServer(c *C) {
1137 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
1139 st := StubGetIndexHandler{
1141 "/index/" + hash[0:3],
1146 ks := RunFakeKeepServer(st)
1147 defer ks.listener.Close()
1149 arv, err := arvadosclient.MakeArvadosClient()
1150 kc, _ := MakeKeepClient(arv)
1151 arv.ApiToken = "abc123"
1152 kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
1154 _, err = kc.GetIndex("y", hash[0:3])
1155 c.Check(err, Equals, ErrNoSuchKeepServer)
1158 func (s *StandaloneSuite) TestGetIndexWithNoSuchPrefix(c *C) {
1159 st := StubGetIndexHandler{
1166 ks := RunFakeKeepServer(st)
1167 defer ks.listener.Close()
1169 arv, err := arvadosclient.MakeArvadosClient()
1170 kc, _ := MakeKeepClient(arv)
1171 arv.ApiToken = "abc123"
1172 kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
1174 r, err := kc.GetIndex("x", "abcd")
1175 c.Check(err, Equals, nil)
1177 content, err2 := ioutil.ReadAll(r)
1178 c.Check(err2, Equals, nil)
1179 c.Check(content, DeepEquals, st.body[0:len(st.body)-1])
1182 type FailThenSucceedPutHandler struct {
1185 successhandler StubPutHandler
1188 func (h *FailThenSucceedPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
1190 resp.WriteHeader(500)
1192 h.handled <- fmt.Sprintf("http://%s", req.Host)
1194 h.successhandler.ServeHTTP(resp, req)
1198 func (s *StandaloneSuite) TestPutBRetry(c *C) {
1199 st := &FailThenSucceedPutHandler{make(chan string, 1), 0,
1205 make(chan string, 5)}}
1207 arv, _ := arvadosclient.MakeArvadosClient()
1208 kc, _ := MakeKeepClient(arv)
1210 kc.Want_replicas = 2
1211 arv.ApiToken = "abc123"
1212 localRoots := make(map[string]string)
1213 writableLocalRoots := make(map[string]string)
1215 ks := RunSomeFakeKeepServers(st, 2)
1217 for i, k := range ks {
1218 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
1219 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
1220 defer k.listener.Close()
1223 kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
1225 hash, replicas, err := kc.PutB([]byte("foo"))
1227 c.Check(err, Equals, nil)
1228 c.Check(hash, Equals, "")
1229 c.Check(replicas, Equals, 2)
1232 func (s *ServerRequiredSuite) TestMakeKeepClientWithNonDiskTypeService(c *C) {
1233 arv, err := arvadosclient.MakeArvadosClient()
1234 c.Assert(err, Equals, nil)
1236 // Add an additional "testblobstore" keepservice
1237 blobKeepService := make(arvadosclient.Dict)
1238 err = arv.Create("keep_services",
1239 arvadosclient.Dict{"keep_service": arvadosclient.Dict{
1240 "service_host": "localhost",
1241 "service_port": "21321",
1242 "service_type": "testblobstore"}},
1244 c.Assert(err, Equals, nil)
1245 defer func() { arv.Delete("keep_services", blobKeepService["uuid"].(string), nil, nil) }()
1246 RefreshServiceDiscovery()
1248 // Make a keepclient and ensure that the testblobstore is included
1249 kc, err := MakeKeepClient(arv)
1250 c.Assert(err, Equals, nil)
1252 // verify kc.LocalRoots
1253 c.Check(len(kc.LocalRoots()), Equals, 3)
1254 for _, root := range kc.LocalRoots() {
1255 c.Check(root, Matches, "http://localhost:\\d+")
1257 c.Assert(kc.LocalRoots()[blobKeepService["uuid"].(string)], Not(Equals), "")
1259 // verify kc.GatewayRoots
1260 c.Check(len(kc.GatewayRoots()), Equals, 3)
1261 for _, root := range kc.GatewayRoots() {
1262 c.Check(root, Matches, "http://localhost:\\d+")
1264 c.Assert(kc.GatewayRoots()[blobKeepService["uuid"].(string)], Not(Equals), "")
1266 // verify kc.WritableLocalRoots
1267 c.Check(len(kc.WritableLocalRoots()), Equals, 3)
1268 for _, root := range kc.WritableLocalRoots() {
1269 c.Check(root, Matches, "http://localhost:\\d+")
1271 c.Assert(kc.WritableLocalRoots()[blobKeepService["uuid"].(string)], Not(Equals), "")
1273 c.Assert(kc.replicasPerService, Equals, 0)
1274 c.Assert(kc.foundNonDiskSvc, Equals, true)
1275 c.Assert(kc.httpClient().(*http.Client).Timeout, Equals, 300*time.Second)