7 "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
8 "git.curoverse.com/arvados.git/sdk/go/arvadostest"
9 "git.curoverse.com/arvados.git/sdk/go/streamer"
22 // Gocheck boilerplate
23 func Test(t *testing.T) {
27 // Gocheck boilerplate
28 var _ = Suite(&ServerRequiredSuite{})
29 var _ = Suite(&StandaloneSuite{})
31 var no_server = flag.Bool("no-server", false, "Skip 'ServerRequireSuite'")
33 // Tests that require the Keep server running
34 type ServerRequiredSuite struct{}
37 type StandaloneSuite struct{}
39 func pythonDir() string {
41 return fmt.Sprintf("%s/../../python/tests", cwd)
44 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
46 c.Skip("Skipping tests that require server")
49 arvadostest.StartAPI()
50 arvadostest.StartKeep(2, false)
53 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
57 arvadostest.StopKeep(2)
61 func (s *ServerRequiredSuite) TestMakeKeepClient(c *C) {
62 arv, err := arvadosclient.MakeArvadosClient()
63 c.Assert(err, Equals, nil)
65 kc, err := MakeKeepClient(&arv)
67 c.Assert(err, Equals, nil)
68 c.Check(len(kc.LocalRoots()), Equals, 2)
69 for _, root := range kc.LocalRoots() {
70 c.Check(root, Matches, "http://localhost:\\d+")
74 func (s *ServerRequiredSuite) TestDefaultReplications(c *C) {
75 arv, err := arvadosclient.MakeArvadosClient()
76 c.Assert(err, Equals, nil)
78 kc, err := MakeKeepClient(&arv)
79 c.Assert(kc.Want_replicas, Equals, 2)
81 arv.DiscoveryDoc["defaultCollectionReplication"] = 3.0
82 kc, err = MakeKeepClient(&arv)
83 c.Assert(kc.Want_replicas, Equals, 3)
85 arv.DiscoveryDoc["defaultCollectionReplication"] = 1.0
86 kc, err = MakeKeepClient(&arv)
87 c.Assert(kc.Want_replicas, Equals, 1)
90 type StubPutHandler struct {
98 func (sph StubPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
99 sph.c.Check(req.URL.Path, Equals, "/"+sph.expectPath)
100 sph.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sph.expectApiToken))
101 body, err := ioutil.ReadAll(req.Body)
102 sph.c.Check(err, Equals, nil)
103 sph.c.Check(body, DeepEquals, []byte(sph.expectBody))
104 resp.WriteHeader(200)
105 sph.handled <- fmt.Sprintf("http://%s", req.Host)
108 func RunFakeKeepServer(st http.Handler) (ks KeepServer) {
110 ks.listener, err = net.ListenTCP("tcp", &net.TCPAddr{Port: 0})
112 panic(fmt.Sprintf("Could not listen on any port"))
114 ks.url = fmt.Sprintf("http://%s", ks.listener.Addr().String())
115 go http.Serve(ks.listener, st)
119 func UploadToStubHelper(c *C, st http.Handler, f func(*KeepClient, string,
120 io.ReadCloser, io.WriteCloser, chan uploadStatus)) {
122 ks := RunFakeKeepServer(st)
123 defer ks.listener.Close()
125 arv, _ := arvadosclient.MakeArvadosClient()
126 arv.ApiToken = "abc123"
128 kc, _ := MakeKeepClient(&arv)
130 reader, writer := io.Pipe()
131 upload_status := make(chan uploadStatus)
133 f(kc, ks.url, reader, writer, upload_status)
136 func (s *StandaloneSuite) TestUploadToStubKeepServer(c *C) {
137 log.Printf("TestUploadToStubKeepServer")
139 st := StubPutHandler{
141 "acbd18db4cc2f85cedef654fccc4a4d8",
146 UploadToStubHelper(c, st,
147 func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, upload_status chan uploadStatus) {
149 go kc.uploadToKeepServer(url, st.expectPath, reader, upload_status, int64(len("foo")), 0)
151 writer.Write([]byte("foo"))
155 status := <-upload_status
156 c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, ""})
159 log.Printf("TestUploadToStubKeepServer done")
162 func (s *StandaloneSuite) TestUploadToStubKeepServerBufferReader(c *C) {
163 log.Printf("TestUploadToStubKeepServerBufferReader")
165 st := StubPutHandler{
167 "acbd18db4cc2f85cedef654fccc4a4d8",
172 UploadToStubHelper(c, st,
173 func(kc *KeepClient, url string, reader io.ReadCloser,
174 writer io.WriteCloser, upload_status chan uploadStatus) {
176 tr := streamer.AsyncStreamFromReader(512, reader)
179 br1 := tr.MakeStreamReader()
181 go kc.uploadToKeepServer(url, st.expectPath, br1, upload_status, 3, 0)
183 writer.Write([]byte("foo"))
188 status := <-upload_status
189 c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, ""})
192 log.Printf("TestUploadToStubKeepServerBufferReader done")
195 type FailHandler struct {
199 func (fh FailHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
200 resp.WriteHeader(500)
201 fh.handled <- fmt.Sprintf("http://%s", req.Host)
204 type FailThenSucceedHandler struct {
207 successhandler StubGetHandler
210 func (fh *FailThenSucceedHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
212 resp.WriteHeader(500)
214 fh.handled <- fmt.Sprintf("http://%s", req.Host)
216 fh.successhandler.ServeHTTP(resp, req)
220 type Error404Handler struct {
224 func (fh Error404Handler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
225 resp.WriteHeader(404)
226 fh.handled <- fmt.Sprintf("http://%s", req.Host)
229 func (s *StandaloneSuite) TestFailedUploadToStubKeepServer(c *C) {
230 log.Printf("TestFailedUploadToStubKeepServer")
235 hash := "acbd18db4cc2f85cedef654fccc4a4d8"
237 UploadToStubHelper(c, st,
238 func(kc *KeepClient, url string, reader io.ReadCloser,
239 writer io.WriteCloser, upload_status chan uploadStatus) {
241 go kc.uploadToKeepServer(url, hash, reader, upload_status, 3, 0)
243 writer.Write([]byte("foo"))
248 status := <-upload_status
249 c.Check(status.url, Equals, fmt.Sprintf("%s/%s", url, hash))
250 c.Check(status.statusCode, Equals, 500)
252 log.Printf("TestFailedUploadToStubKeepServer done")
255 type KeepServer struct {
256 listener net.Listener
260 func RunSomeFakeKeepServers(st http.Handler, n int) (ks []KeepServer) {
261 ks = make([]KeepServer, n)
263 for i := 0; i < n; i += 1 {
264 ks[i] = RunFakeKeepServer(st)
270 func (s *StandaloneSuite) TestPutB(c *C) {
271 log.Printf("TestPutB")
273 hash := Md5String("foo")
275 st := StubPutHandler{
280 make(chan string, 5)}
282 arv, _ := arvadosclient.MakeArvadosClient()
283 kc, _ := MakeKeepClient(&arv)
286 arv.ApiToken = "abc123"
287 localRoots := make(map[string]string)
288 writableLocalRoots := make(map[string]string)
290 ks := RunSomeFakeKeepServers(st, 5)
292 for i, k := range ks {
293 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
294 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
295 defer k.listener.Close()
298 kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
300 kc.PutB([]byte("foo"))
302 shuff := NewRootSorter(
303 kc.LocalRoots(), Md5String("foo")).GetSortedRoots()
307 c.Check((s1 == shuff[0] && s2 == shuff[1]) ||
308 (s1 == shuff[1] && s2 == shuff[0]),
312 log.Printf("TestPutB done")
315 func (s *StandaloneSuite) TestPutHR(c *C) {
316 log.Printf("TestPutHR")
318 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
320 st := StubPutHandler{
325 make(chan string, 5)}
327 arv, _ := arvadosclient.MakeArvadosClient()
328 kc, _ := MakeKeepClient(&arv)
331 arv.ApiToken = "abc123"
332 localRoots := make(map[string]string)
333 writableLocalRoots := make(map[string]string)
335 ks := RunSomeFakeKeepServers(st, 5)
337 for i, k := range ks {
338 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
339 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
340 defer k.listener.Close()
343 kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
345 reader, writer := io.Pipe()
348 writer.Write([]byte("foo"))
352 kc.PutHR(hash, reader, 3)
354 shuff := NewRootSorter(kc.LocalRoots(), hash).GetSortedRoots()
360 c.Check((s1 == shuff[0] && s2 == shuff[1]) ||
361 (s1 == shuff[1] && s2 == shuff[0]),
365 log.Printf("TestPutHR done")
368 func (s *StandaloneSuite) TestPutWithFail(c *C) {
369 log.Printf("TestPutWithFail")
371 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
373 st := StubPutHandler{
378 make(chan string, 4)}
381 make(chan string, 1)}
383 arv, err := arvadosclient.MakeArvadosClient()
384 kc, _ := MakeKeepClient(&arv)
387 arv.ApiToken = "abc123"
388 localRoots := make(map[string]string)
389 writableLocalRoots := make(map[string]string)
391 ks1 := RunSomeFakeKeepServers(st, 4)
392 ks2 := RunSomeFakeKeepServers(fh, 1)
394 for i, k := range ks1 {
395 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
396 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
397 defer k.listener.Close()
399 for i, k := range ks2 {
400 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
401 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
402 defer k.listener.Close()
405 kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
407 shuff := NewRootSorter(
408 kc.LocalRoots(), Md5String("foo")).GetSortedRoots()
410 phash, replicas, err := kc.PutB([]byte("foo"))
414 c.Check(err, Equals, nil)
415 c.Check(phash, Equals, "")
416 c.Check(replicas, Equals, 2)
421 c.Check((s1 == shuff[1] && s2 == shuff[2]) ||
422 (s1 == shuff[2] && s2 == shuff[1]),
427 func (s *StandaloneSuite) TestPutWithTooManyFail(c *C) {
428 log.Printf("TestPutWithTooManyFail")
430 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
432 st := StubPutHandler{
437 make(chan string, 1)}
440 make(chan string, 4)}
442 arv, err := arvadosclient.MakeArvadosClient()
443 kc, _ := MakeKeepClient(&arv)
447 arv.ApiToken = "abc123"
448 localRoots := make(map[string]string)
449 writableLocalRoots := make(map[string]string)
451 ks1 := RunSomeFakeKeepServers(st, 1)
452 ks2 := RunSomeFakeKeepServers(fh, 4)
454 for i, k := range ks1 {
455 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
456 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
457 defer k.listener.Close()
459 for i, k := range ks2 {
460 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
461 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
462 defer k.listener.Close()
465 kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
467 _, replicas, err := kc.PutB([]byte("foo"))
469 c.Check(err, Equals, InsufficientReplicasError)
470 c.Check(replicas, Equals, 1)
471 c.Check(<-st.handled, Equals, ks1[0].url)
473 log.Printf("TestPutWithTooManyFail done")
476 type StubGetHandler struct {
479 expectApiToken string
484 func (sgh StubGetHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
485 sgh.c.Check(req.URL.Path, Equals, "/"+sgh.expectPath)
486 sgh.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sgh.expectApiToken))
487 resp.WriteHeader(sgh.httpStatus)
488 resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(sgh.body)))
492 func (s *StandaloneSuite) TestGet(c *C) {
493 log.Printf("TestGet")
495 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
497 st := StubGetHandler{
504 ks := RunFakeKeepServer(st)
505 defer ks.listener.Close()
507 arv, err := arvadosclient.MakeArvadosClient()
508 kc, _ := MakeKeepClient(&arv)
509 arv.ApiToken = "abc123"
510 kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
512 r, n, url2, err := kc.Get(hash)
514 c.Check(err, Equals, nil)
515 c.Check(n, Equals, int64(3))
516 c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks.url, hash))
518 content, err2 := ioutil.ReadAll(r)
519 c.Check(err2, Equals, nil)
520 c.Check(content, DeepEquals, []byte("foo"))
522 log.Printf("TestGet done")
525 func (s *StandaloneSuite) TestGet404(c *C) {
526 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
528 st := Error404Handler{make(chan string, 1)}
530 ks := RunFakeKeepServer(st)
531 defer ks.listener.Close()
533 arv, err := arvadosclient.MakeArvadosClient()
534 kc, _ := MakeKeepClient(&arv)
535 arv.ApiToken = "abc123"
536 kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
538 r, n, url2, err := kc.Get(hash)
539 c.Check(err, Equals, BlockNotFound)
540 c.Check(n, Equals, int64(0))
541 c.Check(url2, Equals, "")
542 c.Check(r, Equals, nil)
545 func (s *StandaloneSuite) TestGetFail(c *C) {
546 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
548 st := FailHandler{make(chan string, 1)}
550 ks := RunFakeKeepServer(st)
551 defer ks.listener.Close()
553 arv, err := arvadosclient.MakeArvadosClient()
554 kc, _ := MakeKeepClient(&arv)
555 arv.ApiToken = "abc123"
556 kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
559 r, n, url2, err := kc.Get(hash)
560 errNotFound, _ := err.(*ErrNotFound)
561 c.Check(errNotFound, NotNil)
562 c.Check(strings.Contains(errNotFound.Error(), "HTTP 500"), Equals, true)
563 c.Check(errNotFound.Temporary(), Equals, true)
564 c.Check(n, Equals, int64(0))
565 c.Check(url2, Equals, "")
566 c.Check(r, Equals, nil)
569 func (s *StandaloneSuite) TestGetFailRetry(c *C) {
570 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
572 st := &FailThenSucceedHandler{make(chan string, 1), 0,
580 ks := RunFakeKeepServer(st)
581 defer ks.listener.Close()
583 arv, err := arvadosclient.MakeArvadosClient()
584 kc, _ := MakeKeepClient(&arv)
585 arv.ApiToken = "abc123"
586 kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
588 r, n, url2, err := kc.Get(hash)
590 c.Check(err, Equals, nil)
591 c.Check(n, Equals, int64(3))
592 c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks.url, hash))
594 content, err2 := ioutil.ReadAll(r)
595 c.Check(err2, Equals, nil)
596 c.Check(content, DeepEquals, []byte("foo"))
599 func (s *StandaloneSuite) TestGetNetError(c *C) {
600 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
602 arv, err := arvadosclient.MakeArvadosClient()
603 kc, _ := MakeKeepClient(&arv)
604 arv.ApiToken = "abc123"
605 kc.SetServiceRoots(map[string]string{"x": "http://localhost:62222"}, nil, nil)
607 r, n, url2, err := kc.Get(hash)
608 errNotFound, _ := err.(*ErrNotFound)
609 c.Check(errNotFound, NotNil)
610 c.Check(strings.Contains(errNotFound.Error(), "connection refused"), Equals, true)
611 c.Check(errNotFound.Temporary(), Equals, true)
612 c.Check(n, Equals, int64(0))
613 c.Check(url2, Equals, "")
614 c.Check(r, Equals, nil)
617 func (s *StandaloneSuite) TestGetWithServiceHint(c *C) {
618 uuid := "zzzzz-bi6l4-123451234512345"
619 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
621 // This one shouldn't be used:
622 ks0 := RunFakeKeepServer(StubGetHandler{
628 defer ks0.listener.Close()
629 // This one should be used:
630 ks := RunFakeKeepServer(StubGetHandler{
636 defer ks.listener.Close()
638 arv, err := arvadosclient.MakeArvadosClient()
639 kc, _ := MakeKeepClient(&arv)
640 arv.ApiToken = "abc123"
642 map[string]string{"x": ks0.url},
644 map[string]string{uuid: ks.url})
646 r, n, uri, err := kc.Get(hash + "+K@" + uuid)
648 c.Check(err, Equals, nil)
649 c.Check(n, Equals, int64(3))
650 c.Check(uri, Equals, fmt.Sprintf("%s/%s", ks.url, hash+"+K@"+uuid))
652 content, err := ioutil.ReadAll(r)
653 c.Check(err, Equals, nil)
654 c.Check(content, DeepEquals, []byte("foo"))
657 // Use a service hint to fetch from a local disk service, overriding
658 // rendezvous probe order.
659 func (s *StandaloneSuite) TestGetWithLocalServiceHint(c *C) {
660 uuid := "zzzzz-bi6l4-zzzzzzzzzzzzzzz"
661 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
663 // This one shouldn't be used, although it appears first in
664 // rendezvous probe order:
665 ks0 := RunFakeKeepServer(StubGetHandler{
671 defer ks0.listener.Close()
672 // This one should be used:
673 ks := RunFakeKeepServer(StubGetHandler{
679 defer ks.listener.Close()
681 arv, err := arvadosclient.MakeArvadosClient()
682 kc, _ := MakeKeepClient(&arv)
683 arv.ApiToken = "abc123"
686 "zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url,
687 "zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url,
688 "zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url,
692 "zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url,
693 "zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url,
694 "zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url,
698 r, n, uri, err := kc.Get(hash + "+K@" + uuid)
700 c.Check(err, Equals, nil)
701 c.Check(n, Equals, int64(3))
702 c.Check(uri, Equals, fmt.Sprintf("%s/%s", ks.url, hash+"+K@"+uuid))
704 content, err := ioutil.ReadAll(r)
705 c.Check(err, Equals, nil)
706 c.Check(content, DeepEquals, []byte("foo"))
709 func (s *StandaloneSuite) TestGetWithServiceHintFailoverToLocals(c *C) {
710 uuid := "zzzzz-bi6l4-123451234512345"
711 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
713 ksLocal := RunFakeKeepServer(StubGetHandler{
719 defer ksLocal.listener.Close()
720 ksGateway := RunFakeKeepServer(StubGetHandler{
724 http.StatusInternalServerError,
726 defer ksGateway.listener.Close()
728 arv, err := arvadosclient.MakeArvadosClient()
729 kc, _ := MakeKeepClient(&arv)
730 arv.ApiToken = "abc123"
732 map[string]string{"zzzzz-bi6l4-keepdisk0000000": ksLocal.url},
734 map[string]string{uuid: ksGateway.url})
736 r, n, uri, err := kc.Get(hash + "+K@" + uuid)
737 c.Assert(err, Equals, nil)
739 c.Check(n, Equals, int64(3))
740 c.Check(uri, Equals, fmt.Sprintf("%s/%s", ksLocal.url, hash+"+K@"+uuid))
742 content, err := ioutil.ReadAll(r)
743 c.Check(err, Equals, nil)
744 c.Check(content, DeepEquals, []byte("foo"))
747 type BarHandler struct {
751 func (this BarHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
752 resp.Write([]byte("bar"))
753 this.handled <- fmt.Sprintf("http://%s", req.Host)
756 func (s *StandaloneSuite) TestChecksum(c *C) {
757 foohash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
758 barhash := fmt.Sprintf("%x", md5.Sum([]byte("bar")))
760 st := BarHandler{make(chan string, 1)}
762 ks := RunFakeKeepServer(st)
763 defer ks.listener.Close()
765 arv, err := arvadosclient.MakeArvadosClient()
766 kc, _ := MakeKeepClient(&arv)
767 arv.ApiToken = "abc123"
768 kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
770 r, n, _, err := kc.Get(barhash)
771 _, err = ioutil.ReadAll(r)
772 c.Check(n, Equals, int64(3))
773 c.Check(err, Equals, nil)
777 r, n, _, err = kc.Get(foohash)
778 _, err = ioutil.ReadAll(r)
779 c.Check(n, Equals, int64(3))
780 c.Check(err, Equals, BadChecksum)
785 func (s *StandaloneSuite) TestGetWithFailures(c *C) {
786 content := []byte("waz")
787 hash := fmt.Sprintf("%x", md5.Sum(content))
789 fh := Error404Handler{
790 make(chan string, 4)}
792 st := StubGetHandler{
799 arv, err := arvadosclient.MakeArvadosClient()
800 kc, _ := MakeKeepClient(&arv)
801 arv.ApiToken = "abc123"
802 localRoots := make(map[string]string)
803 writableLocalRoots := make(map[string]string)
805 ks1 := RunSomeFakeKeepServers(st, 1)
806 ks2 := RunSomeFakeKeepServers(fh, 4)
808 for i, k := range ks1 {
809 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
810 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
811 defer k.listener.Close()
813 for i, k := range ks2 {
814 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
815 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
816 defer k.listener.Close()
819 kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
822 // This test works only if one of the failing services is
823 // attempted before the succeeding service. Otherwise,
824 // <-fh.handled below will just hang! (Probe order depends on
825 // the choice of block content "waz" and the UUIDs of the fake
826 // servers, so we just tried different strings until we found
827 // an example that passes this Assert.)
828 c.Assert(NewRootSorter(localRoots, hash).GetSortedRoots()[0], Not(Equals), ks1[0].url)
830 r, n, url2, err := kc.Get(hash)
833 c.Check(err, Equals, nil)
834 c.Check(n, Equals, int64(3))
835 c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks1[0].url, hash))
837 read_content, err2 := ioutil.ReadAll(r)
838 c.Check(err2, Equals, nil)
839 c.Check(read_content, DeepEquals, content)
842 func (s *ServerRequiredSuite) TestPutGetHead(c *C) {
843 content := []byte("TestPutGetHead")
845 arv, err := arvadosclient.MakeArvadosClient()
846 kc, err := MakeKeepClient(&arv)
847 c.Assert(err, Equals, nil)
849 hash := fmt.Sprintf("%x", md5.Sum(content))
852 n, _, err := kc.Ask(hash)
853 c.Check(err, Equals, BlockNotFound)
854 c.Check(n, Equals, int64(0))
857 hash2, replicas, err := kc.PutB(content)
858 c.Check(hash2, Matches, fmt.Sprintf(`%s\+%d\b.*`, hash, len(content)))
859 c.Check(replicas, Equals, 2)
860 c.Check(err, Equals, nil)
863 r, n, url2, err := kc.Get(hash)
864 c.Check(err, Equals, nil)
865 c.Check(n, Equals, int64(len(content)))
866 c.Check(url2, Matches, fmt.Sprintf("http://localhost:\\d+/%s", hash))
868 read_content, err2 := ioutil.ReadAll(r)
869 c.Check(err2, Equals, nil)
870 c.Check(read_content, DeepEquals, content)
873 n, url2, err := kc.Ask(hash)
874 c.Check(err, Equals, nil)
875 c.Check(n, Equals, int64(len(content)))
876 c.Check(url2, Matches, fmt.Sprintf("http://localhost:\\d+/%s", hash))
880 type StubProxyHandler struct {
884 func (this StubProxyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
885 resp.Header().Set("X-Keep-Replicas-Stored", "2")
886 this.handled <- fmt.Sprintf("http://%s", req.Host)
889 func (s *StandaloneSuite) TestPutProxy(c *C) {
890 log.Printf("TestPutProxy")
892 st := StubProxyHandler{make(chan string, 1)}
894 arv, err := arvadosclient.MakeArvadosClient()
895 kc, _ := MakeKeepClient(&arv)
898 kc.Using_proxy = true
899 arv.ApiToken = "abc123"
900 localRoots := make(map[string]string)
901 writableLocalRoots := make(map[string]string)
903 ks1 := RunSomeFakeKeepServers(st, 1)
905 for i, k := range ks1 {
906 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
907 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
908 defer k.listener.Close()
911 kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
913 _, replicas, err := kc.PutB([]byte("foo"))
916 c.Check(err, Equals, nil)
917 c.Check(replicas, Equals, 2)
919 log.Printf("TestPutProxy done")
922 func (s *StandaloneSuite) TestPutProxyInsufficientReplicas(c *C) {
923 log.Printf("TestPutProxy")
925 st := StubProxyHandler{make(chan string, 1)}
927 arv, err := arvadosclient.MakeArvadosClient()
928 kc, _ := MakeKeepClient(&arv)
931 kc.Using_proxy = true
932 arv.ApiToken = "abc123"
933 localRoots := make(map[string]string)
934 writableLocalRoots := make(map[string]string)
936 ks1 := RunSomeFakeKeepServers(st, 1)
938 for i, k := range ks1 {
939 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
940 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
941 defer k.listener.Close()
943 kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
945 _, replicas, err := kc.PutB([]byte("foo"))
948 c.Check(err, Equals, InsufficientReplicasError)
949 c.Check(replicas, Equals, 2)
951 log.Printf("TestPutProxy done")
954 func (s *StandaloneSuite) TestMakeLocator(c *C) {
955 l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce+3+Aabcde@12345678")
956 c.Check(err, Equals, nil)
957 c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
958 c.Check(l.Size, Equals, 3)
959 c.Check(l.Hints, DeepEquals, []string{"3", "Aabcde@12345678"})
962 func (s *StandaloneSuite) TestMakeLocatorNoHints(c *C) {
963 l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce")
964 c.Check(err, Equals, nil)
965 c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
966 c.Check(l.Size, Equals, -1)
967 c.Check(l.Hints, DeepEquals, []string{})
970 func (s *StandaloneSuite) TestMakeLocatorNoSizeHint(c *C) {
971 l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce+Aabcde@12345678")
972 c.Check(err, Equals, nil)
973 c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
974 c.Check(l.Size, Equals, -1)
975 c.Check(l.Hints, DeepEquals, []string{"Aabcde@12345678"})
978 func (s *StandaloneSuite) TestMakeLocatorPreservesUnrecognizedHints(c *C) {
979 str := "91f372a266fe2bf2823cb8ec7fda31ce+3+Unknown+Kzzzzz+Afoobar"
980 l, err := MakeLocator(str)
981 c.Check(err, Equals, nil)
982 c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
983 c.Check(l.Size, Equals, 3)
984 c.Check(l.Hints, DeepEquals, []string{"3", "Unknown", "Kzzzzz", "Afoobar"})
985 c.Check(l.String(), Equals, str)
988 func (s *StandaloneSuite) TestMakeLocatorInvalidInput(c *C) {
989 _, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31c")
990 c.Check(err, Equals, InvalidLocatorError)
993 func (s *StandaloneSuite) TestPutBWant2ReplicasWithOnlyOneWritableLocalRoot(c *C) {
994 hash := Md5String("foo")
996 st := StubPutHandler{
1001 make(chan string, 5)}
1003 arv, _ := arvadosclient.MakeArvadosClient()
1004 kc, _ := MakeKeepClient(&arv)
1006 kc.Want_replicas = 2
1007 arv.ApiToken = "abc123"
1008 localRoots := make(map[string]string)
1009 writableLocalRoots := make(map[string]string)
1011 ks := RunSomeFakeKeepServers(st, 5)
1013 for i, k := range ks {
1014 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
1016 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
1018 defer k.listener.Close()
1021 kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
1023 _, replicas, err := kc.PutB([]byte("foo"))
1025 c.Check(err, Equals, InsufficientReplicasError)
1026 c.Check(replicas, Equals, 1)
1028 c.Check(<-st.handled, Equals, localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", 0)])
1031 func (s *StandaloneSuite) TestPutBWithNoWritableLocalRoots(c *C) {
1032 hash := Md5String("foo")
1034 st := StubPutHandler{
1039 make(chan string, 5)}
1041 arv, _ := arvadosclient.MakeArvadosClient()
1042 kc, _ := MakeKeepClient(&arv)
1044 kc.Want_replicas = 2
1045 arv.ApiToken = "abc123"
1046 localRoots := make(map[string]string)
1047 writableLocalRoots := make(map[string]string)
1049 ks := RunSomeFakeKeepServers(st, 5)
1051 for i, k := range ks {
1052 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
1053 defer k.listener.Close()
1056 kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
1058 _, replicas, err := kc.PutB([]byte("foo"))
1060 c.Check(err, Equals, InsufficientReplicasError)
1061 c.Check(replicas, Equals, 0)
1064 type StubGetIndexHandler struct {
1067 expectAPIToken string
1072 func (h StubGetIndexHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
1073 h.c.Check(req.URL.Path, Equals, h.expectPath)
1074 h.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", h.expectAPIToken))
1075 resp.WriteHeader(h.httpStatus)
1076 resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(h.body)))
1080 func (s *StandaloneSuite) TestGetIndexWithNoPrefix(c *C) {
1081 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
1083 st := StubGetIndexHandler{
1088 []byte(hash + "+3 1443559274\n\n")}
1090 ks := RunFakeKeepServer(st)
1091 defer ks.listener.Close()
1093 arv, err := arvadosclient.MakeArvadosClient()
1094 kc, _ := MakeKeepClient(&arv)
1095 arv.ApiToken = "abc123"
1096 kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
1098 r, err := kc.GetIndex("x", "")
1099 c.Check(err, Equals, nil)
1101 content, err2 := ioutil.ReadAll(r)
1102 c.Check(err2, Equals, nil)
1103 c.Check(content, DeepEquals, st.body[0:len(st.body)-1])
1106 func (s *StandaloneSuite) TestGetIndexWithPrefix(c *C) {
1107 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
1109 st := StubGetIndexHandler{
1111 "/index/" + hash[0:3],
1114 []byte(hash + "+3 1443559274\n\n")}
1116 ks := RunFakeKeepServer(st)
1117 defer ks.listener.Close()
1119 arv, err := arvadosclient.MakeArvadosClient()
1120 kc, _ := MakeKeepClient(&arv)
1121 arv.ApiToken = "abc123"
1122 kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
1124 r, err := kc.GetIndex("x", hash[0:3])
1125 c.Check(err, Equals, nil)
1127 content, err2 := ioutil.ReadAll(r)
1128 c.Check(err2, Equals, nil)
1129 c.Check(content, DeepEquals, st.body[0:len(st.body)-1])
1132 func (s *StandaloneSuite) TestGetIndexIncomplete(c *C) {
1133 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
1135 st := StubGetIndexHandler{
1137 "/index/" + hash[0:3],
1142 ks := RunFakeKeepServer(st)
1143 defer ks.listener.Close()
1145 arv, err := arvadosclient.MakeArvadosClient()
1146 kc, _ := MakeKeepClient(&arv)
1147 arv.ApiToken = "abc123"
1148 kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
1150 _, err = kc.GetIndex("x", hash[0:3])
1151 c.Check(err, Equals, ErrIncompleteIndex)
1154 func (s *StandaloneSuite) TestGetIndexWithNoSuchServer(c *C) {
1155 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
1157 st := StubGetIndexHandler{
1159 "/index/" + hash[0:3],
1164 ks := RunFakeKeepServer(st)
1165 defer ks.listener.Close()
1167 arv, err := arvadosclient.MakeArvadosClient()
1168 kc, _ := MakeKeepClient(&arv)
1169 arv.ApiToken = "abc123"
1170 kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
1172 _, err = kc.GetIndex("y", hash[0:3])
1173 c.Check(err, Equals, ErrNoSuchKeepServer)
1176 func (s *StandaloneSuite) TestGetIndexWithNoSuchPrefix(c *C) {
1177 st := StubGetIndexHandler{
1184 ks := RunFakeKeepServer(st)
1185 defer ks.listener.Close()
1187 arv, err := arvadosclient.MakeArvadosClient()
1188 kc, _ := MakeKeepClient(&arv)
1189 arv.ApiToken = "abc123"
1190 kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
1192 r, err := kc.GetIndex("x", "abcd")
1193 c.Check(err, Equals, nil)
1195 content, err2 := ioutil.ReadAll(r)
1196 c.Check(err2, Equals, nil)
1197 c.Check(content, DeepEquals, st.body[0:len(st.body)-1])
1200 type FailThenSucceedPutHandler struct {
1203 successhandler StubPutHandler
1206 func (h *FailThenSucceedPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
1208 resp.WriteHeader(500)
1210 h.handled <- fmt.Sprintf("http://%s", req.Host)
1212 h.successhandler.ServeHTTP(resp, req)
1216 func (s *StandaloneSuite) TestPutBRetry(c *C) {
1217 st := &FailThenSucceedPutHandler{make(chan string, 1), 0,
1223 make(chan string, 5)}}
1225 arv, _ := arvadosclient.MakeArvadosClient()
1226 kc, _ := MakeKeepClient(&arv)
1228 kc.Want_replicas = 2
1229 arv.ApiToken = "abc123"
1230 localRoots := make(map[string]string)
1231 writableLocalRoots := make(map[string]string)
1233 ks := RunSomeFakeKeepServers(st, 2)
1235 for i, k := range ks {
1236 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
1237 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
1238 defer k.listener.Close()
1241 kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
1243 hash, replicas, err := kc.PutB([]byte("foo"))
1245 c.Check(err, Equals, nil)
1246 c.Check(hash, Equals, "")
1247 c.Check(replicas, Equals, 2)
1250 func (s *ServerRequiredSuite) TestMakeKeepClientWithNonDiskTypeService(c *C) {
1251 arv, err := arvadosclient.MakeArvadosClient()
1252 c.Assert(err, Equals, nil)
1254 // Add an additional "testblobstore" keepservice
1255 blobKeepService := make(arvadosclient.Dict)
1256 err = arv.Create("keep_services",
1257 arvadosclient.Dict{"keep_service": arvadosclient.Dict{
1258 "service_host": "localhost",
1259 "service_port": "21321",
1260 "service_type": "testblobstore"}},
1262 c.Assert(err, Equals, nil)
1263 defer func() { arv.Delete("keep_services", blobKeepService["uuid"].(string), nil, nil) }()
1265 // Make a keepclient and ensure that the testblobstore is included
1266 kc, err := MakeKeepClient(&arv)
1267 c.Assert(err, Equals, nil)
1269 // verify kc.LocalRoots
1270 c.Check(len(kc.LocalRoots()), Equals, 3)
1271 for _, root := range kc.LocalRoots() {
1272 c.Check(root, Matches, "http://localhost:\\d+")
1274 c.Assert(kc.LocalRoots()[blobKeepService["uuid"].(string)], NotNil)
1276 // verify kc.GatewayRoots
1277 c.Check(len(kc.GatewayRoots()), Equals, 3)
1278 for _, root := range kc.GatewayRoots() {
1279 c.Check(root, Matches, "http://localhost:\\d+")
1281 c.Assert(kc.GatewayRoots()[blobKeepService["uuid"].(string)], NotNil)
1283 // verify kc.WritableLocalRoots
1284 c.Check(len(kc.WritableLocalRoots()), Equals, 3)
1285 for _, root := range kc.WritableLocalRoots() {
1286 c.Check(root, Matches, "http://localhost:\\d+")
1288 c.Assert(kc.WritableLocalRoots()[blobKeepService["uuid"].(string)], NotNil)
1290 c.Assert(kc.replicasPerService, Equals, 0)
1291 c.Assert(kc.foundNonDiskSvc, Equals, true)
1292 c.Assert(kc.Client.Timeout, Equals, 300*time.Second)