8005: Add tar Ruby build dependency on CentOS 6.
[arvados.git] / sdk / go / keepclient / keepclient_test.go
1 package keepclient
2
3 import (
4         "crypto/md5"
5         "flag"
6         "fmt"
7         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
8         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
9         "git.curoverse.com/arvados.git/sdk/go/streamer"
10         . "gopkg.in/check.v1"
11         "io"
12         "io/ioutil"
13         "log"
14         "net"
15         "net/http"
16         "os"
17         "strings"
18         "testing"
19         "time"
20 )
21
22 // Gocheck boilerplate
23 func Test(t *testing.T) {
24         TestingT(t)
25 }
26
27 // Gocheck boilerplate
28 var _ = Suite(&ServerRequiredSuite{})
29 var _ = Suite(&StandaloneSuite{})
30
31 var no_server = flag.Bool("no-server", false, "Skip 'ServerRequireSuite'")
32
33 // Tests that require the Keep server running
34 type ServerRequiredSuite struct{}
35
36 // Standalone tests
37 type StandaloneSuite struct{}
38
39 func pythonDir() string {
40         cwd, _ := os.Getwd()
41         return fmt.Sprintf("%s/../../python/tests", cwd)
42 }
43
44 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
45         if *no_server {
46                 c.Skip("Skipping tests that require server")
47                 return
48         }
49         arvadostest.StartAPI()
50         arvadostest.StartKeep(2, false)
51 }
52
53 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
54         if *no_server {
55                 return
56         }
57         arvadostest.StopKeep(2)
58         arvadostest.StopAPI()
59 }
60
61 func (s *ServerRequiredSuite) TestMakeKeepClient(c *C) {
62         arv, err := arvadosclient.MakeArvadosClient()
63         c.Assert(err, Equals, nil)
64
65         kc, err := MakeKeepClient(&arv)
66
67         c.Assert(err, Equals, nil)
68         c.Check(len(kc.LocalRoots()), Equals, 2)
69         for _, root := range kc.LocalRoots() {
70                 c.Check(root, Matches, "http://localhost:\\d+")
71         }
72 }
73
74 func (s *ServerRequiredSuite) TestDefaultReplications(c *C) {
75         arv, err := arvadosclient.MakeArvadosClient()
76         c.Assert(err, Equals, nil)
77
78         kc, err := MakeKeepClient(&arv)
79         c.Assert(kc.Want_replicas, Equals, 2)
80
81         arv.DiscoveryDoc["defaultCollectionReplication"] = 3.0
82         kc, err = MakeKeepClient(&arv)
83         c.Assert(kc.Want_replicas, Equals, 3)
84
85         arv.DiscoveryDoc["defaultCollectionReplication"] = 1.0
86         kc, err = MakeKeepClient(&arv)
87         c.Assert(kc.Want_replicas, Equals, 1)
88 }
89
90 type StubPutHandler struct {
91         c              *C
92         expectPath     string
93         expectApiToken string
94         expectBody     string
95         handled        chan string
96 }
97
98 func (sph StubPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
99         sph.c.Check(req.URL.Path, Equals, "/"+sph.expectPath)
100         sph.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sph.expectApiToken))
101         body, err := ioutil.ReadAll(req.Body)
102         sph.c.Check(err, Equals, nil)
103         sph.c.Check(body, DeepEquals, []byte(sph.expectBody))
104         resp.WriteHeader(200)
105         sph.handled <- fmt.Sprintf("http://%s", req.Host)
106 }
107
108 func RunFakeKeepServer(st http.Handler) (ks KeepServer) {
109         var err error
110         ks.listener, err = net.ListenTCP("tcp", &net.TCPAddr{Port: 0})
111         if err != nil {
112                 panic(fmt.Sprintf("Could not listen on any port"))
113         }
114         ks.url = fmt.Sprintf("http://%s", ks.listener.Addr().String())
115         go http.Serve(ks.listener, st)
116         return
117 }
118
119 func UploadToStubHelper(c *C, st http.Handler, f func(*KeepClient, string,
120         io.ReadCloser, io.WriteCloser, chan uploadStatus)) {
121
122         ks := RunFakeKeepServer(st)
123         defer ks.listener.Close()
124
125         arv, _ := arvadosclient.MakeArvadosClient()
126         arv.ApiToken = "abc123"
127
128         kc, _ := MakeKeepClient(&arv)
129
130         reader, writer := io.Pipe()
131         upload_status := make(chan uploadStatus)
132
133         f(kc, ks.url, reader, writer, upload_status)
134 }
135
136 func (s *StandaloneSuite) TestUploadToStubKeepServer(c *C) {
137         log.Printf("TestUploadToStubKeepServer")
138
139         st := StubPutHandler{
140                 c,
141                 "acbd18db4cc2f85cedef654fccc4a4d8",
142                 "abc123",
143                 "foo",
144                 make(chan string)}
145
146         UploadToStubHelper(c, st,
147                 func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, upload_status chan uploadStatus) {
148
149                         go kc.uploadToKeepServer(url, st.expectPath, reader, upload_status, int64(len("foo")), 0)
150
151                         writer.Write([]byte("foo"))
152                         writer.Close()
153
154                         <-st.handled
155                         status := <-upload_status
156                         c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, ""})
157                 })
158 }
159
160 func (s *StandaloneSuite) TestUploadToStubKeepServerBufferReader(c *C) {
161         st := StubPutHandler{
162                 c,
163                 "acbd18db4cc2f85cedef654fccc4a4d8",
164                 "abc123",
165                 "foo",
166                 make(chan string)}
167
168         UploadToStubHelper(c, st,
169                 func(kc *KeepClient, url string, reader io.ReadCloser,
170                         writer io.WriteCloser, upload_status chan uploadStatus) {
171
172                         tr := streamer.AsyncStreamFromReader(512, reader)
173                         defer tr.Close()
174
175                         br1 := tr.MakeStreamReader()
176
177                         go kc.uploadToKeepServer(url, st.expectPath, br1, upload_status, 3, 0)
178
179                         writer.Write([]byte("foo"))
180                         writer.Close()
181
182                         <-st.handled
183
184                         status := <-upload_status
185                         c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, ""})
186                 })
187 }
188
189 type FailHandler struct {
190         handled chan string
191 }
192
193 func (fh FailHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
194         resp.WriteHeader(500)
195         fh.handled <- fmt.Sprintf("http://%s", req.Host)
196 }
197
198 type FailThenSucceedHandler struct {
199         handled        chan string
200         count          int
201         successhandler StubGetHandler
202 }
203
204 func (fh *FailThenSucceedHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
205         if fh.count == 0 {
206                 resp.WriteHeader(500)
207                 fh.count += 1
208                 fh.handled <- fmt.Sprintf("http://%s", req.Host)
209         } else {
210                 fh.successhandler.ServeHTTP(resp, req)
211         }
212 }
213
214 type Error404Handler struct {
215         handled chan string
216 }
217
218 func (fh Error404Handler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
219         resp.WriteHeader(404)
220         fh.handled <- fmt.Sprintf("http://%s", req.Host)
221 }
222
223 func (s *StandaloneSuite) TestFailedUploadToStubKeepServer(c *C) {
224         st := FailHandler{
225                 make(chan string)}
226
227         hash := "acbd18db4cc2f85cedef654fccc4a4d8"
228
229         UploadToStubHelper(c, st,
230                 func(kc *KeepClient, url string, reader io.ReadCloser,
231                         writer io.WriteCloser, upload_status chan uploadStatus) {
232
233                         go kc.uploadToKeepServer(url, hash, reader, upload_status, 3, 0)
234
235                         writer.Write([]byte("foo"))
236                         writer.Close()
237
238                         <-st.handled
239
240                         status := <-upload_status
241                         c.Check(status.url, Equals, fmt.Sprintf("%s/%s", url, hash))
242                         c.Check(status.statusCode, Equals, 500)
243                 })
244 }
245
246 type KeepServer struct {
247         listener net.Listener
248         url      string
249 }
250
251 func RunSomeFakeKeepServers(st http.Handler, n int) (ks []KeepServer) {
252         ks = make([]KeepServer, n)
253
254         for i := 0; i < n; i += 1 {
255                 ks[i] = RunFakeKeepServer(st)
256         }
257
258         return ks
259 }
260
261 func (s *StandaloneSuite) TestPutB(c *C) {
262         hash := Md5String("foo")
263
264         st := StubPutHandler{
265                 c,
266                 hash,
267                 "abc123",
268                 "foo",
269                 make(chan string, 5)}
270
271         arv, _ := arvadosclient.MakeArvadosClient()
272         kc, _ := MakeKeepClient(&arv)
273
274         kc.Want_replicas = 2
275         arv.ApiToken = "abc123"
276         localRoots := make(map[string]string)
277         writableLocalRoots := make(map[string]string)
278
279         ks := RunSomeFakeKeepServers(st, 5)
280
281         for i, k := range ks {
282                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
283                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
284                 defer k.listener.Close()
285         }
286
287         kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
288
289         kc.PutB([]byte("foo"))
290
291         shuff := NewRootSorter(
292                 kc.LocalRoots(), Md5String("foo")).GetSortedRoots()
293
294         s1 := <-st.handled
295         s2 := <-st.handled
296         c.Check((s1 == shuff[0] && s2 == shuff[1]) ||
297                 (s1 == shuff[1] && s2 == shuff[0]),
298                 Equals,
299                 true)
300 }
301
302 func (s *StandaloneSuite) TestPutHR(c *C) {
303         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
304
305         st := StubPutHandler{
306                 c,
307                 hash,
308                 "abc123",
309                 "foo",
310                 make(chan string, 5)}
311
312         arv, _ := arvadosclient.MakeArvadosClient()
313         kc, _ := MakeKeepClient(&arv)
314
315         kc.Want_replicas = 2
316         arv.ApiToken = "abc123"
317         localRoots := make(map[string]string)
318         writableLocalRoots := make(map[string]string)
319
320         ks := RunSomeFakeKeepServers(st, 5)
321
322         for i, k := range ks {
323                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
324                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
325                 defer k.listener.Close()
326         }
327
328         kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
329
330         reader, writer := io.Pipe()
331
332         go func() {
333                 writer.Write([]byte("foo"))
334                 writer.Close()
335         }()
336
337         kc.PutHR(hash, reader, 3)
338
339         shuff := NewRootSorter(kc.LocalRoots(), hash).GetSortedRoots()
340
341         s1 := <-st.handled
342         s2 := <-st.handled
343
344         c.Check((s1 == shuff[0] && s2 == shuff[1]) ||
345                 (s1 == shuff[1] && s2 == shuff[0]),
346                 Equals,
347                 true)
348 }
349
350 func (s *StandaloneSuite) TestPutWithFail(c *C) {
351         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
352
353         st := StubPutHandler{
354                 c,
355                 hash,
356                 "abc123",
357                 "foo",
358                 make(chan string, 4)}
359
360         fh := FailHandler{
361                 make(chan string, 1)}
362
363         arv, err := arvadosclient.MakeArvadosClient()
364         kc, _ := MakeKeepClient(&arv)
365
366         kc.Want_replicas = 2
367         arv.ApiToken = "abc123"
368         localRoots := make(map[string]string)
369         writableLocalRoots := make(map[string]string)
370
371         ks1 := RunSomeFakeKeepServers(st, 4)
372         ks2 := RunSomeFakeKeepServers(fh, 1)
373
374         for i, k := range ks1 {
375                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
376                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
377                 defer k.listener.Close()
378         }
379         for i, k := range ks2 {
380                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
381                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
382                 defer k.listener.Close()
383         }
384
385         kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
386
387         shuff := NewRootSorter(
388                 kc.LocalRoots(), Md5String("foo")).GetSortedRoots()
389         c.Logf("%+v", shuff)
390
391         phash, replicas, err := kc.PutB([]byte("foo"))
392
393         <-fh.handled
394
395         c.Check(err, Equals, nil)
396         c.Check(phash, Equals, "")
397         c.Check(replicas, Equals, 2)
398
399         s1 := <-st.handled
400         s2 := <-st.handled
401
402         c.Check((s1 == shuff[1] && s2 == shuff[2]) ||
403                 (s1 == shuff[2] && s2 == shuff[1]),
404                 Equals,
405                 true)
406 }
407
408 func (s *StandaloneSuite) TestPutWithTooManyFail(c *C) {
409         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
410
411         st := StubPutHandler{
412                 c,
413                 hash,
414                 "abc123",
415                 "foo",
416                 make(chan string, 1)}
417
418         fh := FailHandler{
419                 make(chan string, 4)}
420
421         arv, err := arvadosclient.MakeArvadosClient()
422         kc, _ := MakeKeepClient(&arv)
423
424         kc.Want_replicas = 2
425         kc.Retries = 0
426         arv.ApiToken = "abc123"
427         localRoots := make(map[string]string)
428         writableLocalRoots := make(map[string]string)
429
430         ks1 := RunSomeFakeKeepServers(st, 1)
431         ks2 := RunSomeFakeKeepServers(fh, 4)
432
433         for i, k := range ks1 {
434                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
435                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
436                 defer k.listener.Close()
437         }
438         for i, k := range ks2 {
439                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
440                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
441                 defer k.listener.Close()
442         }
443
444         kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
445
446         _, replicas, err := kc.PutB([]byte("foo"))
447
448         c.Check(err, Equals, InsufficientReplicasError)
449         c.Check(replicas, Equals, 1)
450         c.Check(<-st.handled, Equals, ks1[0].url)
451 }
452
453 type StubGetHandler struct {
454         c              *C
455         expectPath     string
456         expectApiToken string
457         httpStatus     int
458         body           []byte
459 }
460
461 func (sgh StubGetHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
462         sgh.c.Check(req.URL.Path, Equals, "/"+sgh.expectPath)
463         sgh.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sgh.expectApiToken))
464         resp.WriteHeader(sgh.httpStatus)
465         resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(sgh.body)))
466         resp.Write(sgh.body)
467 }
468
469 func (s *StandaloneSuite) TestGet(c *C) {
470         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
471
472         st := StubGetHandler{
473                 c,
474                 hash,
475                 "abc123",
476                 http.StatusOK,
477                 []byte("foo")}
478
479         ks := RunFakeKeepServer(st)
480         defer ks.listener.Close()
481
482         arv, err := arvadosclient.MakeArvadosClient()
483         kc, _ := MakeKeepClient(&arv)
484         arv.ApiToken = "abc123"
485         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
486
487         r, n, url2, err := kc.Get(hash)
488         defer r.Close()
489         c.Check(err, Equals, nil)
490         c.Check(n, Equals, int64(3))
491         c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks.url, hash))
492
493         content, err2 := ioutil.ReadAll(r)
494         c.Check(err2, Equals, nil)
495         c.Check(content, DeepEquals, []byte("foo"))
496 }
497
498 func (s *StandaloneSuite) TestGet404(c *C) {
499         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
500
501         st := Error404Handler{make(chan string, 1)}
502
503         ks := RunFakeKeepServer(st)
504         defer ks.listener.Close()
505
506         arv, err := arvadosclient.MakeArvadosClient()
507         kc, _ := MakeKeepClient(&arv)
508         arv.ApiToken = "abc123"
509         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
510
511         r, n, url2, err := kc.Get(hash)
512         c.Check(err, Equals, BlockNotFound)
513         c.Check(n, Equals, int64(0))
514         c.Check(url2, Equals, "")
515         c.Check(r, Equals, nil)
516 }
517
518 func (s *StandaloneSuite) TestGetFail(c *C) {
519         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
520
521         st := FailHandler{make(chan string, 1)}
522
523         ks := RunFakeKeepServer(st)
524         defer ks.listener.Close()
525
526         arv, err := arvadosclient.MakeArvadosClient()
527         kc, _ := MakeKeepClient(&arv)
528         arv.ApiToken = "abc123"
529         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
530         kc.Retries = 0
531
532         r, n, url2, err := kc.Get(hash)
533         errNotFound, _ := err.(*ErrNotFound)
534         c.Check(errNotFound, NotNil)
535         c.Check(strings.Contains(errNotFound.Error(), "HTTP 500"), Equals, true)
536         c.Check(errNotFound.Temporary(), Equals, true)
537         c.Check(n, Equals, int64(0))
538         c.Check(url2, Equals, "")
539         c.Check(r, Equals, nil)
540 }
541
542 func (s *StandaloneSuite) TestGetFailRetry(c *C) {
543         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
544
545         st := &FailThenSucceedHandler{make(chan string, 1), 0,
546                 StubGetHandler{
547                         c,
548                         hash,
549                         "abc123",
550                         http.StatusOK,
551                         []byte("foo")}}
552
553         ks := RunFakeKeepServer(st)
554         defer ks.listener.Close()
555
556         arv, err := arvadosclient.MakeArvadosClient()
557         kc, _ := MakeKeepClient(&arv)
558         arv.ApiToken = "abc123"
559         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
560
561         r, n, url2, err := kc.Get(hash)
562         defer r.Close()
563         c.Check(err, Equals, nil)
564         c.Check(n, Equals, int64(3))
565         c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks.url, hash))
566
567         content, err2 := ioutil.ReadAll(r)
568         c.Check(err2, Equals, nil)
569         c.Check(content, DeepEquals, []byte("foo"))
570 }
571
572 func (s *StandaloneSuite) TestGetNetError(c *C) {
573         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
574
575         arv, err := arvadosclient.MakeArvadosClient()
576         kc, _ := MakeKeepClient(&arv)
577         arv.ApiToken = "abc123"
578         kc.SetServiceRoots(map[string]string{"x": "http://localhost:62222"}, nil, nil)
579
580         r, n, url2, err := kc.Get(hash)
581         errNotFound, _ := err.(*ErrNotFound)
582         c.Check(errNotFound, NotNil)
583         c.Check(strings.Contains(errNotFound.Error(), "connection refused"), Equals, true)
584         c.Check(errNotFound.Temporary(), Equals, true)
585         c.Check(n, Equals, int64(0))
586         c.Check(url2, Equals, "")
587         c.Check(r, Equals, nil)
588 }
589
590 func (s *StandaloneSuite) TestGetWithServiceHint(c *C) {
591         uuid := "zzzzz-bi6l4-123451234512345"
592         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
593
594         // This one shouldn't be used:
595         ks0 := RunFakeKeepServer(StubGetHandler{
596                 c,
597                 "error if used",
598                 "abc123",
599                 http.StatusOK,
600                 []byte("foo")})
601         defer ks0.listener.Close()
602         // This one should be used:
603         ks := RunFakeKeepServer(StubGetHandler{
604                 c,
605                 hash + "+K@" + uuid,
606                 "abc123",
607                 http.StatusOK,
608                 []byte("foo")})
609         defer ks.listener.Close()
610
611         arv, err := arvadosclient.MakeArvadosClient()
612         kc, _ := MakeKeepClient(&arv)
613         arv.ApiToken = "abc123"
614         kc.SetServiceRoots(
615                 map[string]string{"x": ks0.url},
616                 nil,
617                 map[string]string{uuid: ks.url})
618
619         r, n, uri, err := kc.Get(hash + "+K@" + uuid)
620         defer r.Close()
621         c.Check(err, Equals, nil)
622         c.Check(n, Equals, int64(3))
623         c.Check(uri, Equals, fmt.Sprintf("%s/%s", ks.url, hash+"+K@"+uuid))
624
625         content, err := ioutil.ReadAll(r)
626         c.Check(err, Equals, nil)
627         c.Check(content, DeepEquals, []byte("foo"))
628 }
629
630 // Use a service hint to fetch from a local disk service, overriding
631 // rendezvous probe order.
632 func (s *StandaloneSuite) TestGetWithLocalServiceHint(c *C) {
633         uuid := "zzzzz-bi6l4-zzzzzzzzzzzzzzz"
634         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
635
636         // This one shouldn't be used, although it appears first in
637         // rendezvous probe order:
638         ks0 := RunFakeKeepServer(StubGetHandler{
639                 c,
640                 "error if used",
641                 "abc123",
642                 http.StatusOK,
643                 []byte("foo")})
644         defer ks0.listener.Close()
645         // This one should be used:
646         ks := RunFakeKeepServer(StubGetHandler{
647                 c,
648                 hash + "+K@" + uuid,
649                 "abc123",
650                 http.StatusOK,
651                 []byte("foo")})
652         defer ks.listener.Close()
653
654         arv, err := arvadosclient.MakeArvadosClient()
655         kc, _ := MakeKeepClient(&arv)
656         arv.ApiToken = "abc123"
657         kc.SetServiceRoots(
658                 map[string]string{
659                         "zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url,
660                         "zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url,
661                         "zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url,
662                         uuid: ks.url},
663                 nil,
664                 map[string]string{
665                         "zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url,
666                         "zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url,
667                         "zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url,
668                         uuid: ks.url},
669         )
670
671         r, n, uri, err := kc.Get(hash + "+K@" + uuid)
672         defer r.Close()
673         c.Check(err, Equals, nil)
674         c.Check(n, Equals, int64(3))
675         c.Check(uri, Equals, fmt.Sprintf("%s/%s", ks.url, hash+"+K@"+uuid))
676
677         content, err := ioutil.ReadAll(r)
678         c.Check(err, Equals, nil)
679         c.Check(content, DeepEquals, []byte("foo"))
680 }
681
682 func (s *StandaloneSuite) TestGetWithServiceHintFailoverToLocals(c *C) {
683         uuid := "zzzzz-bi6l4-123451234512345"
684         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
685
686         ksLocal := RunFakeKeepServer(StubGetHandler{
687                 c,
688                 hash + "+K@" + uuid,
689                 "abc123",
690                 http.StatusOK,
691                 []byte("foo")})
692         defer ksLocal.listener.Close()
693         ksGateway := RunFakeKeepServer(StubGetHandler{
694                 c,
695                 hash + "+K@" + uuid,
696                 "abc123",
697                 http.StatusInternalServerError,
698                 []byte("Error")})
699         defer ksGateway.listener.Close()
700
701         arv, err := arvadosclient.MakeArvadosClient()
702         kc, _ := MakeKeepClient(&arv)
703         arv.ApiToken = "abc123"
704         kc.SetServiceRoots(
705                 map[string]string{"zzzzz-bi6l4-keepdisk0000000": ksLocal.url},
706                 nil,
707                 map[string]string{uuid: ksGateway.url})
708
709         r, n, uri, err := kc.Get(hash + "+K@" + uuid)
710         c.Assert(err, Equals, nil)
711         defer r.Close()
712         c.Check(n, Equals, int64(3))
713         c.Check(uri, Equals, fmt.Sprintf("%s/%s", ksLocal.url, hash+"+K@"+uuid))
714
715         content, err := ioutil.ReadAll(r)
716         c.Check(err, Equals, nil)
717         c.Check(content, DeepEquals, []byte("foo"))
718 }
719
720 type BarHandler struct {
721         handled chan string
722 }
723
724 func (this BarHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
725         resp.Write([]byte("bar"))
726         this.handled <- fmt.Sprintf("http://%s", req.Host)
727 }
728
729 func (s *StandaloneSuite) TestChecksum(c *C) {
730         foohash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
731         barhash := fmt.Sprintf("%x", md5.Sum([]byte("bar")))
732
733         st := BarHandler{make(chan string, 1)}
734
735         ks := RunFakeKeepServer(st)
736         defer ks.listener.Close()
737
738         arv, err := arvadosclient.MakeArvadosClient()
739         kc, _ := MakeKeepClient(&arv)
740         arv.ApiToken = "abc123"
741         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
742
743         r, n, _, err := kc.Get(barhash)
744         _, err = ioutil.ReadAll(r)
745         c.Check(n, Equals, int64(3))
746         c.Check(err, Equals, nil)
747
748         <-st.handled
749
750         r, n, _, err = kc.Get(foohash)
751         _, err = ioutil.ReadAll(r)
752         c.Check(n, Equals, int64(3))
753         c.Check(err, Equals, BadChecksum)
754
755         <-st.handled
756 }
757
758 func (s *StandaloneSuite) TestGetWithFailures(c *C) {
759         content := []byte("waz")
760         hash := fmt.Sprintf("%x", md5.Sum(content))
761
762         fh := Error404Handler{
763                 make(chan string, 4)}
764
765         st := StubGetHandler{
766                 c,
767                 hash,
768                 "abc123",
769                 http.StatusOK,
770                 content}
771
772         arv, err := arvadosclient.MakeArvadosClient()
773         kc, _ := MakeKeepClient(&arv)
774         arv.ApiToken = "abc123"
775         localRoots := make(map[string]string)
776         writableLocalRoots := make(map[string]string)
777
778         ks1 := RunSomeFakeKeepServers(st, 1)
779         ks2 := RunSomeFakeKeepServers(fh, 4)
780
781         for i, k := range ks1 {
782                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
783                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
784                 defer k.listener.Close()
785         }
786         for i, k := range ks2 {
787                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
788                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
789                 defer k.listener.Close()
790         }
791
792         kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
793         kc.Retries = 0
794
795         // This test works only if one of the failing services is
796         // attempted before the succeeding service. Otherwise,
797         // <-fh.handled below will just hang! (Probe order depends on
798         // the choice of block content "waz" and the UUIDs of the fake
799         // servers, so we just tried different strings until we found
800         // an example that passes this Assert.)
801         c.Assert(NewRootSorter(localRoots, hash).GetSortedRoots()[0], Not(Equals), ks1[0].url)
802
803         r, n, url2, err := kc.Get(hash)
804
805         <-fh.handled
806         c.Check(err, Equals, nil)
807         c.Check(n, Equals, int64(3))
808         c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks1[0].url, hash))
809
810         read_content, err2 := ioutil.ReadAll(r)
811         c.Check(err2, Equals, nil)
812         c.Check(read_content, DeepEquals, content)
813 }
814
815 func (s *ServerRequiredSuite) TestPutGetHead(c *C) {
816         content := []byte("TestPutGetHead")
817
818         arv, err := arvadosclient.MakeArvadosClient()
819         kc, err := MakeKeepClient(&arv)
820         c.Assert(err, Equals, nil)
821
822         hash := fmt.Sprintf("%x", md5.Sum(content))
823
824         {
825                 n, _, err := kc.Ask(hash)
826                 c.Check(err, Equals, BlockNotFound)
827                 c.Check(n, Equals, int64(0))
828         }
829         {
830                 hash2, replicas, err := kc.PutB(content)
831                 c.Check(hash2, Matches, fmt.Sprintf(`%s\+%d\b.*`, hash, len(content)))
832                 c.Check(replicas, Equals, 2)
833                 c.Check(err, Equals, nil)
834         }
835         {
836                 r, n, url2, err := kc.Get(hash)
837                 c.Check(err, Equals, nil)
838                 c.Check(n, Equals, int64(len(content)))
839                 c.Check(url2, Matches, fmt.Sprintf("http://localhost:\\d+/%s", hash))
840
841                 read_content, err2 := ioutil.ReadAll(r)
842                 c.Check(err2, Equals, nil)
843                 c.Check(read_content, DeepEquals, content)
844         }
845         {
846                 n, url2, err := kc.Ask(hash)
847                 c.Check(err, Equals, nil)
848                 c.Check(n, Equals, int64(len(content)))
849                 c.Check(url2, Matches, fmt.Sprintf("http://localhost:\\d+/%s", hash))
850         }
851 }
852
853 type StubProxyHandler struct {
854         handled chan string
855 }
856
857 func (this StubProxyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
858         resp.Header().Set("X-Keep-Replicas-Stored", "2")
859         this.handled <- fmt.Sprintf("http://%s", req.Host)
860 }
861
862 func (s *StandaloneSuite) TestPutProxy(c *C) {
863         st := StubProxyHandler{make(chan string, 1)}
864
865         arv, err := arvadosclient.MakeArvadosClient()
866         kc, _ := MakeKeepClient(&arv)
867
868         kc.Want_replicas = 2
869         arv.ApiToken = "abc123"
870         localRoots := make(map[string]string)
871         writableLocalRoots := make(map[string]string)
872
873         ks1 := RunSomeFakeKeepServers(st, 1)
874
875         for i, k := range ks1 {
876                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
877                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
878                 defer k.listener.Close()
879         }
880
881         kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
882
883         _, replicas, err := kc.PutB([]byte("foo"))
884         <-st.handled
885
886         c.Check(err, Equals, nil)
887         c.Check(replicas, Equals, 2)
888 }
889
890 func (s *StandaloneSuite) TestPutProxyInsufficientReplicas(c *C) {
891         st := StubProxyHandler{make(chan string, 1)}
892
893         arv, err := arvadosclient.MakeArvadosClient()
894         kc, _ := MakeKeepClient(&arv)
895
896         kc.Want_replicas = 3
897         arv.ApiToken = "abc123"
898         localRoots := make(map[string]string)
899         writableLocalRoots := make(map[string]string)
900
901         ks1 := RunSomeFakeKeepServers(st, 1)
902
903         for i, k := range ks1 {
904                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
905                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
906                 defer k.listener.Close()
907         }
908         kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
909
910         _, replicas, err := kc.PutB([]byte("foo"))
911         <-st.handled
912
913         c.Check(err, Equals, InsufficientReplicasError)
914         c.Check(replicas, Equals, 2)
915 }
916
917 func (s *StandaloneSuite) TestMakeLocator(c *C) {
918         l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce+3+Aabcde@12345678")
919         c.Check(err, Equals, nil)
920         c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
921         c.Check(l.Size, Equals, 3)
922         c.Check(l.Hints, DeepEquals, []string{"3", "Aabcde@12345678"})
923 }
924
925 func (s *StandaloneSuite) TestMakeLocatorNoHints(c *C) {
926         l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce")
927         c.Check(err, Equals, nil)
928         c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
929         c.Check(l.Size, Equals, -1)
930         c.Check(l.Hints, DeepEquals, []string{})
931 }
932
933 func (s *StandaloneSuite) TestMakeLocatorNoSizeHint(c *C) {
934         l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce+Aabcde@12345678")
935         c.Check(err, Equals, nil)
936         c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
937         c.Check(l.Size, Equals, -1)
938         c.Check(l.Hints, DeepEquals, []string{"Aabcde@12345678"})
939 }
940
941 func (s *StandaloneSuite) TestMakeLocatorPreservesUnrecognizedHints(c *C) {
942         str := "91f372a266fe2bf2823cb8ec7fda31ce+3+Unknown+Kzzzzz+Afoobar"
943         l, err := MakeLocator(str)
944         c.Check(err, Equals, nil)
945         c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
946         c.Check(l.Size, Equals, 3)
947         c.Check(l.Hints, DeepEquals, []string{"3", "Unknown", "Kzzzzz", "Afoobar"})
948         c.Check(l.String(), Equals, str)
949 }
950
951 func (s *StandaloneSuite) TestMakeLocatorInvalidInput(c *C) {
952         _, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31c")
953         c.Check(err, Equals, InvalidLocatorError)
954 }
955
956 func (s *StandaloneSuite) TestPutBWant2ReplicasWithOnlyOneWritableLocalRoot(c *C) {
957         hash := Md5String("foo")
958
959         st := StubPutHandler{
960                 c,
961                 hash,
962                 "abc123",
963                 "foo",
964                 make(chan string, 5)}
965
966         arv, _ := arvadosclient.MakeArvadosClient()
967         kc, _ := MakeKeepClient(&arv)
968
969         kc.Want_replicas = 2
970         arv.ApiToken = "abc123"
971         localRoots := make(map[string]string)
972         writableLocalRoots := make(map[string]string)
973
974         ks := RunSomeFakeKeepServers(st, 5)
975
976         for i, k := range ks {
977                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
978                 if i == 0 {
979                         writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
980                 }
981                 defer k.listener.Close()
982         }
983
984         kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
985
986         _, replicas, err := kc.PutB([]byte("foo"))
987
988         c.Check(err, Equals, InsufficientReplicasError)
989         c.Check(replicas, Equals, 1)
990
991         c.Check(<-st.handled, Equals, localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", 0)])
992 }
993
994 func (s *StandaloneSuite) TestPutBWithNoWritableLocalRoots(c *C) {
995         hash := Md5String("foo")
996
997         st := StubPutHandler{
998                 c,
999                 hash,
1000                 "abc123",
1001                 "foo",
1002                 make(chan string, 5)}
1003
1004         arv, _ := arvadosclient.MakeArvadosClient()
1005         kc, _ := MakeKeepClient(&arv)
1006
1007         kc.Want_replicas = 2
1008         arv.ApiToken = "abc123"
1009         localRoots := make(map[string]string)
1010         writableLocalRoots := make(map[string]string)
1011
1012         ks := RunSomeFakeKeepServers(st, 5)
1013
1014         for i, k := range ks {
1015                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
1016                 defer k.listener.Close()
1017         }
1018
1019         kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
1020
1021         _, replicas, err := kc.PutB([]byte("foo"))
1022
1023         c.Check(err, Equals, InsufficientReplicasError)
1024         c.Check(replicas, Equals, 0)
1025 }
1026
1027 type StubGetIndexHandler struct {
1028         c              *C
1029         expectPath     string
1030         expectAPIToken string
1031         httpStatus     int
1032         body           []byte
1033 }
1034
1035 func (h StubGetIndexHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
1036         h.c.Check(req.URL.Path, Equals, h.expectPath)
1037         h.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", h.expectAPIToken))
1038         resp.WriteHeader(h.httpStatus)
1039         resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(h.body)))
1040         resp.Write(h.body)
1041 }
1042
1043 func (s *StandaloneSuite) TestGetIndexWithNoPrefix(c *C) {
1044         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
1045
1046         st := StubGetIndexHandler{
1047                 c,
1048                 "/index",
1049                 "abc123",
1050                 http.StatusOK,
1051                 []byte(hash + "+3 1443559274\n\n")}
1052
1053         ks := RunFakeKeepServer(st)
1054         defer ks.listener.Close()
1055
1056         arv, err := arvadosclient.MakeArvadosClient()
1057         kc, _ := MakeKeepClient(&arv)
1058         arv.ApiToken = "abc123"
1059         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
1060
1061         r, err := kc.GetIndex("x", "")
1062         c.Check(err, Equals, nil)
1063
1064         content, err2 := ioutil.ReadAll(r)
1065         c.Check(err2, Equals, nil)
1066         c.Check(content, DeepEquals, st.body[0:len(st.body)-1])
1067 }
1068
1069 func (s *StandaloneSuite) TestGetIndexWithPrefix(c *C) {
1070         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
1071
1072         st := StubGetIndexHandler{
1073                 c,
1074                 "/index/" + hash[0:3],
1075                 "abc123",
1076                 http.StatusOK,
1077                 []byte(hash + "+3 1443559274\n\n")}
1078
1079         ks := RunFakeKeepServer(st)
1080         defer ks.listener.Close()
1081
1082         arv, err := arvadosclient.MakeArvadosClient()
1083         kc, _ := MakeKeepClient(&arv)
1084         arv.ApiToken = "abc123"
1085         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
1086
1087         r, err := kc.GetIndex("x", hash[0:3])
1088         c.Check(err, Equals, nil)
1089
1090         content, err2 := ioutil.ReadAll(r)
1091         c.Check(err2, Equals, nil)
1092         c.Check(content, DeepEquals, st.body[0:len(st.body)-1])
1093 }
1094
1095 func (s *StandaloneSuite) TestGetIndexIncomplete(c *C) {
1096         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
1097
1098         st := StubGetIndexHandler{
1099                 c,
1100                 "/index/" + hash[0:3],
1101                 "abc123",
1102                 http.StatusOK,
1103                 []byte(hash)}
1104
1105         ks := RunFakeKeepServer(st)
1106         defer ks.listener.Close()
1107
1108         arv, err := arvadosclient.MakeArvadosClient()
1109         kc, _ := MakeKeepClient(&arv)
1110         arv.ApiToken = "abc123"
1111         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
1112
1113         _, err = kc.GetIndex("x", hash[0:3])
1114         c.Check(err, Equals, ErrIncompleteIndex)
1115 }
1116
1117 func (s *StandaloneSuite) TestGetIndexWithNoSuchServer(c *C) {
1118         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
1119
1120         st := StubGetIndexHandler{
1121                 c,
1122                 "/index/" + hash[0:3],
1123                 "abc123",
1124                 http.StatusOK,
1125                 []byte(hash)}
1126
1127         ks := RunFakeKeepServer(st)
1128         defer ks.listener.Close()
1129
1130         arv, err := arvadosclient.MakeArvadosClient()
1131         kc, _ := MakeKeepClient(&arv)
1132         arv.ApiToken = "abc123"
1133         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
1134
1135         _, err = kc.GetIndex("y", hash[0:3])
1136         c.Check(err, Equals, ErrNoSuchKeepServer)
1137 }
1138
1139 func (s *StandaloneSuite) TestGetIndexWithNoSuchPrefix(c *C) {
1140         st := StubGetIndexHandler{
1141                 c,
1142                 "/index/abcd",
1143                 "abc123",
1144                 http.StatusOK,
1145                 []byte("\n")}
1146
1147         ks := RunFakeKeepServer(st)
1148         defer ks.listener.Close()
1149
1150         arv, err := arvadosclient.MakeArvadosClient()
1151         kc, _ := MakeKeepClient(&arv)
1152         arv.ApiToken = "abc123"
1153         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
1154
1155         r, err := kc.GetIndex("x", "abcd")
1156         c.Check(err, Equals, nil)
1157
1158         content, err2 := ioutil.ReadAll(r)
1159         c.Check(err2, Equals, nil)
1160         c.Check(content, DeepEquals, st.body[0:len(st.body)-1])
1161 }
1162
1163 type FailThenSucceedPutHandler struct {
1164         handled        chan string
1165         count          int
1166         successhandler StubPutHandler
1167 }
1168
1169 func (h *FailThenSucceedPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
1170         if h.count == 0 {
1171                 resp.WriteHeader(500)
1172                 h.count += 1
1173                 h.handled <- fmt.Sprintf("http://%s", req.Host)
1174         } else {
1175                 h.successhandler.ServeHTTP(resp, req)
1176         }
1177 }
1178
1179 func (s *StandaloneSuite) TestPutBRetry(c *C) {
1180         st := &FailThenSucceedPutHandler{make(chan string, 1), 0,
1181                 StubPutHandler{
1182                         c,
1183                         Md5String("foo"),
1184                         "abc123",
1185                         "foo",
1186                         make(chan string, 5)}}
1187
1188         arv, _ := arvadosclient.MakeArvadosClient()
1189         kc, _ := MakeKeepClient(&arv)
1190
1191         kc.Want_replicas = 2
1192         arv.ApiToken = "abc123"
1193         localRoots := make(map[string]string)
1194         writableLocalRoots := make(map[string]string)
1195
1196         ks := RunSomeFakeKeepServers(st, 2)
1197
1198         for i, k := range ks {
1199                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
1200                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
1201                 defer k.listener.Close()
1202         }
1203
1204         kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
1205
1206         hash, replicas, err := kc.PutB([]byte("foo"))
1207
1208         c.Check(err, Equals, nil)
1209         c.Check(hash, Equals, "")
1210         c.Check(replicas, Equals, 2)
1211 }
1212
1213 func (s *ServerRequiredSuite) TestMakeKeepClientWithNonDiskTypeService(c *C) {
1214         arv, err := arvadosclient.MakeArvadosClient()
1215         c.Assert(err, Equals, nil)
1216
1217         // Add an additional "testblobstore" keepservice
1218         blobKeepService := make(arvadosclient.Dict)
1219         err = arv.Create("keep_services",
1220                 arvadosclient.Dict{"keep_service": arvadosclient.Dict{
1221                         "service_host": "localhost",
1222                         "service_port": "21321",
1223                         "service_type": "testblobstore"}},
1224                 &blobKeepService)
1225         c.Assert(err, Equals, nil)
1226         defer func() { arv.Delete("keep_services", blobKeepService["uuid"].(string), nil, nil) }()
1227
1228         // Make a keepclient and ensure that the testblobstore is included
1229         kc, err := MakeKeepClient(&arv)
1230         c.Assert(err, Equals, nil)
1231
1232         // verify kc.LocalRoots
1233         c.Check(len(kc.LocalRoots()), Equals, 3)
1234         for _, root := range kc.LocalRoots() {
1235                 c.Check(root, Matches, "http://localhost:\\d+")
1236         }
1237         c.Assert(kc.LocalRoots()[blobKeepService["uuid"].(string)], Not(Equals), "")
1238
1239         // verify kc.GatewayRoots
1240         c.Check(len(kc.GatewayRoots()), Equals, 3)
1241         for _, root := range kc.GatewayRoots() {
1242                 c.Check(root, Matches, "http://localhost:\\d+")
1243         }
1244         c.Assert(kc.GatewayRoots()[blobKeepService["uuid"].(string)], Not(Equals), "")
1245
1246         // verify kc.WritableLocalRoots
1247         c.Check(len(kc.WritableLocalRoots()), Equals, 3)
1248         for _, root := range kc.WritableLocalRoots() {
1249                 c.Check(root, Matches, "http://localhost:\\d+")
1250         }
1251         c.Assert(kc.WritableLocalRoots()[blobKeepService["uuid"].(string)], Not(Equals), "")
1252
1253         c.Assert(kc.replicasPerService, Equals, 0)
1254         c.Assert(kc.foundNonDiskSvc, Equals, true)
1255         c.Assert(kc.Client.Timeout, Equals, 300*time.Second)
1256 }