sdk/go/keepclient/keepclient_test.go

   1 package keepclient
   2
   3 import (
   4         "crypto/md5"
   5         "flag"
   6         "fmt"
   7         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
   8         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
   9         "git.curoverse.com/arvados.git/sdk/go/streamer"
  10         . "gopkg.in/check.v1"
  11         "io"
  12         "io/ioutil"
  13         "log"
  14         "net"
  15         "net/http"
  16         "os"
  17         "strings"
  18         "testing"
  19         "time"
  20 )
  21
  22 // Gocheck boilerplate
  23 func Test(t *testing.T) {
  24         TestingT(t)
  25 }
  26
  27 // Gocheck boilerplate
  28 var _ = Suite(&ServerRequiredSuite{})
  29 var _ = Suite(&StandaloneSuite{})
  30
  31 var no_server = flag.Bool("no-server", false, "Skip 'ServerRequireSuite'")
  32
  33 // Tests that require the Keep server running
  34 type ServerRequiredSuite struct{}
  35
  36 // Standalone tests
  37 type StandaloneSuite struct{}
  38
  39 func pythonDir() string {
  40         cwd, _ := os.Getwd()
  41         return fmt.Sprintf("%s/../../python/tests", cwd)
  42 }
  43
  44 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
  45         if *no_server {
  46                 c.Skip("Skipping tests that require server")
  47                 return
  48         }
  49         arvadostest.StartAPI()
  50         arvadostest.StartKeep(2, false)
  51 }
  52
  53 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
  54         if *no_server {
  55                 return
  56         }
  57         arvadostest.StopKeep(2)
  58         arvadostest.StopAPI()
  59 }
  60
  61 func (s *ServerRequiredSuite) TestMakeKeepClient(c *C) {
  62         arv, err := arvadosclient.MakeArvadosClient()
  63         c.Assert(err, Equals, nil)
  64
  65         kc, err := MakeKeepClient(&arv)
  66
  67         c.Assert(err, Equals, nil)
  68         c.Check(len(kc.LocalRoots()), Equals, 2)
  69         for _, root := range kc.LocalRoots() {
  70                 c.Check(root, Matches, "http://localhost:\\d+")
  71         }
  72 }
  73
  74 func (s *ServerRequiredSuite) TestDefaultReplications(c *C) {
  75         arv, err := arvadosclient.MakeArvadosClient()
  76         c.Assert(err, Equals, nil)
  77
  78         kc, err := MakeKeepClient(&arv)
  79         c.Assert(kc.Want_replicas, Equals, 2)
  80
  81         arv.DiscoveryDoc["defaultCollectionReplication"] = 3.0
  82         kc, err = MakeKeepClient(&arv)
  83         c.Assert(kc.Want_replicas, Equals, 3)
  84
  85         arv.DiscoveryDoc["defaultCollectionReplication"] = 1.0
  86         kc, err = MakeKeepClient(&arv)
  87         c.Assert(kc.Want_replicas, Equals, 1)
  88 }
  89
  90 type StubPutHandler struct {
  91         c              *C
  92         expectPath     string
  93         expectApiToken string
  94         expectBody     string
  95         handled        chan string
  96 }
  97
  98 func (sph StubPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
  99         sph.c.Check(req.URL.Path, Equals, "/"+sph.expectPath)
 100         sph.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sph.expectApiToken))
 101         body, err := ioutil.ReadAll(req.Body)
 102         sph.c.Check(err, Equals, nil)
 103         sph.c.Check(body, DeepEquals, []byte(sph.expectBody))
 104         resp.WriteHeader(200)
 105         sph.handled <- fmt.Sprintf("http://%s", req.Host)
 106 }
 107
 108 func RunFakeKeepServer(st http.Handler) (ks KeepServer) {
 109         var err error
 110         ks.listener, err = net.ListenTCP("tcp", &net.TCPAddr{Port: 0})
 111         if err != nil {
 112                 panic(fmt.Sprintf("Could not listen on any port"))
 113         }
 114         ks.url = fmt.Sprintf("http://%s", ks.listener.Addr().String())
 115         go http.Serve(ks.listener, st)
 116         return
 117 }
 118
 119 func UploadToStubHelper(c *C, st http.Handler, f func(*KeepClient, string,
 120         io.ReadCloser, io.WriteCloser, chan uploadStatus)) {
 121
 122         ks := RunFakeKeepServer(st)
 123         defer ks.listener.Close()
 124
 125         arv, _ := arvadosclient.MakeArvadosClient()
 126         arv.ApiToken = "abc123"
 127
 128         kc, _ := MakeKeepClient(&arv)
 129
 130         reader, writer := io.Pipe()
 131         upload_status := make(chan uploadStatus)
 132
 133         f(kc, ks.url, reader, writer, upload_status)
 134 }
 135
 136 func (s *StandaloneSuite) TestUploadToStubKeepServer(c *C) {
 137         log.Printf("TestUploadToStubKeepServer")
 138
 139         st := StubPutHandler{
 140                 c,
 141                 "acbd18db4cc2f85cedef654fccc4a4d8",
 142                 "abc123",
 143                 "foo",
 144                 make(chan string)}
 145
 146         UploadToStubHelper(c, st,
 147                 func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, upload_status chan uploadStatus) {
 148
 149                         go kc.uploadToKeepServer(url, st.expectPath, reader, upload_status, int64(len("foo")), 0)
 150
 151                         writer.Write([]byte("foo"))
 152                         writer.Close()
 153
 154                         <-st.handled
 155                         status := <-upload_status
 156                         c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, ""})
 157                 })
 158
 159         log.Printf("TestUploadToStubKeepServer done")
 160 }
 161
 162 func (s *StandaloneSuite) TestUploadToStubKeepServerBufferReader(c *C) {
 163         log.Printf("TestUploadToStubKeepServerBufferReader")
 164
 165         st := StubPutHandler{
 166                 c,
 167                 "acbd18db4cc2f85cedef654fccc4a4d8",
 168                 "abc123",
 169                 "foo",
 170                 make(chan string)}
 171
 172         UploadToStubHelper(c, st,
 173                 func(kc *KeepClient, url string, reader io.ReadCloser,
 174                         writer io.WriteCloser, upload_status chan uploadStatus) {
 175
 176                         tr := streamer.AsyncStreamFromReader(512, reader)
 177                         defer tr.Close()
 178
 179                         br1 := tr.MakeStreamReader()
 180
 181                         go kc.uploadToKeepServer(url, st.expectPath, br1, upload_status, 3, 0)
 182
 183                         writer.Write([]byte("foo"))
 184                         writer.Close()
 185
 186                         <-st.handled
 187
 188                         status := <-upload_status
 189                         c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, ""})
 190                 })
 191
 192         log.Printf("TestUploadToStubKeepServerBufferReader done")
 193 }
 194
 195 type FailHandler struct {
 196         handled chan string
 197 }
 198
 199 func (fh FailHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 200         resp.WriteHeader(500)
 201         fh.handled <- fmt.Sprintf("http://%s", req.Host)
 202 }
 203
 204 type FailThenSucceedHandler struct {
 205         handled        chan string
 206         count          int
 207         successhandler StubGetHandler
 208 }
 209
 210 func (fh *FailThenSucceedHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 211         if fh.count == 0 {
 212                 resp.WriteHeader(500)
 213                 fh.count += 1
 214                 fh.handled <- fmt.Sprintf("http://%s", req.Host)
 215         } else {
 216                 fh.successhandler.ServeHTTP(resp, req)
 217         }
 218 }
 219
 220 type Error404Handler struct {
 221         handled chan string
 222 }
 223
 224 func (fh Error404Handler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 225         resp.WriteHeader(404)
 226         fh.handled <- fmt.Sprintf("http://%s", req.Host)
 227 }
 228
 229 func (s *StandaloneSuite) TestFailedUploadToStubKeepServer(c *C) {
 230         log.Printf("TestFailedUploadToStubKeepServer")
 231
 232         st := FailHandler{
 233                 make(chan string)}
 234
 235         hash := "acbd18db4cc2f85cedef654fccc4a4d8"
 236
 237         UploadToStubHelper(c, st,
 238                 func(kc *KeepClient, url string, reader io.ReadCloser,
 239                         writer io.WriteCloser, upload_status chan uploadStatus) {
 240
 241                         go kc.uploadToKeepServer(url, hash, reader, upload_status, 3, 0)
 242
 243                         writer.Write([]byte("foo"))
 244                         writer.Close()
 245
 246                         <-st.handled
 247
 248                         status := <-upload_status
 249                         c.Check(status.url, Equals, fmt.Sprintf("%s/%s", url, hash))
 250                         c.Check(status.statusCode, Equals, 500)
 251                 })
 252         log.Printf("TestFailedUploadToStubKeepServer done")
 253 }
 254
 255 type KeepServer struct {
 256         listener net.Listener
 257         url      string
 258 }
 259
 260 func RunSomeFakeKeepServers(st http.Handler, n int) (ks []KeepServer) {
 261         ks = make([]KeepServer, n)
 262
 263         for i := 0; i < n; i += 1 {
 264                 ks[i] = RunFakeKeepServer(st)
 265         }
 266
 267         return ks
 268 }
 269
 270 func (s *StandaloneSuite) TestPutB(c *C) {
 271         log.Printf("TestPutB")
 272
 273         hash := Md5String("foo")
 274
 275         st := StubPutHandler{
 276                 c,
 277                 hash,
 278                 "abc123",
 279                 "foo",
 280                 make(chan string, 5)}
 281
 282         arv, _ := arvadosclient.MakeArvadosClient()
 283         kc, _ := MakeKeepClient(&arv)
 284
 285         kc.Want_replicas = 2
 286         arv.ApiToken = "abc123"
 287         localRoots := make(map[string]string)
 288         writableLocalRoots := make(map[string]string)
 289
 290         ks := RunSomeFakeKeepServers(st, 5)
 291
 292         for i, k := range ks {
 293                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 294                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 295                 defer k.listener.Close()
 296         }
 297
 298         kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
 299
 300         kc.PutB([]byte("foo"))
 301
 302         shuff := NewRootSorter(
 303                 kc.LocalRoots(), Md5String("foo")).GetSortedRoots()
 304
 305         s1 := <-st.handled
 306         s2 := <-st.handled
 307         c.Check((s1 == shuff[0] && s2 == shuff[1]) ||
 308                 (s1 == shuff[1] && s2 == shuff[0]),
 309                 Equals,
 310                 true)
 311
 312         log.Printf("TestPutB done")
 313 }
 314
 315 func (s *StandaloneSuite) TestPutHR(c *C) {
 316         log.Printf("TestPutHR")
 317
 318         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 319
 320         st := StubPutHandler{
 321                 c,
 322                 hash,
 323                 "abc123",
 324                 "foo",
 325                 make(chan string, 5)}
 326
 327         arv, _ := arvadosclient.MakeArvadosClient()
 328         kc, _ := MakeKeepClient(&arv)
 329
 330         kc.Want_replicas = 2
 331         arv.ApiToken = "abc123"
 332         localRoots := make(map[string]string)
 333         writableLocalRoots := make(map[string]string)
 334
 335         ks := RunSomeFakeKeepServers(st, 5)
 336
 337         for i, k := range ks {
 338                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 339                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 340                 defer k.listener.Close()
 341         }
 342
 343         kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
 344
 345         reader, writer := io.Pipe()
 346
 347         go func() {
 348                 writer.Write([]byte("foo"))
 349                 writer.Close()
 350         }()
 351
 352         kc.PutHR(hash, reader, 3)
 353
 354         shuff := NewRootSorter(kc.LocalRoots(), hash).GetSortedRoots()
 355         log.Print(shuff)
 356
 357         s1 := <-st.handled
 358         s2 := <-st.handled
 359
 360         c.Check((s1 == shuff[0] && s2 == shuff[1]) ||
 361                 (s1 == shuff[1] && s2 == shuff[0]),
 362                 Equals,
 363                 true)
 364
 365         log.Printf("TestPutHR done")
 366 }
 367
 368 func (s *StandaloneSuite) TestPutWithFail(c *C) {
 369         log.Printf("TestPutWithFail")
 370
 371         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 372
 373         st := StubPutHandler{
 374                 c,
 375                 hash,
 376                 "abc123",
 377                 "foo",
 378                 make(chan string, 4)}
 379
 380         fh := FailHandler{
 381                 make(chan string, 1)}
 382
 383         arv, err := arvadosclient.MakeArvadosClient()
 384         kc, _ := MakeKeepClient(&arv)
 385
 386         kc.Want_replicas = 2
 387         arv.ApiToken = "abc123"
 388         localRoots := make(map[string]string)
 389         writableLocalRoots := make(map[string]string)
 390
 391         ks1 := RunSomeFakeKeepServers(st, 4)
 392         ks2 := RunSomeFakeKeepServers(fh, 1)
 393
 394         for i, k := range ks1 {
 395                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 396                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 397                 defer k.listener.Close()
 398         }
 399         for i, k := range ks2 {
 400                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
 401                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
 402                 defer k.listener.Close()
 403         }
 404
 405         kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
 406
 407         shuff := NewRootSorter(
 408                 kc.LocalRoots(), Md5String("foo")).GetSortedRoots()
 409
 410         phash, replicas, err := kc.PutB([]byte("foo"))
 411
 412         <-fh.handled
 413
 414         c.Check(err, Equals, nil)
 415         c.Check(phash, Equals, "")
 416         c.Check(replicas, Equals, 2)
 417
 418         s1 := <-st.handled
 419         s2 := <-st.handled
 420
 421         c.Check((s1 == shuff[1] && s2 == shuff[2]) ||
 422                 (s1 == shuff[2] && s2 == shuff[1]),
 423                 Equals,
 424                 true)
 425 }
 426
 427 func (s *StandaloneSuite) TestPutWithTooManyFail(c *C) {
 428         log.Printf("TestPutWithTooManyFail")
 429
 430         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 431
 432         st := StubPutHandler{
 433                 c,
 434                 hash,
 435                 "abc123",
 436                 "foo",
 437                 make(chan string, 1)}
 438
 439         fh := FailHandler{
 440                 make(chan string, 4)}
 441
 442         arv, err := arvadosclient.MakeArvadosClient()
 443         kc, _ := MakeKeepClient(&arv)
 444
 445         kc.Want_replicas = 2
 446         kc.Retries = 0
 447         arv.ApiToken = "abc123"
 448         localRoots := make(map[string]string)
 449         writableLocalRoots := make(map[string]string)
 450
 451         ks1 := RunSomeFakeKeepServers(st, 1)
 452         ks2 := RunSomeFakeKeepServers(fh, 4)
 453
 454         for i, k := range ks1 {
 455                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 456                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 457                 defer k.listener.Close()
 458         }
 459         for i, k := range ks2 {
 460                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
 461                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
 462                 defer k.listener.Close()
 463         }
 464
 465         kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
 466
 467         _, replicas, err := kc.PutB([]byte("foo"))
 468
 469         c.Check(err, Equals, InsufficientReplicasError)
 470         c.Check(replicas, Equals, 1)
 471         c.Check(<-st.handled, Equals, ks1[0].url)
 472
 473         log.Printf("TestPutWithTooManyFail done")
 474 }
 475
 476 type StubGetHandler struct {
 477         c              *C
 478         expectPath     string
 479         expectApiToken string
 480         httpStatus     int
 481         body           []byte
 482 }
 483
 484 func (sgh StubGetHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 485         sgh.c.Check(req.URL.Path, Equals, "/"+sgh.expectPath)
 486         sgh.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sgh.expectApiToken))
 487         resp.WriteHeader(sgh.httpStatus)
 488         resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(sgh.body)))
 489         resp.Write(sgh.body)
 490 }
 491
 492 func (s *StandaloneSuite) TestGet(c *C) {
 493         log.Printf("TestGet")
 494
 495         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 496
 497         st := StubGetHandler{
 498                 c,
 499                 hash,
 500                 "abc123",
 501                 http.StatusOK,
 502                 []byte("foo")}
 503
 504         ks := RunFakeKeepServer(st)
 505         defer ks.listener.Close()
 506
 507         arv, err := arvadosclient.MakeArvadosClient()
 508         kc, _ := MakeKeepClient(&arv)
 509         arv.ApiToken = "abc123"
 510         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
 511
 512         r, n, url2, err := kc.Get(hash)
 513         defer r.Close()
 514         c.Check(err, Equals, nil)
 515         c.Check(n, Equals, int64(3))
 516         c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks.url, hash))
 517
 518         content, err2 := ioutil.ReadAll(r)
 519         c.Check(err2, Equals, nil)
 520         c.Check(content, DeepEquals, []byte("foo"))
 521
 522         log.Printf("TestGet done")
 523 }
 524
 525 func (s *StandaloneSuite) TestGet404(c *C) {
 526         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 527
 528         st := Error404Handler{make(chan string, 1)}
 529
 530         ks := RunFakeKeepServer(st)
 531         defer ks.listener.Close()
 532
 533         arv, err := arvadosclient.MakeArvadosClient()
 534         kc, _ := MakeKeepClient(&arv)
 535         arv.ApiToken = "abc123"
 536         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
 537
 538         r, n, url2, err := kc.Get(hash)
 539         c.Check(err, Equals, BlockNotFound)
 540         c.Check(n, Equals, int64(0))
 541         c.Check(url2, Equals, "")
 542         c.Check(r, Equals, nil)
 543 }
 544
 545 func (s *StandaloneSuite) TestGetFail(c *C) {
 546         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 547
 548         st := FailHandler{make(chan string, 1)}
 549
 550         ks := RunFakeKeepServer(st)
 551         defer ks.listener.Close()
 552
 553         arv, err := arvadosclient.MakeArvadosClient()
 554         kc, _ := MakeKeepClient(&arv)
 555         arv.ApiToken = "abc123"
 556         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
 557         kc.Retries = 0
 558
 559         r, n, url2, err := kc.Get(hash)
 560         errNotFound, _ := err.(*ErrNotFound)
 561         c.Check(errNotFound, NotNil)
 562         c.Check(strings.Contains(errNotFound.Error(), "HTTP 500"), Equals, true)
 563         c.Check(errNotFound.Temporary(), Equals, true)
 564         c.Check(n, Equals, int64(0))
 565         c.Check(url2, Equals, "")
 566         c.Check(r, Equals, nil)
 567 }
 568
 569 func (s *StandaloneSuite) TestGetFailRetry(c *C) {
 570         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 571
 572         st := &FailThenSucceedHandler{make(chan string, 1), 0,
 573                 StubGetHandler{
 574                         c,
 575                         hash,
 576                         "abc123",
 577                         http.StatusOK,
 578                         []byte("foo")}}
 579
 580         ks := RunFakeKeepServer(st)
 581         defer ks.listener.Close()
 582
 583         arv, err := arvadosclient.MakeArvadosClient()
 584         kc, _ := MakeKeepClient(&arv)
 585         arv.ApiToken = "abc123"
 586         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
 587
 588         r, n, url2, err := kc.Get(hash)
 589         defer r.Close()
 590         c.Check(err, Equals, nil)
 591         c.Check(n, Equals, int64(3))
 592         c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks.url, hash))
 593
 594         content, err2 := ioutil.ReadAll(r)
 595         c.Check(err2, Equals, nil)
 596         c.Check(content, DeepEquals, []byte("foo"))
 597 }
 598
 599 func (s *StandaloneSuite) TestGetNetError(c *C) {
 600         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 601
 602         arv, err := arvadosclient.MakeArvadosClient()
 603         kc, _ := MakeKeepClient(&arv)
 604         arv.ApiToken = "abc123"
 605         kc.SetServiceRoots(map[string]string{"x": "http://localhost:62222"}, nil, nil)
 606
 607         r, n, url2, err := kc.Get(hash)
 608         errNotFound, _ := err.(*ErrNotFound)
 609         c.Check(errNotFound, NotNil)
 610         c.Check(strings.Contains(errNotFound.Error(), "connection refused"), Equals, true)
 611         c.Check(errNotFound.Temporary(), Equals, true)
 612         c.Check(n, Equals, int64(0))
 613         c.Check(url2, Equals, "")
 614         c.Check(r, Equals, nil)
 615 }
 616
 617 func (s *StandaloneSuite) TestGetWithServiceHint(c *C) {
 618         uuid := "zzzzz-bi6l4-123451234512345"
 619         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 620
 621         // This one shouldn't be used:
 622         ks0 := RunFakeKeepServer(StubGetHandler{
 623                 c,
 624                 "error if used",
 625                 "abc123",
 626                 http.StatusOK,
 627                 []byte("foo")})
 628         defer ks0.listener.Close()
 629         // This one should be used:
 630         ks := RunFakeKeepServer(StubGetHandler{
 631                 c,
 632                 hash + "+K@" + uuid,
 633                 "abc123",
 634                 http.StatusOK,
 635                 []byte("foo")})
 636         defer ks.listener.Close()
 637
 638         arv, err := arvadosclient.MakeArvadosClient()
 639         kc, _ := MakeKeepClient(&arv)
 640         arv.ApiToken = "abc123"
 641         kc.SetServiceRoots(
 642                 map[string]string{"x": ks0.url},
 643                 nil,
 644                 map[string]string{uuid: ks.url})
 645
 646         r, n, uri, err := kc.Get(hash + "+K@" + uuid)
 647         defer r.Close()
 648         c.Check(err, Equals, nil)
 649         c.Check(n, Equals, int64(3))
 650         c.Check(uri, Equals, fmt.Sprintf("%s/%s", ks.url, hash+"+K@"+uuid))
 651
 652         content, err := ioutil.ReadAll(r)
 653         c.Check(err, Equals, nil)
 654         c.Check(content, DeepEquals, []byte("foo"))
 655 }
 656
 657 // Use a service hint to fetch from a local disk service, overriding
 658 // rendezvous probe order.
 659 func (s *StandaloneSuite) TestGetWithLocalServiceHint(c *C) {
 660         uuid := "zzzzz-bi6l4-zzzzzzzzzzzzzzz"
 661         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 662
 663         // This one shouldn't be used, although it appears first in
 664         // rendezvous probe order:
 665         ks0 := RunFakeKeepServer(StubGetHandler{
 666                 c,
 667                 "error if used",
 668                 "abc123",
 669                 http.StatusOK,
 670                 []byte("foo")})
 671         defer ks0.listener.Close()
 672         // This one should be used:
 673         ks := RunFakeKeepServer(StubGetHandler{
 674                 c,
 675                 hash + "+K@" + uuid,
 676                 "abc123",
 677                 http.StatusOK,
 678                 []byte("foo")})
 679         defer ks.listener.Close()
 680
 681         arv, err := arvadosclient.MakeArvadosClient()
 682         kc, _ := MakeKeepClient(&arv)
 683         arv.ApiToken = "abc123"
 684         kc.SetServiceRoots(
 685                 map[string]string{
 686                         "zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url,
 687                         "zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url,
 688                         "zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url,
 689                         uuid: ks.url},
 690                 nil,
 691                 map[string]string{
 692                         "zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url,
 693                         "zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url,
 694                         "zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url,
 695                         uuid: ks.url},
 696         )
 697
 698         r, n, uri, err := kc.Get(hash + "+K@" + uuid)
 699         defer r.Close()
 700         c.Check(err, Equals, nil)
 701         c.Check(n, Equals, int64(3))
 702         c.Check(uri, Equals, fmt.Sprintf("%s/%s", ks.url, hash+"+K@"+uuid))
 703
 704         content, err := ioutil.ReadAll(r)
 705         c.Check(err, Equals, nil)
 706         c.Check(content, DeepEquals, []byte("foo"))
 707 }
 708
 709 func (s *StandaloneSuite) TestGetWithServiceHintFailoverToLocals(c *C) {
 710         uuid := "zzzzz-bi6l4-123451234512345"
 711         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 712
 713         ksLocal := RunFakeKeepServer(StubGetHandler{
 714                 c,
 715                 hash + "+K@" + uuid,
 716                 "abc123",
 717                 http.StatusOK,
 718                 []byte("foo")})
 719         defer ksLocal.listener.Close()
 720         ksGateway := RunFakeKeepServer(StubGetHandler{
 721                 c,
 722                 hash + "+K@" + uuid,
 723                 "abc123",
 724                 http.StatusInternalServerError,
 725                 []byte("Error")})
 726         defer ksGateway.listener.Close()
 727
 728         arv, err := arvadosclient.MakeArvadosClient()
 729         kc, _ := MakeKeepClient(&arv)
 730         arv.ApiToken = "abc123"
 731         kc.SetServiceRoots(
 732                 map[string]string{"zzzzz-bi6l4-keepdisk0000000": ksLocal.url},
 733                 nil,
 734                 map[string]string{uuid: ksGateway.url})
 735
 736         r, n, uri, err := kc.Get(hash + "+K@" + uuid)
 737         c.Assert(err, Equals, nil)
 738         defer r.Close()
 739         c.Check(n, Equals, int64(3))
 740         c.Check(uri, Equals, fmt.Sprintf("%s/%s", ksLocal.url, hash+"+K@"+uuid))
 741
 742         content, err := ioutil.ReadAll(r)
 743         c.Check(err, Equals, nil)
 744         c.Check(content, DeepEquals, []byte("foo"))
 745 }
 746
 747 type BarHandler struct {
 748         handled chan string
 749 }
 750
 751 func (this BarHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 752         resp.Write([]byte("bar"))
 753         this.handled <- fmt.Sprintf("http://%s", req.Host)
 754 }
 755
 756 func (s *StandaloneSuite) TestChecksum(c *C) {
 757         foohash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 758         barhash := fmt.Sprintf("%x", md5.Sum([]byte("bar")))
 759
 760         st := BarHandler{make(chan string, 1)}
 761
 762         ks := RunFakeKeepServer(st)
 763         defer ks.listener.Close()
 764
 765         arv, err := arvadosclient.MakeArvadosClient()
 766         kc, _ := MakeKeepClient(&arv)
 767         arv.ApiToken = "abc123"
 768         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
 769
 770         r, n, _, err := kc.Get(barhash)
 771         _, err = ioutil.ReadAll(r)
 772         c.Check(n, Equals, int64(3))
 773         c.Check(err, Equals, nil)
 774
 775         <-st.handled
 776
 777         r, n, _, err = kc.Get(foohash)
 778         _, err = ioutil.ReadAll(r)
 779         c.Check(n, Equals, int64(3))
 780         c.Check(err, Equals, BadChecksum)
 781
 782         <-st.handled
 783 }
 784
 785 func (s *StandaloneSuite) TestGetWithFailures(c *C) {
 786         content := []byte("waz")
 787         hash := fmt.Sprintf("%x", md5.Sum(content))
 788
 789         fh := Error404Handler{
 790                 make(chan string, 4)}
 791
 792         st := StubGetHandler{
 793                 c,
 794                 hash,
 795                 "abc123",
 796                 http.StatusOK,
 797                 content}
 798
 799         arv, err := arvadosclient.MakeArvadosClient()
 800         kc, _ := MakeKeepClient(&arv)
 801         arv.ApiToken = "abc123"
 802         localRoots := make(map[string]string)
 803         writableLocalRoots := make(map[string]string)
 804
 805         ks1 := RunSomeFakeKeepServers(st, 1)
 806         ks2 := RunSomeFakeKeepServers(fh, 4)
 807
 808         for i, k := range ks1 {
 809                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 810                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 811                 defer k.listener.Close()
 812         }
 813         for i, k := range ks2 {
 814                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
 815                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
 816                 defer k.listener.Close()
 817         }
 818
 819         kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
 820         kc.Retries = 0
 821
 822         // This test works only if one of the failing services is
 823         // attempted before the succeeding service. Otherwise,
 824         // <-fh.handled below will just hang! (Probe order depends on
 825         // the choice of block content "waz" and the UUIDs of the fake
 826         // servers, so we just tried different strings until we found
 827         // an example that passes this Assert.)
 828         c.Assert(NewRootSorter(localRoots, hash).GetSortedRoots()[0], Not(Equals), ks1[0].url)
 829
 830         r, n, url2, err := kc.Get(hash)
 831
 832         <-fh.handled
 833         c.Check(err, Equals, nil)
 834         c.Check(n, Equals, int64(3))
 835         c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks1[0].url, hash))
 836
 837         read_content, err2 := ioutil.ReadAll(r)
 838         c.Check(err2, Equals, nil)
 839         c.Check(read_content, DeepEquals, content)
 840 }
 841
 842 func (s *ServerRequiredSuite) TestPutGetHead(c *C) {
 843         content := []byte("TestPutGetHead")
 844
 845         arv, err := arvadosclient.MakeArvadosClient()
 846         kc, err := MakeKeepClient(&arv)
 847         c.Assert(err, Equals, nil)
 848
 849         hash := fmt.Sprintf("%x", md5.Sum(content))
 850
 851         {
 852                 n, _, err := kc.Ask(hash)
 853                 c.Check(err, Equals, BlockNotFound)
 854                 c.Check(n, Equals, int64(0))
 855         }
 856         {
 857                 hash2, replicas, err := kc.PutB(content)
 858                 c.Check(hash2, Matches, fmt.Sprintf(`%s\+%d\b.*`, hash, len(content)))
 859                 c.Check(replicas, Equals, 2)
 860                 c.Check(err, Equals, nil)
 861         }
 862         {
 863                 r, n, url2, err := kc.Get(hash)
 864                 c.Check(err, Equals, nil)
 865                 c.Check(n, Equals, int64(len(content)))
 866                 c.Check(url2, Matches, fmt.Sprintf("http://localhost:\\d+/%s", hash))
 867
 868                 read_content, err2 := ioutil.ReadAll(r)
 869                 c.Check(err2, Equals, nil)
 870                 c.Check(read_content, DeepEquals, content)
 871         }
 872         {
 873                 n, url2, err := kc.Ask(hash)
 874                 c.Check(err, Equals, nil)
 875                 c.Check(n, Equals, int64(len(content)))
 876                 c.Check(url2, Matches, fmt.Sprintf("http://localhost:\\d+/%s", hash))
 877         }
 878 }
 879
 880 type StubProxyHandler struct {
 881         handled chan string
 882 }
 883
 884 func (this StubProxyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 885         resp.Header().Set("X-Keep-Replicas-Stored", "2")
 886         this.handled <- fmt.Sprintf("http://%s", req.Host)
 887 }
 888
 889 func (s *StandaloneSuite) TestPutProxy(c *C) {
 890         log.Printf("TestPutProxy")
 891
 892         st := StubProxyHandler{make(chan string, 1)}
 893
 894         arv, err := arvadosclient.MakeArvadosClient()
 895         kc, _ := MakeKeepClient(&arv)
 896
 897         kc.Want_replicas = 2
 898         arv.ApiToken = "abc123"
 899         localRoots := make(map[string]string)
 900         writableLocalRoots := make(map[string]string)
 901
 902         ks1 := RunSomeFakeKeepServers(st, 1)
 903
 904         for i, k := range ks1 {
 905                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 906                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 907                 defer k.listener.Close()
 908         }
 909
 910         kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
 911
 912         _, replicas, err := kc.PutB([]byte("foo"))
 913         <-st.handled
 914
 915         c.Check(err, Equals, nil)
 916         c.Check(replicas, Equals, 2)
 917
 918         log.Printf("TestPutProxy done")
 919 }
 920
 921 func (s *StandaloneSuite) TestPutProxyInsufficientReplicas(c *C) {
 922         log.Printf("TestPutProxy")
 923
 924         st := StubProxyHandler{make(chan string, 1)}
 925
 926         arv, err := arvadosclient.MakeArvadosClient()
 927         kc, _ := MakeKeepClient(&arv)
 928
 929         kc.Want_replicas = 3
 930         arv.ApiToken = "abc123"
 931         localRoots := make(map[string]string)
 932         writableLocalRoots := make(map[string]string)
 933
 934         ks1 := RunSomeFakeKeepServers(st, 1)
 935
 936         for i, k := range ks1 {
 937                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 938                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 939                 defer k.listener.Close()
 940         }
 941         kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
 942
 943         _, replicas, err := kc.PutB([]byte("foo"))
 944         <-st.handled
 945
 946         c.Check(err, Equals, InsufficientReplicasError)
 947         c.Check(replicas, Equals, 2)
 948
 949         log.Printf("TestPutProxy done")
 950 }
 951
 952 func (s *StandaloneSuite) TestMakeLocator(c *C) {
 953         l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce+3+Aabcde@12345678")
 954         c.Check(err, Equals, nil)
 955         c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
 956         c.Check(l.Size, Equals, 3)
 957         c.Check(l.Hints, DeepEquals, []string{"3", "Aabcde@12345678"})
 958 }
 959
 960 func (s *StandaloneSuite) TestMakeLocatorNoHints(c *C) {
 961         l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce")
 962         c.Check(err, Equals, nil)
 963         c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
 964         c.Check(l.Size, Equals, -1)
 965         c.Check(l.Hints, DeepEquals, []string{})
 966 }
 967
 968 func (s *StandaloneSuite) TestMakeLocatorNoSizeHint(c *C) {
 969         l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce+Aabcde@12345678")
 970         c.Check(err, Equals, nil)
 971         c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
 972         c.Check(l.Size, Equals, -1)
 973         c.Check(l.Hints, DeepEquals, []string{"Aabcde@12345678"})
 974 }
 975
 976 func (s *StandaloneSuite) TestMakeLocatorPreservesUnrecognizedHints(c *C) {
 977         str := "91f372a266fe2bf2823cb8ec7fda31ce+3+Unknown+Kzzzzz+Afoobar"
 978         l, err := MakeLocator(str)
 979         c.Check(err, Equals, nil)
 980         c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
 981         c.Check(l.Size, Equals, 3)
 982         c.Check(l.Hints, DeepEquals, []string{"3", "Unknown", "Kzzzzz", "Afoobar"})
 983         c.Check(l.String(), Equals, str)
 984 }
 985
 986 func (s *StandaloneSuite) TestMakeLocatorInvalidInput(c *C) {
 987         _, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31c")
 988         c.Check(err, Equals, InvalidLocatorError)
 989 }
 990
 991 func (s *StandaloneSuite) TestPutBWant2ReplicasWithOnlyOneWritableLocalRoot(c *C) {
 992         hash := Md5String("foo")
 993
 994         st := StubPutHandler{
 995                 c,
 996                 hash,
 997                 "abc123",
 998                 "foo",
 999                 make(chan string, 5)}
1000
1001         arv, _ := arvadosclient.MakeArvadosClient()
1002         kc, _ := MakeKeepClient(&arv)
1003
1004         kc.Want_replicas = 2
1005         arv.ApiToken = "abc123"
1006         localRoots := make(map[string]string)
1007         writableLocalRoots := make(map[string]string)
1008
1009         ks := RunSomeFakeKeepServers(st, 5)
1010
1011         for i, k := range ks {
1012                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
1013                 if i == 0 {
1014                         writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
1015                 }
1016                 defer k.listener.Close()
1017         }
1018
1019         kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
1020
1021         _, replicas, err := kc.PutB([]byte("foo"))
1022
1023         c.Check(err, Equals, InsufficientReplicasError)
1024         c.Check(replicas, Equals, 1)
1025
1026         c.Check(<-st.handled, Equals, localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", 0)])
1027 }
1028
1029 func (s *StandaloneSuite) TestPutBWithNoWritableLocalRoots(c *C) {
1030         hash := Md5String("foo")
1031
1032         st := StubPutHandler{
1033                 c,
1034                 hash,
1035                 "abc123",
1036                 "foo",
1037                 make(chan string, 5)}
1038
1039         arv, _ := arvadosclient.MakeArvadosClient()
1040         kc, _ := MakeKeepClient(&arv)
1041
1042         kc.Want_replicas = 2
1043         arv.ApiToken = "abc123"
1044         localRoots := make(map[string]string)
1045         writableLocalRoots := make(map[string]string)
1046
1047         ks := RunSomeFakeKeepServers(st, 5)
1048
1049         for i, k := range ks {
1050                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
1051                 defer k.listener.Close()
1052         }
1053
1054         kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
1055
1056         _, replicas, err := kc.PutB([]byte("foo"))
1057
1058         c.Check(err, Equals, InsufficientReplicasError)
1059         c.Check(replicas, Equals, 0)
1060 }
1061
1062 type StubGetIndexHandler struct {
1063         c              *C
1064         expectPath     string
1065         expectAPIToken string
1066         httpStatus     int
1067         body           []byte
1068 }
1069
1070 func (h StubGetIndexHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
1071         h.c.Check(req.URL.Path, Equals, h.expectPath)
1072         h.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", h.expectAPIToken))
1073         resp.WriteHeader(h.httpStatus)
1074         resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(h.body)))
1075         resp.Write(h.body)
1076 }
1077
1078 func (s *StandaloneSuite) TestGetIndexWithNoPrefix(c *C) {
1079         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
1080
1081         st := StubGetIndexHandler{
1082                 c,
1083                 "/index",
1084                 "abc123",
1085                 http.StatusOK,
1086                 []byte(hash + "+3 1443559274\n\n")}
1087
1088         ks := RunFakeKeepServer(st)
1089         defer ks.listener.Close()
1090
1091         arv, err := arvadosclient.MakeArvadosClient()
1092         kc, _ := MakeKeepClient(&arv)
1093         arv.ApiToken = "abc123"
1094         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
1095
1096         r, err := kc.GetIndex("x", "")
1097         c.Check(err, Equals, nil)
1098
1099         content, err2 := ioutil.ReadAll(r)
1100         c.Check(err2, Equals, nil)
1101         c.Check(content, DeepEquals, st.body[0:len(st.body)-1])
1102 }
1103
1104 func (s *StandaloneSuite) TestGetIndexWithPrefix(c *C) {
1105         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
1106
1107         st := StubGetIndexHandler{
1108                 c,
1109                 "/index/" + hash[0:3],
1110                 "abc123",
1111                 http.StatusOK,
1112                 []byte(hash + "+3 1443559274\n\n")}
1113
1114         ks := RunFakeKeepServer(st)
1115         defer ks.listener.Close()
1116
1117         arv, err := arvadosclient.MakeArvadosClient()
1118         kc, _ := MakeKeepClient(&arv)
1119         arv.ApiToken = "abc123"
1120         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
1121
1122         r, err := kc.GetIndex("x", hash[0:3])
1123         c.Check(err, Equals, nil)
1124
1125         content, err2 := ioutil.ReadAll(r)
1126         c.Check(err2, Equals, nil)
1127         c.Check(content, DeepEquals, st.body[0:len(st.body)-1])
1128 }
1129
1130 func (s *StandaloneSuite) TestGetIndexIncomplete(c *C) {
1131         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
1132
1133         st := StubGetIndexHandler{
1134                 c,
1135                 "/index/" + hash[0:3],
1136                 "abc123",
1137                 http.StatusOK,
1138                 []byte(hash)}
1139
1140         ks := RunFakeKeepServer(st)
1141         defer ks.listener.Close()
1142
1143         arv, err := arvadosclient.MakeArvadosClient()
1144         kc, _ := MakeKeepClient(&arv)
1145         arv.ApiToken = "abc123"
1146         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
1147
1148         _, err = kc.GetIndex("x", hash[0:3])
1149         c.Check(err, Equals, ErrIncompleteIndex)
1150 }
1151
1152 func (s *StandaloneSuite) TestGetIndexWithNoSuchServer(c *C) {
1153         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
1154
1155         st := StubGetIndexHandler{
1156                 c,
1157                 "/index/" + hash[0:3],
1158                 "abc123",
1159                 http.StatusOK,
1160                 []byte(hash)}
1161
1162         ks := RunFakeKeepServer(st)
1163         defer ks.listener.Close()
1164
1165         arv, err := arvadosclient.MakeArvadosClient()
1166         kc, _ := MakeKeepClient(&arv)
1167         arv.ApiToken = "abc123"
1168         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
1169
1170         _, err = kc.GetIndex("y", hash[0:3])
1171         c.Check(err, Equals, ErrNoSuchKeepServer)
1172 }
1173
1174 func (s *StandaloneSuite) TestGetIndexWithNoSuchPrefix(c *C) {
1175         st := StubGetIndexHandler{
1176                 c,
1177                 "/index/abcd",
1178                 "abc123",
1179                 http.StatusOK,
1180                 []byte("\n")}
1181
1182         ks := RunFakeKeepServer(st)
1183         defer ks.listener.Close()
1184
1185         arv, err := arvadosclient.MakeArvadosClient()
1186         kc, _ := MakeKeepClient(&arv)
1187         arv.ApiToken = "abc123"
1188         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
1189
1190         r, err := kc.GetIndex("x", "abcd")
1191         c.Check(err, Equals, nil)
1192
1193         content, err2 := ioutil.ReadAll(r)
1194         c.Check(err2, Equals, nil)
1195         c.Check(content, DeepEquals, st.body[0:len(st.body)-1])
1196 }
1197
1198 type FailThenSucceedPutHandler struct {
1199         handled        chan string
1200         count          int
1201         successhandler StubPutHandler
1202 }
1203
1204 func (h *FailThenSucceedPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
1205         if h.count == 0 {
1206                 resp.WriteHeader(500)
1207                 h.count += 1
1208                 h.handled <- fmt.Sprintf("http://%s", req.Host)
1209         } else {
1210                 h.successhandler.ServeHTTP(resp, req)
1211         }
1212 }
1213
1214 func (s *StandaloneSuite) TestPutBRetry(c *C) {
1215         st := &FailThenSucceedPutHandler{make(chan string, 1), 0,
1216                 StubPutHandler{
1217                         c,
1218                         Md5String("foo"),
1219                         "abc123",
1220                         "foo",
1221                         make(chan string, 5)}}
1222
1223         arv, _ := arvadosclient.MakeArvadosClient()
1224         kc, _ := MakeKeepClient(&arv)
1225
1226         kc.Want_replicas = 2
1227         arv.ApiToken = "abc123"
1228         localRoots := make(map[string]string)
1229         writableLocalRoots := make(map[string]string)
1230
1231         ks := RunSomeFakeKeepServers(st, 2)
1232
1233         for i, k := range ks {
1234                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
1235                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
1236                 defer k.listener.Close()
1237         }
1238
1239         kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
1240
1241         hash, replicas, err := kc.PutB([]byte("foo"))
1242
1243         c.Check(err, Equals, nil)
1244         c.Check(hash, Equals, "")
1245         c.Check(replicas, Equals, 2)
1246 }
1247
1248 func (s *ServerRequiredSuite) TestMakeKeepClientWithNonDiskTypeService(c *C) {
1249         arv, err := arvadosclient.MakeArvadosClient()
1250         c.Assert(err, Equals, nil)
1251
1252         // Add an additional "testblobstore" keepservice
1253         blobKeepService := make(arvadosclient.Dict)
1254         err = arv.Create("keep_services",
1255                 arvadosclient.Dict{"keep_service": arvadosclient.Dict{
1256                         "service_host": "localhost",
1257                         "service_port": "21321",
1258                         "service_type": "testblobstore"}},
1259                 &blobKeepService)
1260         c.Assert(err, Equals, nil)
1261         defer func() { arv.Delete("keep_services", blobKeepService["uuid"].(string), nil, nil) }()
1262
1263         // Make a keepclient and ensure that the testblobstore is included
1264         kc, err := MakeKeepClient(&arv)
1265         c.Assert(err, Equals, nil)
1266
1267         // verify kc.LocalRoots
1268         c.Check(len(kc.LocalRoots()), Equals, 3)
1269         for _, root := range kc.LocalRoots() {
1270                 c.Check(root, Matches, "http://localhost:\\d+")
1271         }
1272         c.Assert(kc.LocalRoots()[blobKeepService["uuid"].(string)], Not(Equals), "")
1273
1274         // verify kc.GatewayRoots
1275         c.Check(len(kc.GatewayRoots()), Equals, 3)
1276         for _, root := range kc.GatewayRoots() {
1277                 c.Check(root, Matches, "http://localhost:\\d+")
1278         }
1279         c.Assert(kc.GatewayRoots()[blobKeepService["uuid"].(string)], Not(Equals), "")
1280
1281         // verify kc.WritableLocalRoots
1282         c.Check(len(kc.WritableLocalRoots()), Equals, 3)
1283         for _, root := range kc.WritableLocalRoots() {
1284                 c.Check(root, Matches, "http://localhost:\\d+")
1285         }
1286         c.Assert(kc.WritableLocalRoots()[blobKeepService["uuid"].(string)], Not(Equals), "")
1287
1288         c.Assert(kc.replicasPerService, Equals, 0)
1289         c.Assert(kc.foundNonDiskSvc, Equals, true)
1290         c.Assert(kc.Client.Timeout, Equals, 300*time.Second)
1291 }