sdk/go/keepclient/keepclient_test.go

   1 package keepclient
   2
   3 import (
   4         "crypto/md5"
   5         "flag"
   6         "fmt"
   7         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
   8         "git.curoverse.com/arvados.git/sdk/go/streamer"
   9         . "gopkg.in/check.v1"
  10         "io"
  11         "io/ioutil"
  12         "log"
  13         "net"
  14         "net/http"
  15         "os"
  16         "os/exec"
  17         "testing"
  18 )
  19
  20 // Gocheck boilerplate
  21 func Test(t *testing.T) {
  22         TestingT(t)
  23 }
  24
  25 // Gocheck boilerplate
  26 var _ = Suite(&ServerRequiredSuite{})
  27 var _ = Suite(&StandaloneSuite{})
  28
  29 var no_server = flag.Bool("no-server", false, "Skip 'ServerRequireSuite'")
  30
  31 // Tests that require the Keep server running
  32 type ServerRequiredSuite struct{}
  33
  34 // Standalone tests
  35 type StandaloneSuite struct{}
  36
  37 func pythonDir() string {
  38         cwd, _ := os.Getwd()
  39         return fmt.Sprintf("%s/../../python/tests", cwd)
  40 }
  41
  42 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
  43         if *no_server {
  44                 c.Skip("Skipping tests that require server")
  45                 return
  46         }
  47         os.Chdir(pythonDir())
  48         {
  49                 cmd := exec.Command("python", "run_test_server.py", "start")
  50                 stderr, err := cmd.StderrPipe()
  51                 if err != nil {
  52                         log.Fatalf("Setting up stderr pipe: %s", err)
  53                 }
  54                 go io.Copy(os.Stderr, stderr)
  55                 if err := cmd.Run(); err != nil {
  56                         panic(fmt.Sprintf("'python run_test_server.py start' returned error %s", err))
  57                 }
  58         }
  59         {
  60                 cmd := exec.Command("python", "run_test_server.py", "start_keep")
  61                 stderr, err := cmd.StderrPipe()
  62                 if err != nil {
  63                         log.Fatalf("Setting up stderr pipe: %s", err)
  64                 }
  65                 go io.Copy(os.Stderr, stderr)
  66                 if err := cmd.Run(); err != nil {
  67                         panic(fmt.Sprintf("'python run_test_server.py start_keep' returned error %s", err))
  68                 }
  69         }
  70 }
  71
  72 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
  73         os.Chdir(pythonDir())
  74         exec.Command("python", "run_test_server.py", "stop_keep").Run()
  75         exec.Command("python", "run_test_server.py", "stop").Run()
  76 }
  77
  78 func (s *ServerRequiredSuite) TestMakeKeepClient(c *C) {
  79         os.Setenv("ARVADOS_API_HOST", "localhost:3000")
  80         os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
  81         os.Setenv("ARVADOS_API_HOST_INSECURE", "true")
  82
  83         arv, err := arvadosclient.MakeArvadosClient()
  84         c.Assert(err, Equals, nil)
  85
  86         kc, err := MakeKeepClient(&arv)
  87
  88         c.Assert(err, Equals, nil)
  89         c.Check(len(kc.ServiceRoots()), Equals, 2)
  90         for _, root := range kc.ServiceRoots() {
  91                 c.Check(root, Matches, "http://localhost:2510[\\d]")
  92         }
  93 }
  94
  95 type StubPutHandler struct {
  96         c              *C
  97         expectPath     string
  98         expectApiToken string
  99         expectBody     string
 100         handled        chan string
 101 }
 102
 103 func (this StubPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 104         this.c.Check(req.URL.Path, Equals, "/"+this.expectPath)
 105         this.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", this.expectApiToken))
 106         body, err := ioutil.ReadAll(req.Body)
 107         this.c.Check(err, Equals, nil)
 108         this.c.Check(body, DeepEquals, []byte(this.expectBody))
 109         resp.WriteHeader(200)
 110         this.handled <- fmt.Sprintf("http://%s", req.Host)
 111 }
 112
 113 func RunFakeKeepServer(st http.Handler) (ks KeepServer) {
 114         var err error
 115         ks.listener, err = net.ListenTCP("tcp", &net.TCPAddr{Port: 0})
 116         if err != nil {
 117                 panic(fmt.Sprintf("Could not listen on any port"))
 118         }
 119         ks.url = fmt.Sprintf("http://%s", ks.listener.Addr().String())
 120         go http.Serve(ks.listener, st)
 121         return
 122 }
 123
 124 func UploadToStubHelper(c *C, st http.Handler, f func(KeepClient, string,
 125         io.ReadCloser, io.WriteCloser, chan uploadStatus)) {
 126
 127         ks := RunFakeKeepServer(st)
 128         defer ks.listener.Close()
 129
 130         arv, _ := arvadosclient.MakeArvadosClient()
 131         arv.ApiToken = "abc123"
 132
 133         kc, _ := MakeKeepClient(&arv)
 134
 135         reader, writer := io.Pipe()
 136         upload_status := make(chan uploadStatus)
 137
 138         f(kc, ks.url, reader, writer, upload_status)
 139 }
 140
 141 func (s *StandaloneSuite) TestUploadToStubKeepServer(c *C) {
 142         log.Printf("TestUploadToStubKeepServer")
 143
 144         st := StubPutHandler{
 145                 c,
 146                 "acbd18db4cc2f85cedef654fccc4a4d8",
 147                 "abc123",
 148                 "foo",
 149                 make(chan string)}
 150
 151         UploadToStubHelper(c, st,
 152                 func(kc KeepClient, url string, reader io.ReadCloser,
 153                         writer io.WriteCloser, upload_status chan uploadStatus) {
 154
 155                         go kc.uploadToKeepServer(url, st.expectPath, reader, upload_status, int64(len("foo")), "TestUploadToStubKeepServer")
 156
 157                         writer.Write([]byte("foo"))
 158                         writer.Close()
 159
 160                         <-st.handled
 161                         status := <-upload_status
 162                         c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, ""})
 163                 })
 164
 165         log.Printf("TestUploadToStubKeepServer done")
 166 }
 167
 168 func (s *StandaloneSuite) TestUploadToStubKeepServerBufferReader(c *C) {
 169         log.Printf("TestUploadToStubKeepServerBufferReader")
 170
 171         st := StubPutHandler{
 172                 c,
 173                 "acbd18db4cc2f85cedef654fccc4a4d8",
 174                 "abc123",
 175                 "foo",
 176                 make(chan string)}
 177
 178         UploadToStubHelper(c, st,
 179                 func(kc KeepClient, url string, reader io.ReadCloser,
 180                         writer io.WriteCloser, upload_status chan uploadStatus) {
 181
 182                         tr := streamer.AsyncStreamFromReader(512, reader)
 183                         defer tr.Close()
 184
 185                         br1 := tr.MakeStreamReader()
 186
 187                         go kc.uploadToKeepServer(url, st.expectPath, br1, upload_status, 3, "TestUploadToStubKeepServerBufferReader")
 188
 189                         writer.Write([]byte("foo"))
 190                         writer.Close()
 191
 192                         <-st.handled
 193
 194                         status := <-upload_status
 195                         c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, ""})
 196                 })
 197
 198         log.Printf("TestUploadToStubKeepServerBufferReader done")
 199 }
 200
 201 type FailHandler struct {
 202         handled chan string
 203 }
 204
 205 func (this FailHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 206         resp.WriteHeader(500)
 207         this.handled <- fmt.Sprintf("http://%s", req.Host)
 208 }
 209
 210 func (s *StandaloneSuite) TestFailedUploadToStubKeepServer(c *C) {
 211         log.Printf("TestFailedUploadToStubKeepServer")
 212
 213         st := FailHandler{
 214                 make(chan string)}
 215
 216         hash := "acbd18db4cc2f85cedef654fccc4a4d8"
 217
 218         UploadToStubHelper(c, st,
 219                 func(kc KeepClient, url string, reader io.ReadCloser,
 220                         writer io.WriteCloser, upload_status chan uploadStatus) {
 221
 222                         go kc.uploadToKeepServer(url, hash, reader, upload_status, 3, "TestFailedUploadToStubKeepServer")
 223
 224                         writer.Write([]byte("foo"))
 225                         writer.Close()
 226
 227                         <-st.handled
 228
 229                         status := <-upload_status
 230                         c.Check(status.url, Equals, fmt.Sprintf("%s/%s", url, hash))
 231                         c.Check(status.statusCode, Equals, 500)
 232                 })
 233         log.Printf("TestFailedUploadToStubKeepServer done")
 234 }
 235
 236 type KeepServer struct {
 237         listener net.Listener
 238         url      string
 239 }
 240
 241 func RunSomeFakeKeepServers(st http.Handler, n int) (ks []KeepServer) {
 242         ks = make([]KeepServer, n)
 243
 244         for i := 0; i < n; i += 1 {
 245                 ks[i] = RunFakeKeepServer(st)
 246         }
 247
 248         return ks
 249 }
 250
 251 func (s *StandaloneSuite) TestPutB(c *C) {
 252         log.Printf("TestPutB")
 253
 254         hash := Md5String("foo")
 255
 256         st := StubPutHandler{
 257                 c,
 258                 hash,
 259                 "abc123",
 260                 "foo",
 261                 make(chan string, 5)}
 262
 263         arv, _ := arvadosclient.MakeArvadosClient()
 264         kc, _ := MakeKeepClient(&arv)
 265
 266         kc.Want_replicas = 2
 267         arv.ApiToken = "abc123"
 268         service_roots := make(map[string]string)
 269
 270         ks := RunSomeFakeKeepServers(st, 5)
 271
 272         for i, k := range ks {
 273                 service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 274                 defer k.listener.Close()
 275         }
 276
 277         kc.SetServiceRoots(service_roots)
 278
 279         kc.PutB([]byte("foo"))
 280
 281         shuff := NewRootSorter(
 282                 kc.ServiceRoots(), Md5String("foo")).GetSortedRoots()
 283
 284         s1 := <-st.handled
 285         s2 := <-st.handled
 286         c.Check((s1 == shuff[0] && s2 == shuff[1]) ||
 287                 (s1 == shuff[1] && s2 == shuff[0]),
 288                 Equals,
 289                 true)
 290
 291         log.Printf("TestPutB done")
 292 }
 293
 294 func (s *StandaloneSuite) TestPutHR(c *C) {
 295         log.Printf("TestPutHR")
 296
 297         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 298
 299         st := StubPutHandler{
 300                 c,
 301                 hash,
 302                 "abc123",
 303                 "foo",
 304                 make(chan string, 5)}
 305
 306         arv, _ := arvadosclient.MakeArvadosClient()
 307         kc, _ := MakeKeepClient(&arv)
 308
 309         kc.Want_replicas = 2
 310         arv.ApiToken = "abc123"
 311         service_roots := make(map[string]string)
 312
 313         ks := RunSomeFakeKeepServers(st, 5)
 314
 315         for i, k := range ks {
 316                 service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 317                 defer k.listener.Close()
 318         }
 319
 320         kc.SetServiceRoots(service_roots)
 321
 322         reader, writer := io.Pipe()
 323
 324         go func() {
 325                 writer.Write([]byte("foo"))
 326                 writer.Close()
 327         }()
 328
 329         kc.PutHR(hash, reader, 3)
 330
 331         shuff := NewRootSorter(kc.ServiceRoots(), hash).GetSortedRoots()
 332         log.Print(shuff)
 333
 334         s1 := <-st.handled
 335         s2 := <-st.handled
 336
 337         c.Check((s1 == shuff[0] && s2 == shuff[1]) ||
 338                 (s1 == shuff[1] && s2 == shuff[0]),
 339                 Equals,
 340                 true)
 341
 342         log.Printf("TestPutHR done")
 343 }
 344
 345 func (s *StandaloneSuite) TestPutWithFail(c *C) {
 346         log.Printf("TestPutWithFail")
 347
 348         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 349
 350         st := StubPutHandler{
 351                 c,
 352                 hash,
 353                 "abc123",
 354                 "foo",
 355                 make(chan string, 4)}
 356
 357         fh := FailHandler{
 358                 make(chan string, 1)}
 359
 360         arv, err := arvadosclient.MakeArvadosClient()
 361         kc, _ := MakeKeepClient(&arv)
 362
 363         kc.Want_replicas = 2
 364         arv.ApiToken = "abc123"
 365         service_roots := make(map[string]string)
 366
 367         ks1 := RunSomeFakeKeepServers(st, 4)
 368         ks2 := RunSomeFakeKeepServers(fh, 1)
 369
 370         for i, k := range ks1 {
 371                 service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 372                 defer k.listener.Close()
 373         }
 374         for i, k := range ks2 {
 375                 service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
 376                 defer k.listener.Close()
 377         }
 378
 379         kc.SetServiceRoots(service_roots)
 380
 381         shuff := NewRootSorter(
 382                 kc.ServiceRoots(), Md5String("foo")).GetSortedRoots()
 383
 384         phash, replicas, err := kc.PutB([]byte("foo"))
 385
 386         <-fh.handled
 387
 388         c.Check(err, Equals, nil)
 389         c.Check(phash, Equals, "")
 390         c.Check(replicas, Equals, 2)
 391
 392         s1 := <-st.handled
 393         s2 := <-st.handled
 394
 395         c.Check((s1 == shuff[1] && s2 == shuff[2]) ||
 396                 (s1 == shuff[2] && s2 == shuff[1]),
 397                 Equals,
 398                 true)
 399 }
 400
 401 func (s *StandaloneSuite) TestPutWithTooManyFail(c *C) {
 402         log.Printf("TestPutWithTooManyFail")
 403
 404         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 405
 406         st := StubPutHandler{
 407                 c,
 408                 hash,
 409                 "abc123",
 410                 "foo",
 411                 make(chan string, 1)}
 412
 413         fh := FailHandler{
 414                 make(chan string, 4)}
 415
 416         arv, err := arvadosclient.MakeArvadosClient()
 417         kc, _ := MakeKeepClient(&arv)
 418
 419         kc.Want_replicas = 2
 420         arv.ApiToken = "abc123"
 421         service_roots := make(map[string]string)
 422
 423         ks1 := RunSomeFakeKeepServers(st, 1)
 424         ks2 := RunSomeFakeKeepServers(fh, 4)
 425
 426         for i, k := range ks1 {
 427                 service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 428                 defer k.listener.Close()
 429         }
 430         for i, k := range ks2 {
 431                 service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
 432                 defer k.listener.Close()
 433         }
 434
 435         kc.SetServiceRoots(service_roots)
 436
 437         _, replicas, err := kc.PutB([]byte("foo"))
 438
 439         c.Check(err, Equals, InsufficientReplicasError)
 440         c.Check(replicas, Equals, 1)
 441         c.Check(<-st.handled, Equals, ks1[0].url)
 442
 443         log.Printf("TestPutWithTooManyFail done")
 444 }
 445
 446 type StubGetHandler struct {
 447         c              *C
 448         expectPath     string
 449         expectApiToken string
 450         returnBody     []byte
 451 }
 452
 453 func (this StubGetHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 454         this.c.Check(req.URL.Path, Equals, "/"+this.expectPath)
 455         this.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", this.expectApiToken))
 456         resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(this.returnBody)))
 457         resp.Write(this.returnBody)
 458 }
 459
 460 func (s *StandaloneSuite) TestGet(c *C) {
 461         log.Printf("TestGet")
 462
 463         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 464
 465         st := StubGetHandler{
 466                 c,
 467                 hash,
 468                 "abc123",
 469                 []byte("foo")}
 470
 471         ks := RunFakeKeepServer(st)
 472         defer ks.listener.Close()
 473
 474         arv, err := arvadosclient.MakeArvadosClient()
 475         kc, _ := MakeKeepClient(&arv)
 476         arv.ApiToken = "abc123"
 477         kc.SetServiceRoots(map[string]string{"x": ks.url})
 478
 479         r, n, url2, err := kc.Get(hash)
 480         defer r.Close()
 481         c.Check(err, Equals, nil)
 482         c.Check(n, Equals, int64(3))
 483         c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks.url, hash))
 484
 485         content, err2 := ioutil.ReadAll(r)
 486         c.Check(err2, Equals, nil)
 487         c.Check(content, DeepEquals, []byte("foo"))
 488
 489         log.Printf("TestGet done")
 490 }
 491
 492 func (s *StandaloneSuite) TestGetFail(c *C) {
 493         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 494
 495         st := FailHandler{make(chan string, 1)}
 496
 497         ks := RunFakeKeepServer(st)
 498         defer ks.listener.Close()
 499
 500         arv, err := arvadosclient.MakeArvadosClient()
 501         kc, _ := MakeKeepClient(&arv)
 502         arv.ApiToken = "abc123"
 503         kc.SetServiceRoots(map[string]string{"x": ks.url})
 504
 505         r, n, url2, err := kc.Get(hash)
 506         c.Check(err, Equals, BlockNotFound)
 507         c.Check(n, Equals, int64(0))
 508         c.Check(url2, Equals, "")
 509         c.Check(r, Equals, nil)
 510 }
 511
 512 type BarHandler struct {
 513         handled chan string
 514 }
 515
 516 func (this BarHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 517         resp.Write([]byte("bar"))
 518         this.handled <- fmt.Sprintf("http://%s", req.Host)
 519 }
 520
 521 func (s *StandaloneSuite) TestChecksum(c *C) {
 522         foohash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 523         barhash := fmt.Sprintf("%x", md5.Sum([]byte("bar")))
 524
 525         st := BarHandler{make(chan string, 1)}
 526
 527         ks := RunFakeKeepServer(st)
 528         defer ks.listener.Close()
 529
 530         arv, err := arvadosclient.MakeArvadosClient()
 531         kc, _ := MakeKeepClient(&arv)
 532         arv.ApiToken = "abc123"
 533         kc.SetServiceRoots(map[string]string{"x": ks.url})
 534
 535         r, n, _, err := kc.Get(barhash)
 536         _, err = ioutil.ReadAll(r)
 537         c.Check(n, Equals, int64(3))
 538         c.Check(err, Equals, nil)
 539
 540         <-st.handled
 541
 542         r, n, _, err = kc.Get(foohash)
 543         _, err = ioutil.ReadAll(r)
 544         c.Check(n, Equals, int64(3))
 545         c.Check(err, Equals, BadChecksum)
 546
 547         <-st.handled
 548 }
 549
 550 func (s *StandaloneSuite) TestGetWithFailures(c *C) {
 551         content := []byte("waz")
 552         hash := fmt.Sprintf("%x", md5.Sum(content))
 553
 554         fh := FailHandler{
 555                 make(chan string, 4)}
 556
 557         st := StubGetHandler{
 558                 c,
 559                 hash,
 560                 "abc123",
 561                 content}
 562
 563         arv, err := arvadosclient.MakeArvadosClient()
 564         kc, _ := MakeKeepClient(&arv)
 565         arv.ApiToken = "abc123"
 566         service_roots := make(map[string]string)
 567
 568         ks1 := RunSomeFakeKeepServers(st, 1)
 569         ks2 := RunSomeFakeKeepServers(fh, 4)
 570
 571         for i, k := range ks1 {
 572                 service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 573                 defer k.listener.Close()
 574         }
 575         for i, k := range ks2 {
 576                 service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
 577                 defer k.listener.Close()
 578         }
 579
 580         kc.SetServiceRoots(service_roots)
 581
 582         // This test works only if one of the failing services is
 583         // attempted before the succeeding service. Otherwise,
 584         // <-fh.handled below will just hang! (Probe order depends on
 585         // the choice of block content "waz" and the UUIDs of the fake
 586         // servers, so we just tried different strings until we found
 587         // an example that passes this Assert.)
 588         c.Assert(NewRootSorter(service_roots, hash).GetSortedRoots()[0], Not(Equals), ks1[0].url)
 589
 590         r, n, url2, err := kc.Get(hash)
 591
 592         <-fh.handled
 593         c.Check(err, Equals, nil)
 594         c.Check(n, Equals, int64(3))
 595         c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks1[0].url, hash))
 596
 597         read_content, err2 := ioutil.ReadAll(r)
 598         c.Check(err2, Equals, nil)
 599         c.Check(read_content, DeepEquals, content)
 600 }
 601
 602 func (s *ServerRequiredSuite) TestPutGetHead(c *C) {
 603         os.Setenv("ARVADOS_API_HOST", "localhost:3000")
 604         os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
 605         os.Setenv("ARVADOS_API_HOST_INSECURE", "true")
 606         content := []byte("TestPutGetHead")
 607
 608         arv, err := arvadosclient.MakeArvadosClient()
 609         kc, err := MakeKeepClient(&arv)
 610         c.Assert(err, Equals, nil)
 611
 612         hash := fmt.Sprintf("%x", md5.Sum(content))
 613
 614         {
 615                 n, _, err := kc.Ask(hash)
 616                 c.Check(err, Equals, BlockNotFound)
 617                 c.Check(n, Equals, int64(0))
 618         }
 619         {
 620                 hash2, replicas, err := kc.PutB(content)
 621                 c.Check(hash2, Equals, fmt.Sprintf("%s+%d", hash, len(content)))
 622                 c.Check(replicas, Equals, 2)
 623                 c.Check(err, Equals, nil)
 624         }
 625         {
 626                 r, n, url2, err := kc.Get(hash)
 627                 c.Check(err, Equals, nil)
 628                 c.Check(n, Equals, int64(len(content)))
 629                 c.Check(url2, Equals, fmt.Sprintf("http://localhost:25108/%s", hash))
 630
 631                 read_content, err2 := ioutil.ReadAll(r)
 632                 c.Check(err2, Equals, nil)
 633                 c.Check(read_content, DeepEquals, content)
 634         }
 635         {
 636                 n, url2, err := kc.Ask(hash)
 637                 c.Check(err, Equals, nil)
 638                 c.Check(n, Equals, int64(len(content)))
 639                 c.Check(url2, Equals, fmt.Sprintf("http://localhost:25108/%s", hash))
 640         }
 641 }
 642
 643 type StubProxyHandler struct {
 644         handled chan string
 645 }
 646
 647 func (this StubProxyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 648         resp.Header().Set("X-Keep-Replicas-Stored", "2")
 649         this.handled <- fmt.Sprintf("http://%s", req.Host)
 650 }
 651
 652 func (s *StandaloneSuite) TestPutProxy(c *C) {
 653         log.Printf("TestPutProxy")
 654
 655         st := StubProxyHandler{make(chan string, 1)}
 656
 657         arv, err := arvadosclient.MakeArvadosClient()
 658         kc, _ := MakeKeepClient(&arv)
 659
 660         kc.Want_replicas = 2
 661         kc.Using_proxy = true
 662         arv.ApiToken = "abc123"
 663         service_roots := make(map[string]string)
 664
 665         ks1 := RunSomeFakeKeepServers(st, 1)
 666
 667         for i, k := range ks1 {
 668                 service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 669                 defer k.listener.Close()
 670         }
 671
 672         kc.SetServiceRoots(service_roots)
 673
 674         _, replicas, err := kc.PutB([]byte("foo"))
 675         <-st.handled
 676
 677         c.Check(err, Equals, nil)
 678         c.Check(replicas, Equals, 2)
 679
 680         log.Printf("TestPutProxy done")
 681 }
 682
 683 func (s *StandaloneSuite) TestPutProxyInsufficientReplicas(c *C) {
 684         log.Printf("TestPutProxy")
 685
 686         st := StubProxyHandler{make(chan string, 1)}
 687
 688         arv, err := arvadosclient.MakeArvadosClient()
 689         kc, _ := MakeKeepClient(&arv)
 690
 691         kc.Want_replicas = 3
 692         kc.Using_proxy = true
 693         arv.ApiToken = "abc123"
 694         service_roots := make(map[string]string)
 695
 696         ks1 := RunSomeFakeKeepServers(st, 1)
 697
 698         for i, k := range ks1 {
 699                 service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 700                 defer k.listener.Close()
 701         }
 702         kc.SetServiceRoots(service_roots)
 703
 704         _, replicas, err := kc.PutB([]byte("foo"))
 705         <-st.handled
 706
 707         c.Check(err, Equals, InsufficientReplicasError)
 708         c.Check(replicas, Equals, 2)
 709
 710         log.Printf("TestPutProxy done")
 711 }
 712
 713 func (s *StandaloneSuite) TestMakeLocator(c *C) {
 714         l := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce+3+Aabcde@12345678")
 715
 716         c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
 717         c.Check(l.Size, Equals, 3)
 718         c.Check(l.Signature, Equals, "abcde")
 719         c.Check(l.Timestamp, Equals, "12345678")
 720 }