sdk/go/keepclient/keepclient_test.go

   1 package keepclient
   2
   3 import (
   4         "crypto/md5"
   5         "flag"
   6         "fmt"
   7         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
   8         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
   9         "git.curoverse.com/arvados.git/sdk/go/streamer"
  10         . "gopkg.in/check.v1"
  11         "io"
  12         "io/ioutil"
  13         "log"
  14         "net"
  15         "net/http"
  16         "os"
  17         "testing"
  18 )
  19
  20 // Gocheck boilerplate
  21 func Test(t *testing.T) {
  22         TestingT(t)
  23 }
  24
  25 // Gocheck boilerplate
  26 var _ = Suite(&ServerRequiredSuite{})
  27 var _ = Suite(&StandaloneSuite{})
  28
  29 var no_server = flag.Bool("no-server", false, "Skip 'ServerRequireSuite'")
  30
  31 // Tests that require the Keep server running
  32 type ServerRequiredSuite struct{}
  33
  34 // Standalone tests
  35 type StandaloneSuite struct{}
  36
  37 func pythonDir() string {
  38         cwd, _ := os.Getwd()
  39         return fmt.Sprintf("%s/../../python/tests", cwd)
  40 }
  41
  42 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
  43         if *no_server {
  44                 c.Skip("Skipping tests that require server")
  45                 return
  46         }
  47         arvadostest.StartAPI()
  48         arvadostest.StartKeep()
  49 }
  50
  51 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
  52         if *no_server {
  53                 return
  54         }
  55         arvadostest.StopKeep()
  56         arvadostest.StopAPI()
  57 }
  58
  59 func (s *ServerRequiredSuite) TestMakeKeepClient(c *C) {
  60         arv, err := arvadosclient.MakeArvadosClient()
  61         c.Assert(err, Equals, nil)
  62
  63         kc, err := MakeKeepClient(&arv)
  64
  65         c.Assert(err, Equals, nil)
  66         c.Check(len(kc.LocalRoots()), Equals, 2)
  67         for _, root := range kc.LocalRoots() {
  68                 c.Check(root, Matches, "http://localhost:\\d+")
  69         }
  70 }
  71
  72 type StubPutHandler struct {
  73         c              *C
  74         expectPath     string
  75         expectApiToken string
  76         expectBody     string
  77         handled        chan string
  78 }
  79
  80 func (sph StubPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
  81         sph.c.Check(req.URL.Path, Equals, "/"+sph.expectPath)
  82         sph.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sph.expectApiToken))
  83         body, err := ioutil.ReadAll(req.Body)
  84         sph.c.Check(err, Equals, nil)
  85         sph.c.Check(body, DeepEquals, []byte(sph.expectBody))
  86         resp.WriteHeader(200)
  87         sph.handled <- fmt.Sprintf("http://%s", req.Host)
  88 }
  89
  90 func RunFakeKeepServer(st http.Handler) (ks KeepServer) {
  91         var err error
  92         ks.listener, err = net.ListenTCP("tcp", &net.TCPAddr{Port: 0})
  93         if err != nil {
  94                 panic(fmt.Sprintf("Could not listen on any port"))
  95         }
  96         ks.url = fmt.Sprintf("http://%s", ks.listener.Addr().String())
  97         go http.Serve(ks.listener, st)
  98         return
  99 }
 100
 101 func UploadToStubHelper(c *C, st http.Handler, f func(*KeepClient, string,
 102         io.ReadCloser, io.WriteCloser, chan uploadStatus)) {
 103
 104         ks := RunFakeKeepServer(st)
 105         defer ks.listener.Close()
 106
 107         arv, _ := arvadosclient.MakeArvadosClient()
 108         arv.ApiToken = "abc123"
 109
 110         kc, _ := MakeKeepClient(&arv)
 111
 112         reader, writer := io.Pipe()
 113         upload_status := make(chan uploadStatus)
 114
 115         f(kc, ks.url, reader, writer, upload_status)
 116 }
 117
 118 func (s *StandaloneSuite) TestUploadToStubKeepServer(c *C) {
 119         log.Printf("TestUploadToStubKeepServer")
 120
 121         st := StubPutHandler{
 122                 c,
 123                 "acbd18db4cc2f85cedef654fccc4a4d8",
 124                 "abc123",
 125                 "foo",
 126                 make(chan string)}
 127
 128         UploadToStubHelper(c, st,
 129                 func(kc *KeepClient, url string, reader io.ReadCloser,
 130                         writer io.WriteCloser, upload_status chan uploadStatus) {
 131
 132                         go kc.uploadToKeepServer(url, st.expectPath, reader, upload_status, int64(len("foo")), "TestUploadToStubKeepServer")
 133
 134                         writer.Write([]byte("foo"))
 135                         writer.Close()
 136
 137                         <-st.handled
 138                         status := <-upload_status
 139                         c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, ""})
 140                 })
 141
 142         log.Printf("TestUploadToStubKeepServer done")
 143 }
 144
 145 func (s *StandaloneSuite) TestUploadToStubKeepServerBufferReader(c *C) {
 146         log.Printf("TestUploadToStubKeepServerBufferReader")
 147
 148         st := StubPutHandler{
 149                 c,
 150                 "acbd18db4cc2f85cedef654fccc4a4d8",
 151                 "abc123",
 152                 "foo",
 153                 make(chan string)}
 154
 155         UploadToStubHelper(c, st,
 156                 func(kc *KeepClient, url string, reader io.ReadCloser,
 157                         writer io.WriteCloser, upload_status chan uploadStatus) {
 158
 159                         tr := streamer.AsyncStreamFromReader(512, reader)
 160                         defer tr.Close()
 161
 162                         br1 := tr.MakeStreamReader()
 163
 164                         go kc.uploadToKeepServer(url, st.expectPath, br1, upload_status, 3, "TestUploadToStubKeepServerBufferReader")
 165
 166                         writer.Write([]byte("foo"))
 167                         writer.Close()
 168
 169                         <-st.handled
 170
 171                         status := <-upload_status
 172                         c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, ""})
 173                 })
 174
 175         log.Printf("TestUploadToStubKeepServerBufferReader done")
 176 }
 177
 178 type FailHandler struct {
 179         handled chan string
 180 }
 181
 182 func (fh FailHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 183         resp.WriteHeader(500)
 184         fh.handled <- fmt.Sprintf("http://%s", req.Host)
 185 }
 186
 187 func (s *StandaloneSuite) TestFailedUploadToStubKeepServer(c *C) {
 188         log.Printf("TestFailedUploadToStubKeepServer")
 189
 190         st := FailHandler{
 191                 make(chan string)}
 192
 193         hash := "acbd18db4cc2f85cedef654fccc4a4d8"
 194
 195         UploadToStubHelper(c, st,
 196                 func(kc *KeepClient, url string, reader io.ReadCloser,
 197                         writer io.WriteCloser, upload_status chan uploadStatus) {
 198
 199                         go kc.uploadToKeepServer(url, hash, reader, upload_status, 3, "TestFailedUploadToStubKeepServer")
 200
 201                         writer.Write([]byte("foo"))
 202                         writer.Close()
 203
 204                         <-st.handled
 205
 206                         status := <-upload_status
 207                         c.Check(status.url, Equals, fmt.Sprintf("%s/%s", url, hash))
 208                         c.Check(status.statusCode, Equals, 500)
 209                 })
 210         log.Printf("TestFailedUploadToStubKeepServer done")
 211 }
 212
 213 type KeepServer struct {
 214         listener net.Listener
 215         url      string
 216 }
 217
 218 func RunSomeFakeKeepServers(st http.Handler, n int) (ks []KeepServer) {
 219         ks = make([]KeepServer, n)
 220
 221         for i := 0; i < n; i += 1 {
 222                 ks[i] = RunFakeKeepServer(st)
 223         }
 224
 225         return ks
 226 }
 227
 228 func (s *StandaloneSuite) TestPutB(c *C) {
 229         log.Printf("TestPutB")
 230
 231         hash := Md5String("foo")
 232
 233         st := StubPutHandler{
 234                 c,
 235                 hash,
 236                 "abc123",
 237                 "foo",
 238                 make(chan string, 5)}
 239
 240         arv, _ := arvadosclient.MakeArvadosClient()
 241         kc, _ := MakeKeepClient(&arv)
 242
 243         kc.Want_replicas = 2
 244         arv.ApiToken = "abc123"
 245         localRoots := make(map[string]string)
 246
 247         ks := RunSomeFakeKeepServers(st, 5)
 248
 249         for i, k := range ks {
 250                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 251                 defer k.listener.Close()
 252         }
 253
 254         kc.SetServiceRoots(localRoots, nil)
 255
 256         kc.PutB([]byte("foo"))
 257
 258         shuff := NewRootSorter(
 259                 kc.LocalRoots(), Md5String("foo")).GetSortedRoots()
 260
 261         s1 := <-st.handled
 262         s2 := <-st.handled
 263         c.Check((s1 == shuff[0] && s2 == shuff[1]) ||
 264                 (s1 == shuff[1] && s2 == shuff[0]),
 265                 Equals,
 266                 true)
 267
 268         log.Printf("TestPutB done")
 269 }
 270
 271 func (s *StandaloneSuite) TestPutHR(c *C) {
 272         log.Printf("TestPutHR")
 273
 274         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 275
 276         st := StubPutHandler{
 277                 c,
 278                 hash,
 279                 "abc123",
 280                 "foo",
 281                 make(chan string, 5)}
 282
 283         arv, _ := arvadosclient.MakeArvadosClient()
 284         kc, _ := MakeKeepClient(&arv)
 285
 286         kc.Want_replicas = 2
 287         arv.ApiToken = "abc123"
 288         localRoots := make(map[string]string)
 289
 290         ks := RunSomeFakeKeepServers(st, 5)
 291
 292         for i, k := range ks {
 293                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 294                 defer k.listener.Close()
 295         }
 296
 297         kc.SetServiceRoots(localRoots, nil)
 298
 299         reader, writer := io.Pipe()
 300
 301         go func() {
 302                 writer.Write([]byte("foo"))
 303                 writer.Close()
 304         }()
 305
 306         kc.PutHR(hash, reader, 3)
 307
 308         shuff := NewRootSorter(kc.LocalRoots(), hash).GetSortedRoots()
 309         log.Print(shuff)
 310
 311         s1 := <-st.handled
 312         s2 := <-st.handled
 313
 314         c.Check((s1 == shuff[0] && s2 == shuff[1]) ||
 315                 (s1 == shuff[1] && s2 == shuff[0]),
 316                 Equals,
 317                 true)
 318
 319         log.Printf("TestPutHR done")
 320 }
 321
 322 func (s *StandaloneSuite) TestPutWithFail(c *C) {
 323         log.Printf("TestPutWithFail")
 324
 325         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 326
 327         st := StubPutHandler{
 328                 c,
 329                 hash,
 330                 "abc123",
 331                 "foo",
 332                 make(chan string, 4)}
 333
 334         fh := FailHandler{
 335                 make(chan string, 1)}
 336
 337         arv, err := arvadosclient.MakeArvadosClient()
 338         kc, _ := MakeKeepClient(&arv)
 339
 340         kc.Want_replicas = 2
 341         arv.ApiToken = "abc123"
 342         localRoots := make(map[string]string)
 343
 344         ks1 := RunSomeFakeKeepServers(st, 4)
 345         ks2 := RunSomeFakeKeepServers(fh, 1)
 346
 347         for i, k := range ks1 {
 348                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 349                 defer k.listener.Close()
 350         }
 351         for i, k := range ks2 {
 352                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
 353                 defer k.listener.Close()
 354         }
 355
 356         kc.SetServiceRoots(localRoots, nil)
 357
 358         shuff := NewRootSorter(
 359                 kc.LocalRoots(), Md5String("foo")).GetSortedRoots()
 360
 361         phash, replicas, err := kc.PutB([]byte("foo"))
 362
 363         <-fh.handled
 364
 365         c.Check(err, Equals, nil)
 366         c.Check(phash, Equals, "")
 367         c.Check(replicas, Equals, 2)
 368
 369         s1 := <-st.handled
 370         s2 := <-st.handled
 371
 372         c.Check((s1 == shuff[1] && s2 == shuff[2]) ||
 373                 (s1 == shuff[2] && s2 == shuff[1]),
 374                 Equals,
 375                 true)
 376 }
 377
 378 func (s *StandaloneSuite) TestPutWithTooManyFail(c *C) {
 379         log.Printf("TestPutWithTooManyFail")
 380
 381         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 382
 383         st := StubPutHandler{
 384                 c,
 385                 hash,
 386                 "abc123",
 387                 "foo",
 388                 make(chan string, 1)}
 389
 390         fh := FailHandler{
 391                 make(chan string, 4)}
 392
 393         arv, err := arvadosclient.MakeArvadosClient()
 394         kc, _ := MakeKeepClient(&arv)
 395
 396         kc.Want_replicas = 2
 397         arv.ApiToken = "abc123"
 398         localRoots := make(map[string]string)
 399
 400         ks1 := RunSomeFakeKeepServers(st, 1)
 401         ks2 := RunSomeFakeKeepServers(fh, 4)
 402
 403         for i, k := range ks1 {
 404                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 405                 defer k.listener.Close()
 406         }
 407         for i, k := range ks2 {
 408                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
 409                 defer k.listener.Close()
 410         }
 411
 412         kc.SetServiceRoots(localRoots, nil)
 413
 414         _, replicas, err := kc.PutB([]byte("foo"))
 415
 416         c.Check(err, Equals, InsufficientReplicasError)
 417         c.Check(replicas, Equals, 1)
 418         c.Check(<-st.handled, Equals, ks1[0].url)
 419
 420         log.Printf("TestPutWithTooManyFail done")
 421 }
 422
 423 type StubGetHandler struct {
 424         c              *C
 425         expectPath     string
 426         expectApiToken string
 427         httpStatus     int
 428         body           []byte
 429 }
 430
 431 func (sgh StubGetHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 432         sgh.c.Check(req.URL.Path, Equals, "/"+sgh.expectPath)
 433         sgh.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sgh.expectApiToken))
 434         resp.WriteHeader(sgh.httpStatus)
 435         resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(sgh.body)))
 436         resp.Write(sgh.body)
 437 }
 438
 439 func (s *StandaloneSuite) TestGet(c *C) {
 440         log.Printf("TestGet")
 441
 442         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 443
 444         st := StubGetHandler{
 445                 c,
 446                 hash,
 447                 "abc123",
 448                 http.StatusOK,
 449                 []byte("foo")}
 450
 451         ks := RunFakeKeepServer(st)
 452         defer ks.listener.Close()
 453
 454         arv, err := arvadosclient.MakeArvadosClient()
 455         kc, _ := MakeKeepClient(&arv)
 456         arv.ApiToken = "abc123"
 457         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil)
 458
 459         r, n, url2, err := kc.Get(hash)
 460         defer r.Close()
 461         c.Check(err, Equals, nil)
 462         c.Check(n, Equals, int64(3))
 463         c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks.url, hash))
 464
 465         content, err2 := ioutil.ReadAll(r)
 466         c.Check(err2, Equals, nil)
 467         c.Check(content, DeepEquals, []byte("foo"))
 468
 469         log.Printf("TestGet done")
 470 }
 471
 472 func (s *StandaloneSuite) TestGetFail(c *C) {
 473         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 474
 475         st := FailHandler{make(chan string, 1)}
 476
 477         ks := RunFakeKeepServer(st)
 478         defer ks.listener.Close()
 479
 480         arv, err := arvadosclient.MakeArvadosClient()
 481         kc, _ := MakeKeepClient(&arv)
 482         arv.ApiToken = "abc123"
 483         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil)
 484
 485         r, n, url2, err := kc.Get(hash)
 486         c.Check(err, Equals, BlockNotFound)
 487         c.Check(n, Equals, int64(0))
 488         c.Check(url2, Equals, "")
 489         c.Check(r, Equals, nil)
 490 }
 491
 492 func (s *StandaloneSuite) TestGetWithServiceHint(c *C) {
 493         uuid := "zzzzz-bi6l4-123451234512345"
 494         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 495
 496         // This one shouldn't be used:
 497         ks0 := RunFakeKeepServer(StubGetHandler{
 498                 c,
 499                 "error if used",
 500                 "abc123",
 501                 http.StatusOK,
 502                 []byte("foo")})
 503         defer ks0.listener.Close()
 504         // This one should be used:
 505         ks := RunFakeKeepServer(StubGetHandler{
 506                 c,
 507                 hash+"+K@"+uuid,
 508                 "abc123",
 509                 http.StatusOK,
 510                 []byte("foo")})
 511         defer ks.listener.Close()
 512
 513         arv, err := arvadosclient.MakeArvadosClient()
 514         kc, _ := MakeKeepClient(&arv)
 515         arv.ApiToken = "abc123"
 516         kc.SetServiceRoots(
 517                 map[string]string{"x": ks0.url},
 518                 map[string]string{uuid: ks.url})
 519
 520         r, n, uri, err := kc.Get(hash+"+K@"+uuid)
 521         defer r.Close()
 522         c.Check(err, Equals, nil)
 523         c.Check(n, Equals, int64(3))
 524         c.Check(uri, Equals, fmt.Sprintf("%s/%s", ks.url, hash+"+K@"+uuid))
 525
 526         content, err := ioutil.ReadAll(r)
 527         c.Check(err, Equals, nil)
 528         c.Check(content, DeepEquals, []byte("foo"))
 529 }
 530
 531 func (s *StandaloneSuite) TestGetWithServiceHintFailoverToLocals(c *C) {
 532         uuid := "zzzzz-bi6l4-123451234512345"
 533         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 534
 535         ksLocal := RunFakeKeepServer(StubGetHandler{
 536                 c,
 537                 hash+"+K@"+uuid,
 538                 "abc123",
 539                 http.StatusOK,
 540                 []byte("foo")})
 541         defer ksLocal.listener.Close()
 542         ksGateway := RunFakeKeepServer(StubGetHandler{
 543                 c,
 544                 hash+"+K@"+uuid,
 545                 "abc123",
 546                 http.StatusInternalServerError,
 547                 []byte("Error")})
 548         defer ksGateway.listener.Close()
 549
 550         arv, err := arvadosclient.MakeArvadosClient()
 551         kc, _ := MakeKeepClient(&arv)
 552         arv.ApiToken = "abc123"
 553         kc.SetServiceRoots(
 554                 map[string]string{"zzzzz-bi6l4-keepdisk0000000": ksLocal.url},
 555                 map[string]string{uuid: ksGateway.url})
 556
 557         r, n, uri, err := kc.Get(hash+"+K@"+uuid)
 558         c.Assert(err, Equals, nil)
 559         defer r.Close()
 560         c.Check(n, Equals, int64(3))
 561         c.Check(uri, Equals, fmt.Sprintf("%s/%s", ksLocal.url, hash+"+K@"+uuid))
 562
 563         content, err := ioutil.ReadAll(r)
 564         c.Check(err, Equals, nil)
 565         c.Check(content, DeepEquals, []byte("foo"))
 566 }
 567
 568 type BarHandler struct {
 569         handled chan string
 570 }
 571
 572 func (this BarHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 573         resp.Write([]byte("bar"))
 574         this.handled <- fmt.Sprintf("http://%s", req.Host)
 575 }
 576
 577 func (s *StandaloneSuite) TestChecksum(c *C) {
 578         foohash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 579         barhash := fmt.Sprintf("%x", md5.Sum([]byte("bar")))
 580
 581         st := BarHandler{make(chan string, 1)}
 582
 583         ks := RunFakeKeepServer(st)
 584         defer ks.listener.Close()
 585
 586         arv, err := arvadosclient.MakeArvadosClient()
 587         kc, _ := MakeKeepClient(&arv)
 588         arv.ApiToken = "abc123"
 589         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil)
 590
 591         r, n, _, err := kc.Get(barhash)
 592         _, err = ioutil.ReadAll(r)
 593         c.Check(n, Equals, int64(3))
 594         c.Check(err, Equals, nil)
 595
 596         <-st.handled
 597
 598         r, n, _, err = kc.Get(foohash)
 599         _, err = ioutil.ReadAll(r)
 600         c.Check(n, Equals, int64(3))
 601         c.Check(err, Equals, BadChecksum)
 602
 603         <-st.handled
 604 }
 605
 606 func (s *StandaloneSuite) TestGetWithFailures(c *C) {
 607         content := []byte("waz")
 608         hash := fmt.Sprintf("%x", md5.Sum(content))
 609
 610         fh := FailHandler{
 611                 make(chan string, 4)}
 612
 613         st := StubGetHandler{
 614                 c,
 615                 hash,
 616                 "abc123",
 617                 http.StatusOK,
 618                 content}
 619
 620         arv, err := arvadosclient.MakeArvadosClient()
 621         kc, _ := MakeKeepClient(&arv)
 622         arv.ApiToken = "abc123"
 623         localRoots := make(map[string]string)
 624
 625         ks1 := RunSomeFakeKeepServers(st, 1)
 626         ks2 := RunSomeFakeKeepServers(fh, 4)
 627
 628         for i, k := range ks1 {
 629                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 630                 defer k.listener.Close()
 631         }
 632         for i, k := range ks2 {
 633                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
 634                 defer k.listener.Close()
 635         }
 636
 637         kc.SetServiceRoots(localRoots, nil)
 638
 639         // This test works only if one of the failing services is
 640         // attempted before the succeeding service. Otherwise,
 641         // <-fh.handled below will just hang! (Probe order depends on
 642         // the choice of block content "waz" and the UUIDs of the fake
 643         // servers, so we just tried different strings until we found
 644         // an example that passes this Assert.)
 645         c.Assert(NewRootSorter(localRoots, hash).GetSortedRoots()[0], Not(Equals), ks1[0].url)
 646
 647         r, n, url2, err := kc.Get(hash)
 648
 649         <-fh.handled
 650         c.Check(err, Equals, nil)
 651         c.Check(n, Equals, int64(3))
 652         c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks1[0].url, hash))
 653
 654         read_content, err2 := ioutil.ReadAll(r)
 655         c.Check(err2, Equals, nil)
 656         c.Check(read_content, DeepEquals, content)
 657 }
 658
 659 func (s *ServerRequiredSuite) TestPutGetHead(c *C) {
 660         content := []byte("TestPutGetHead")
 661
 662         arv, err := arvadosclient.MakeArvadosClient()
 663         kc, err := MakeKeepClient(&arv)
 664         c.Assert(err, Equals, nil)
 665
 666         hash := fmt.Sprintf("%x", md5.Sum(content))
 667
 668         {
 669                 n, _, err := kc.Ask(hash)
 670                 c.Check(err, Equals, BlockNotFound)
 671                 c.Check(n, Equals, int64(0))
 672         }
 673         {
 674                 hash2, replicas, err := kc.PutB(content)
 675                 c.Check(hash2, Equals, fmt.Sprintf("%s+%d", hash, len(content)))
 676                 c.Check(replicas, Equals, 2)
 677                 c.Check(err, Equals, nil)
 678         }
 679         {
 680                 r, n, url2, err := kc.Get(hash)
 681                 c.Check(err, Equals, nil)
 682                 c.Check(n, Equals, int64(len(content)))
 683                 c.Check(url2, Matches, fmt.Sprintf("http://localhost:\\d+/%s", hash))
 684
 685                 read_content, err2 := ioutil.ReadAll(r)
 686                 c.Check(err2, Equals, nil)
 687                 c.Check(read_content, DeepEquals, content)
 688         }
 689         {
 690                 n, url2, err := kc.Ask(hash)
 691                 c.Check(err, Equals, nil)
 692                 c.Check(n, Equals, int64(len(content)))
 693                 c.Check(url2, Matches, fmt.Sprintf("http://localhost:\\d+/%s", hash))
 694         }
 695 }
 696
 697 type StubProxyHandler struct {
 698         handled chan string
 699 }
 700
 701 func (this StubProxyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 702         resp.Header().Set("X-Keep-Replicas-Stored", "2")
 703         this.handled <- fmt.Sprintf("http://%s", req.Host)
 704 }
 705
 706 func (s *StandaloneSuite) TestPutProxy(c *C) {
 707         log.Printf("TestPutProxy")
 708
 709         st := StubProxyHandler{make(chan string, 1)}
 710
 711         arv, err := arvadosclient.MakeArvadosClient()
 712         kc, _ := MakeKeepClient(&arv)
 713
 714         kc.Want_replicas = 2
 715         kc.Using_proxy = true
 716         arv.ApiToken = "abc123"
 717         localRoots := make(map[string]string)
 718
 719         ks1 := RunSomeFakeKeepServers(st, 1)
 720
 721         for i, k := range ks1 {
 722                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 723                 defer k.listener.Close()
 724         }
 725
 726         kc.SetServiceRoots(localRoots, nil)
 727
 728         _, replicas, err := kc.PutB([]byte("foo"))
 729         <-st.handled
 730
 731         c.Check(err, Equals, nil)
 732         c.Check(replicas, Equals, 2)
 733
 734         log.Printf("TestPutProxy done")
 735 }
 736
 737 func (s *StandaloneSuite) TestPutProxyInsufficientReplicas(c *C) {
 738         log.Printf("TestPutProxy")
 739
 740         st := StubProxyHandler{make(chan string, 1)}
 741
 742         arv, err := arvadosclient.MakeArvadosClient()
 743         kc, _ := MakeKeepClient(&arv)
 744
 745         kc.Want_replicas = 3
 746         kc.Using_proxy = true
 747         arv.ApiToken = "abc123"
 748         localRoots := make(map[string]string)
 749
 750         ks1 := RunSomeFakeKeepServers(st, 1)
 751
 752         for i, k := range ks1 {
 753                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 754                 defer k.listener.Close()
 755         }
 756         kc.SetServiceRoots(localRoots, nil)
 757
 758         _, replicas, err := kc.PutB([]byte("foo"))
 759         <-st.handled
 760
 761         c.Check(err, Equals, InsufficientReplicasError)
 762         c.Check(replicas, Equals, 2)
 763
 764         log.Printf("TestPutProxy done")
 765 }
 766
 767 func (s *StandaloneSuite) TestMakeLocator(c *C) {
 768         l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce+3+Aabcde@12345678")
 769         c.Check(err, Equals, nil)
 770         c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
 771         c.Check(l.Size, Equals, 3)
 772         c.Check(l.Hints, DeepEquals, []string{"3", "Aabcde@12345678"})
 773 }
 774
 775 func (s *StandaloneSuite) TestMakeLocatorNoHints(c *C) {
 776         l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce")
 777         c.Check(err, Equals, nil)
 778         c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
 779         c.Check(l.Size, Equals, -1)
 780         c.Check(l.Hints, DeepEquals, []string{})
 781 }
 782
 783 func (s *StandaloneSuite) TestMakeLocatorNoSizeHint(c *C) {
 784         l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce+Aabcde@12345678")
 785         c.Check(err, Equals, nil)
 786         c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
 787         c.Check(l.Size, Equals, -1)
 788         c.Check(l.Hints, DeepEquals, []string{"Aabcde@12345678"})
 789 }
 790
 791 func (s *StandaloneSuite) TestMakeLocatorInvalidInput(c *C) {
 792         _, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31c")
 793         c.Check(err, Equals, InvalidLocatorError)
 794 }