sdk/go/keepclient/keepclient_test.go

   1 package keepclient
   2
   3 import (
   4         "crypto/md5"
   5         "flag"
   6         "fmt"
   7         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
   8         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
   9         "git.curoverse.com/arvados.git/sdk/go/streamer"
  10         . "gopkg.in/check.v1"
  11         "io"
  12         "io/ioutil"
  13         "log"
  14         "net"
  15         "net/http"
  16         "os"
  17         "testing"
  18 )
  19
  20 // Gocheck boilerplate
  21 func Test(t *testing.T) {
  22         TestingT(t)
  23 }
  24
  25 // Gocheck boilerplate
  26 var _ = Suite(&ServerRequiredSuite{})
  27 var _ = Suite(&StandaloneSuite{})
  28
  29 var no_server = flag.Bool("no-server", false, "Skip 'ServerRequireSuite'")
  30
  31 // Tests that require the Keep server running
  32 type ServerRequiredSuite struct{}
  33
  34 // Standalone tests
  35 type StandaloneSuite struct{}
  36
  37 func pythonDir() string {
  38         cwd, _ := os.Getwd()
  39         return fmt.Sprintf("%s/../../python/tests", cwd)
  40 }
  41
  42 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
  43         if *no_server {
  44                 c.Skip("Skipping tests that require server")
  45                 return
  46         }
  47         arvadostest.StartAPI()
  48         arvadostest.StartKeep()
  49 }
  50
  51 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
  52         if *no_server {
  53                 return
  54         }
  55         arvadostest.StopKeep()
  56         arvadostest.StopAPI()
  57 }
  58
  59 func (s *ServerRequiredSuite) TestMakeKeepClient(c *C) {
  60         arv, err := arvadosclient.MakeArvadosClient()
  61         c.Assert(err, Equals, nil)
  62
  63         kc, err := MakeKeepClient(&arv)
  64
  65         c.Assert(err, Equals, nil)
  66         c.Check(len(kc.LocalRoots()), Equals, 2)
  67         for _, root := range kc.LocalRoots() {
  68                 c.Check(root, Matches, "http://localhost:\\d+")
  69         }
  70 }
  71
  72 type StubPutHandler struct {
  73         c              *C
  74         expectPath     string
  75         expectApiToken string
  76         expectBody     string
  77         handled        chan string
  78 }
  79
  80 func (sph StubPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
  81         sph.c.Check(req.URL.Path, Equals, "/"+sph.expectPath)
  82         sph.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sph.expectApiToken))
  83         body, err := ioutil.ReadAll(req.Body)
  84         sph.c.Check(err, Equals, nil)
  85         sph.c.Check(body, DeepEquals, []byte(sph.expectBody))
  86         resp.WriteHeader(200)
  87         sph.handled <- fmt.Sprintf("http://%s", req.Host)
  88 }
  89
  90 func RunFakeKeepServer(st http.Handler) (ks KeepServer) {
  91         var err error
  92         ks.listener, err = net.ListenTCP("tcp", &net.TCPAddr{Port: 0})
  93         if err != nil {
  94                 panic(fmt.Sprintf("Could not listen on any port"))
  95         }
  96         ks.url = fmt.Sprintf("http://%s", ks.listener.Addr().String())
  97         go http.Serve(ks.listener, st)
  98         return
  99 }
 100
 101 func UploadToStubHelper(c *C, st http.Handler, f func(*KeepClient, string,
 102         io.ReadCloser, io.WriteCloser, chan uploadStatus)) {
 103
 104         ks := RunFakeKeepServer(st)
 105         defer ks.listener.Close()
 106
 107         arv, _ := arvadosclient.MakeArvadosClient()
 108         arv.ApiToken = "abc123"
 109
 110         kc, _ := MakeKeepClient(&arv)
 111
 112         reader, writer := io.Pipe()
 113         upload_status := make(chan uploadStatus)
 114
 115         f(kc, ks.url, reader, writer, upload_status)
 116 }
 117
 118 func (s *StandaloneSuite) TestUploadToStubKeepServer(c *C) {
 119         log.Printf("TestUploadToStubKeepServer")
 120
 121         st := StubPutHandler{
 122                 c,
 123                 "acbd18db4cc2f85cedef654fccc4a4d8",
 124                 "abc123",
 125                 "foo",
 126                 make(chan string)}
 127
 128         UploadToStubHelper(c, st,
 129                 func(kc *KeepClient, url string, reader io.ReadCloser,
 130                         writer io.WriteCloser, upload_status chan uploadStatus) {
 131
 132                         go kc.uploadToKeepServer(url, st.expectPath, reader, upload_status, int64(len("foo")), "TestUploadToStubKeepServer")
 133
 134                         writer.Write([]byte("foo"))
 135                         writer.Close()
 136
 137                         <-st.handled
 138                         status := <-upload_status
 139                         c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, ""})
 140                 })
 141
 142         log.Printf("TestUploadToStubKeepServer done")
 143 }
 144
 145 func (s *StandaloneSuite) TestUploadToStubKeepServerBufferReader(c *C) {
 146         log.Printf("TestUploadToStubKeepServerBufferReader")
 147
 148         st := StubPutHandler{
 149                 c,
 150                 "acbd18db4cc2f85cedef654fccc4a4d8",
 151                 "abc123",
 152                 "foo",
 153                 make(chan string)}
 154
 155         UploadToStubHelper(c, st,
 156                 func(kc *KeepClient, url string, reader io.ReadCloser,
 157                         writer io.WriteCloser, upload_status chan uploadStatus) {
 158
 159                         tr := streamer.AsyncStreamFromReader(512, reader)
 160                         defer tr.Close()
 161
 162                         br1 := tr.MakeStreamReader()
 163
 164                         go kc.uploadToKeepServer(url, st.expectPath, br1, upload_status, 3, "TestUploadToStubKeepServerBufferReader")
 165
 166                         writer.Write([]byte("foo"))
 167                         writer.Close()
 168
 169                         <-st.handled
 170
 171                         status := <-upload_status
 172                         c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, ""})
 173                 })
 174
 175         log.Printf("TestUploadToStubKeepServerBufferReader done")
 176 }
 177
 178 type FailHandler struct {
 179         handled chan string
 180 }
 181
 182 func (fh FailHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 183         resp.WriteHeader(500)
 184         fh.handled <- fmt.Sprintf("http://%s", req.Host)
 185 }
 186
 187 func (s *StandaloneSuite) TestFailedUploadToStubKeepServer(c *C) {
 188         log.Printf("TestFailedUploadToStubKeepServer")
 189
 190         st := FailHandler{
 191                 make(chan string)}
 192
 193         hash := "acbd18db4cc2f85cedef654fccc4a4d8"
 194
 195         UploadToStubHelper(c, st,
 196                 func(kc *KeepClient, url string, reader io.ReadCloser,
 197                         writer io.WriteCloser, upload_status chan uploadStatus) {
 198
 199                         go kc.uploadToKeepServer(url, hash, reader, upload_status, 3, "TestFailedUploadToStubKeepServer")
 200
 201                         writer.Write([]byte("foo"))
 202                         writer.Close()
 203
 204                         <-st.handled
 205
 206                         status := <-upload_status
 207                         c.Check(status.url, Equals, fmt.Sprintf("%s/%s", url, hash))
 208                         c.Check(status.statusCode, Equals, 500)
 209                 })
 210         log.Printf("TestFailedUploadToStubKeepServer done")
 211 }
 212
 213 type KeepServer struct {
 214         listener net.Listener
 215         url      string
 216 }
 217
 218 func RunSomeFakeKeepServers(st http.Handler, n int) (ks []KeepServer) {
 219         ks = make([]KeepServer, n)
 220
 221         for i := 0; i < n; i += 1 {
 222                 ks[i] = RunFakeKeepServer(st)
 223         }
 224
 225         return ks
 226 }
 227
 228 func (s *StandaloneSuite) TestPutB(c *C) {
 229         log.Printf("TestPutB")
 230
 231         hash := Md5String("foo")
 232
 233         st := StubPutHandler{
 234                 c,
 235                 hash,
 236                 "abc123",
 237                 "foo",
 238                 make(chan string, 5)}
 239
 240         arv, _ := arvadosclient.MakeArvadosClient()
 241         kc, _ := MakeKeepClient(&arv)
 242
 243         kc.Want_replicas = 2
 244         arv.ApiToken = "abc123"
 245         localRoots := make(map[string]string)
 246
 247         ks := RunSomeFakeKeepServers(st, 5)
 248
 249         for i, k := range ks {
 250                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 251                 defer k.listener.Close()
 252         }
 253
 254         kc.SetServiceRoots(localRoots, nil)
 255
 256         kc.PutB([]byte("foo"))
 257
 258         shuff := NewRootSorter(
 259                 kc.LocalRoots(), Md5String("foo")).GetSortedRoots()
 260
 261         s1 := <-st.handled
 262         s2 := <-st.handled
 263         c.Check((s1 == shuff[0] && s2 == shuff[1]) ||
 264                 (s1 == shuff[1] && s2 == shuff[0]),
 265                 Equals,
 266                 true)
 267
 268         log.Printf("TestPutB done")
 269 }
 270
 271 func (s *StandaloneSuite) TestPutHR(c *C) {
 272         log.Printf("TestPutHR")
 273
 274         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 275
 276         st := StubPutHandler{
 277                 c,
 278                 hash,
 279                 "abc123",
 280                 "foo",
 281                 make(chan string, 5)}
 282
 283         arv, _ := arvadosclient.MakeArvadosClient()
 284         kc, _ := MakeKeepClient(&arv)
 285
 286         kc.Want_replicas = 2
 287         arv.ApiToken = "abc123"
 288         localRoots := make(map[string]string)
 289
 290         ks := RunSomeFakeKeepServers(st, 5)
 291
 292         for i, k := range ks {
 293                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 294                 defer k.listener.Close()
 295         }
 296
 297         kc.SetServiceRoots(localRoots, nil)
 298
 299         reader, writer := io.Pipe()
 300
 301         go func() {
 302                 writer.Write([]byte("foo"))
 303                 writer.Close()
 304         }()
 305
 306         kc.PutHR(hash, reader, 3)
 307
 308         shuff := NewRootSorter(kc.LocalRoots(), hash).GetSortedRoots()
 309         log.Print(shuff)
 310
 311         s1 := <-st.handled
 312         s2 := <-st.handled
 313
 314         c.Check((s1 == shuff[0] && s2 == shuff[1]) ||
 315                 (s1 == shuff[1] && s2 == shuff[0]),
 316                 Equals,
 317                 true)
 318
 319         log.Printf("TestPutHR done")
 320 }
 321
 322 func (s *StandaloneSuite) TestPutWithFail(c *C) {
 323         log.Printf("TestPutWithFail")
 324
 325         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 326
 327         st := StubPutHandler{
 328                 c,
 329                 hash,
 330                 "abc123",
 331                 "foo",
 332                 make(chan string, 4)}
 333
 334         fh := FailHandler{
 335                 make(chan string, 1)}
 336
 337         arv, err := arvadosclient.MakeArvadosClient()
 338         kc, _ := MakeKeepClient(&arv)
 339
 340         kc.Want_replicas = 2
 341         arv.ApiToken = "abc123"
 342         localRoots := make(map[string]string)
 343
 344         ks1 := RunSomeFakeKeepServers(st, 4)
 345         ks2 := RunSomeFakeKeepServers(fh, 1)
 346
 347         for i, k := range ks1 {
 348                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 349                 defer k.listener.Close()
 350         }
 351         for i, k := range ks2 {
 352                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
 353                 defer k.listener.Close()
 354         }
 355
 356         kc.SetServiceRoots(localRoots, nil)
 357
 358         shuff := NewRootSorter(
 359                 kc.LocalRoots(), Md5String("foo")).GetSortedRoots()
 360
 361         phash, replicas, err := kc.PutB([]byte("foo"))
 362
 363         <-fh.handled
 364
 365         c.Check(err, Equals, nil)
 366         c.Check(phash, Equals, "")
 367         c.Check(replicas, Equals, 2)
 368
 369         s1 := <-st.handled
 370         s2 := <-st.handled
 371
 372         c.Check((s1 == shuff[1] && s2 == shuff[2]) ||
 373                 (s1 == shuff[2] && s2 == shuff[1]),
 374                 Equals,
 375                 true)
 376 }
 377
 378 func (s *StandaloneSuite) TestPutWithTooManyFail(c *C) {
 379         log.Printf("TestPutWithTooManyFail")
 380
 381         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 382
 383         st := StubPutHandler{
 384                 c,
 385                 hash,
 386                 "abc123",
 387                 "foo",
 388                 make(chan string, 1)}
 389
 390         fh := FailHandler{
 391                 make(chan string, 4)}
 392
 393         arv, err := arvadosclient.MakeArvadosClient()
 394         kc, _ := MakeKeepClient(&arv)
 395
 396         kc.Want_replicas = 2
 397         arv.ApiToken = "abc123"
 398         localRoots := make(map[string]string)
 399
 400         ks1 := RunSomeFakeKeepServers(st, 1)
 401         ks2 := RunSomeFakeKeepServers(fh, 4)
 402
 403         for i, k := range ks1 {
 404                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 405                 defer k.listener.Close()
 406         }
 407         for i, k := range ks2 {
 408                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
 409                 defer k.listener.Close()
 410         }
 411
 412         kc.SetServiceRoots(localRoots, nil)
 413
 414         _, replicas, err := kc.PutB([]byte("foo"))
 415
 416         c.Check(err, Equals, InsufficientReplicasError)
 417         c.Check(replicas, Equals, 1)
 418         c.Check(<-st.handled, Equals, ks1[0].url)
 419
 420         log.Printf("TestPutWithTooManyFail done")
 421 }
 422
 423 type StubGetHandler struct {
 424         c              *C
 425         expectPath     string
 426         expectApiToken string
 427         httpStatus     int
 428         body           []byte
 429 }
 430
 431 func (sgh StubGetHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 432         sgh.c.Check(req.URL.Path, Equals, "/"+sgh.expectPath)
 433         sgh.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sgh.expectApiToken))
 434         resp.WriteHeader(sgh.httpStatus)
 435         resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(sgh.body)))
 436         resp.Write(sgh.body)
 437 }
 438
 439 func (s *StandaloneSuite) TestGet(c *C) {
 440         log.Printf("TestGet")
 441
 442         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 443
 444         st := StubGetHandler{
 445                 c,
 446                 hash,
 447                 "abc123",
 448                 http.StatusOK,
 449                 []byte("foo")}
 450
 451         ks := RunFakeKeepServer(st)
 452         defer ks.listener.Close()
 453
 454         arv, err := arvadosclient.MakeArvadosClient()
 455         kc, _ := MakeKeepClient(&arv)
 456         arv.ApiToken = "abc123"
 457         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil)
 458
 459         r, n, url2, err := kc.Get(hash)
 460         defer r.Close()
 461         c.Check(err, Equals, nil)
 462         c.Check(n, Equals, int64(3))
 463         c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks.url, hash))
 464
 465         content, err2 := ioutil.ReadAll(r)
 466         c.Check(err2, Equals, nil)
 467         c.Check(content, DeepEquals, []byte("foo"))
 468
 469         log.Printf("TestGet done")
 470 }
 471
 472 func (s *StandaloneSuite) TestGetFail(c *C) {
 473         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 474
 475         st := FailHandler{make(chan string, 1)}
 476
 477         ks := RunFakeKeepServer(st)
 478         defer ks.listener.Close()
 479
 480         arv, err := arvadosclient.MakeArvadosClient()
 481         kc, _ := MakeKeepClient(&arv)
 482         arv.ApiToken = "abc123"
 483         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil)
 484
 485         r, n, url2, err := kc.Get(hash)
 486         c.Check(err, Equals, BlockNotFound)
 487         c.Check(n, Equals, int64(0))
 488         c.Check(url2, Equals, "")
 489         c.Check(r, Equals, nil)
 490 }
 491
 492 func (s *StandaloneSuite) TestGetWithServiceHint(c *C) {
 493         uuid := "zzzzz-bi6l4-123451234512345"
 494         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 495
 496         // This one shouldn't be used:
 497         ks0 := RunFakeKeepServer(StubGetHandler{
 498                 c,
 499                 "error if used",
 500                 "abc123",
 501                 http.StatusOK,
 502                 []byte("foo")})
 503         defer ks0.listener.Close()
 504         // This one should be used:
 505         ks := RunFakeKeepServer(StubGetHandler{
 506                 c,
 507                 hash + "+K@" + uuid,
 508                 "abc123",
 509                 http.StatusOK,
 510                 []byte("foo")})
 511         defer ks.listener.Close()
 512
 513         arv, err := arvadosclient.MakeArvadosClient()
 514         kc, _ := MakeKeepClient(&arv)
 515         arv.ApiToken = "abc123"
 516         kc.SetServiceRoots(
 517                 map[string]string{"x": ks0.url},
 518                 map[string]string{uuid: ks.url})
 519
 520         r, n, uri, err := kc.Get(hash + "+K@" + uuid)
 521         defer r.Close()
 522         c.Check(err, Equals, nil)
 523         c.Check(n, Equals, int64(3))
 524         c.Check(uri, Equals, fmt.Sprintf("%s/%s", ks.url, hash+"+K@"+uuid))
 525
 526         content, err := ioutil.ReadAll(r)
 527         c.Check(err, Equals, nil)
 528         c.Check(content, DeepEquals, []byte("foo"))
 529 }
 530
 531 // Use a service hint to fetch from a local disk service, overriding
 532 // rendezvous probe order.
 533 func (s *StandaloneSuite) TestGetWithLocalServiceHint(c *C) {
 534         uuid := "zzzzz-bi6l4-zzzzzzzzzzzzzzz"
 535         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 536
 537         // This one shouldn't be used, although it appears first in
 538         // rendezvous probe order:
 539         ks0 := RunFakeKeepServer(StubGetHandler{
 540                 c,
 541                 "error if used",
 542                 "abc123",
 543                 http.StatusOK,
 544                 []byte("foo")})
 545         defer ks0.listener.Close()
 546         // This one should be used:
 547         ks := RunFakeKeepServer(StubGetHandler{
 548                 c,
 549                 hash + "+K@" + uuid,
 550                 "abc123",
 551                 http.StatusOK,
 552                 []byte("foo")})
 553         defer ks.listener.Close()
 554
 555         arv, err := arvadosclient.MakeArvadosClient()
 556         kc, _ := MakeKeepClient(&arv)
 557         arv.ApiToken = "abc123"
 558         kc.SetServiceRoots(
 559                 map[string]string{
 560                         "zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url,
 561                         "zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url,
 562                         "zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url,
 563                         uuid: ks.url},
 564                 map[string]string{
 565                         "zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url,
 566                         "zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url,
 567                         "zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url,
 568                         uuid: ks.url},
 569         )
 570
 571         r, n, uri, err := kc.Get(hash + "+K@" + uuid)
 572         defer r.Close()
 573         c.Check(err, Equals, nil)
 574         c.Check(n, Equals, int64(3))
 575         c.Check(uri, Equals, fmt.Sprintf("%s/%s", ks.url, hash+"+K@"+uuid))
 576
 577         content, err := ioutil.ReadAll(r)
 578         c.Check(err, Equals, nil)
 579         c.Check(content, DeepEquals, []byte("foo"))
 580 }
 581
 582 func (s *StandaloneSuite) TestGetWithServiceHintFailoverToLocals(c *C) {
 583         uuid := "zzzzz-bi6l4-123451234512345"
 584         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 585
 586         ksLocal := RunFakeKeepServer(StubGetHandler{
 587                 c,
 588                 hash + "+K@" + uuid,
 589                 "abc123",
 590                 http.StatusOK,
 591                 []byte("foo")})
 592         defer ksLocal.listener.Close()
 593         ksGateway := RunFakeKeepServer(StubGetHandler{
 594                 c,
 595                 hash + "+K@" + uuid,
 596                 "abc123",
 597                 http.StatusInternalServerError,
 598                 []byte("Error")})
 599         defer ksGateway.listener.Close()
 600
 601         arv, err := arvadosclient.MakeArvadosClient()
 602         kc, _ := MakeKeepClient(&arv)
 603         arv.ApiToken = "abc123"
 604         kc.SetServiceRoots(
 605                 map[string]string{"zzzzz-bi6l4-keepdisk0000000": ksLocal.url},
 606                 map[string]string{uuid: ksGateway.url})
 607
 608         r, n, uri, err := kc.Get(hash + "+K@" + uuid)
 609         c.Assert(err, Equals, nil)
 610         defer r.Close()
 611         c.Check(n, Equals, int64(3))
 612         c.Check(uri, Equals, fmt.Sprintf("%s/%s", ksLocal.url, hash+"+K@"+uuid))
 613
 614         content, err := ioutil.ReadAll(r)
 615         c.Check(err, Equals, nil)
 616         c.Check(content, DeepEquals, []byte("foo"))
 617 }
 618
 619 type BarHandler struct {
 620         handled chan string
 621 }
 622
 623 func (this BarHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 624         resp.Write([]byte("bar"))
 625         this.handled <- fmt.Sprintf("http://%s", req.Host)
 626 }
 627
 628 func (s *StandaloneSuite) TestChecksum(c *C) {
 629         foohash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 630         barhash := fmt.Sprintf("%x", md5.Sum([]byte("bar")))
 631
 632         st := BarHandler{make(chan string, 1)}
 633
 634         ks := RunFakeKeepServer(st)
 635         defer ks.listener.Close()
 636
 637         arv, err := arvadosclient.MakeArvadosClient()
 638         kc, _ := MakeKeepClient(&arv)
 639         arv.ApiToken = "abc123"
 640         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil)
 641
 642         r, n, _, err := kc.Get(barhash)
 643         _, err = ioutil.ReadAll(r)
 644         c.Check(n, Equals, int64(3))
 645         c.Check(err, Equals, nil)
 646
 647         <-st.handled
 648
 649         r, n, _, err = kc.Get(foohash)
 650         _, err = ioutil.ReadAll(r)
 651         c.Check(n, Equals, int64(3))
 652         c.Check(err, Equals, BadChecksum)
 653
 654         <-st.handled
 655 }
 656
 657 func (s *StandaloneSuite) TestGetWithFailures(c *C) {
 658         content := []byte("waz")
 659         hash := fmt.Sprintf("%x", md5.Sum(content))
 660
 661         fh := FailHandler{
 662                 make(chan string, 4)}
 663
 664         st := StubGetHandler{
 665                 c,
 666                 hash,
 667                 "abc123",
 668                 http.StatusOK,
 669                 content}
 670
 671         arv, err := arvadosclient.MakeArvadosClient()
 672         kc, _ := MakeKeepClient(&arv)
 673         arv.ApiToken = "abc123"
 674         localRoots := make(map[string]string)
 675
 676         ks1 := RunSomeFakeKeepServers(st, 1)
 677         ks2 := RunSomeFakeKeepServers(fh, 4)
 678
 679         for i, k := range ks1 {
 680                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 681                 defer k.listener.Close()
 682         }
 683         for i, k := range ks2 {
 684                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
 685                 defer k.listener.Close()
 686         }
 687
 688         kc.SetServiceRoots(localRoots, nil)
 689
 690         // This test works only if one of the failing services is
 691         // attempted before the succeeding service. Otherwise,
 692         // <-fh.handled below will just hang! (Probe order depends on
 693         // the choice of block content "waz" and the UUIDs of the fake
 694         // servers, so we just tried different strings until we found
 695         // an example that passes this Assert.)
 696         c.Assert(NewRootSorter(localRoots, hash).GetSortedRoots()[0], Not(Equals), ks1[0].url)
 697
 698         r, n, url2, err := kc.Get(hash)
 699
 700         <-fh.handled
 701         c.Check(err, Equals, nil)
 702         c.Check(n, Equals, int64(3))
 703         c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks1[0].url, hash))
 704
 705         read_content, err2 := ioutil.ReadAll(r)
 706         c.Check(err2, Equals, nil)
 707         c.Check(read_content, DeepEquals, content)
 708 }
 709
 710 func (s *ServerRequiredSuite) TestPutGetHead(c *C) {
 711         content := []byte("TestPutGetHead")
 712
 713         arv, err := arvadosclient.MakeArvadosClient()
 714         kc, err := MakeKeepClient(&arv)
 715         c.Assert(err, Equals, nil)
 716
 717         hash := fmt.Sprintf("%x", md5.Sum(content))
 718
 719         {
 720                 n, _, err := kc.Ask(hash)
 721                 c.Check(err, Equals, BlockNotFound)
 722                 c.Check(n, Equals, int64(0))
 723         }
 724         {
 725                 hash2, replicas, err := kc.PutB(content)
 726                 c.Check(hash2, Equals, fmt.Sprintf("%s+%d", hash, len(content)))
 727                 c.Check(replicas, Equals, 2)
 728                 c.Check(err, Equals, nil)
 729         }
 730         {
 731                 r, n, url2, err := kc.Get(hash)
 732                 c.Check(err, Equals, nil)
 733                 c.Check(n, Equals, int64(len(content)))
 734                 c.Check(url2, Matches, fmt.Sprintf("http://localhost:\\d+/%s", hash))
 735
 736                 read_content, err2 := ioutil.ReadAll(r)
 737                 c.Check(err2, Equals, nil)
 738                 c.Check(read_content, DeepEquals, content)
 739         }
 740         {
 741                 n, url2, err := kc.Ask(hash)
 742                 c.Check(err, Equals, nil)
 743                 c.Check(n, Equals, int64(len(content)))
 744                 c.Check(url2, Matches, fmt.Sprintf("http://localhost:\\d+/%s", hash))
 745         }
 746 }
 747
 748 type StubProxyHandler struct {
 749         handled chan string
 750 }
 751
 752 func (this StubProxyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 753         resp.Header().Set("X-Keep-Replicas-Stored", "2")
 754         this.handled <- fmt.Sprintf("http://%s", req.Host)
 755 }
 756
 757 func (s *StandaloneSuite) TestPutProxy(c *C) {
 758         log.Printf("TestPutProxy")
 759
 760         st := StubProxyHandler{make(chan string, 1)}
 761
 762         arv, err := arvadosclient.MakeArvadosClient()
 763         kc, _ := MakeKeepClient(&arv)
 764
 765         kc.Want_replicas = 2
 766         kc.Using_proxy = true
 767         arv.ApiToken = "abc123"
 768         localRoots := make(map[string]string)
 769
 770         ks1 := RunSomeFakeKeepServers(st, 1)
 771
 772         for i, k := range ks1 {
 773                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 774                 defer k.listener.Close()
 775         }
 776
 777         kc.SetServiceRoots(localRoots, nil)
 778
 779         _, replicas, err := kc.PutB([]byte("foo"))
 780         <-st.handled
 781
 782         c.Check(err, Equals, nil)
 783         c.Check(replicas, Equals, 2)
 784
 785         log.Printf("TestPutProxy done")
 786 }
 787
 788 func (s *StandaloneSuite) TestPutProxyInsufficientReplicas(c *C) {
 789         log.Printf("TestPutProxy")
 790
 791         st := StubProxyHandler{make(chan string, 1)}
 792
 793         arv, err := arvadosclient.MakeArvadosClient()
 794         kc, _ := MakeKeepClient(&arv)
 795
 796         kc.Want_replicas = 3
 797         kc.Using_proxy = true
 798         arv.ApiToken = "abc123"
 799         localRoots := make(map[string]string)
 800
 801         ks1 := RunSomeFakeKeepServers(st, 1)
 802
 803         for i, k := range ks1 {
 804                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 805                 defer k.listener.Close()
 806         }
 807         kc.SetServiceRoots(localRoots, nil)
 808
 809         _, replicas, err := kc.PutB([]byte("foo"))
 810         <-st.handled
 811
 812         c.Check(err, Equals, InsufficientReplicasError)
 813         c.Check(replicas, Equals, 2)
 814
 815         log.Printf("TestPutProxy done")
 816 }
 817
 818 func (s *StandaloneSuite) TestMakeLocator(c *C) {
 819         l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce+3+Aabcde@12345678")
 820         c.Check(err, Equals, nil)
 821         c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
 822         c.Check(l.Size, Equals, 3)
 823         c.Check(l.Hints, DeepEquals, []string{"3", "Aabcde@12345678"})
 824 }
 825
 826 func (s *StandaloneSuite) TestMakeLocatorNoHints(c *C) {
 827         l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce")
 828         c.Check(err, Equals, nil)
 829         c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
 830         c.Check(l.Size, Equals, -1)
 831         c.Check(l.Hints, DeepEquals, []string{})
 832 }
 833
 834 func (s *StandaloneSuite) TestMakeLocatorNoSizeHint(c *C) {
 835         l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce+Aabcde@12345678")
 836         c.Check(err, Equals, nil)
 837         c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
 838         c.Check(l.Size, Equals, -1)
 839         c.Check(l.Hints, DeepEquals, []string{"Aabcde@12345678"})
 840 }
 841
 842 func (s *StandaloneSuite) TestMakeLocatorPreservesUnrecognizedHints(c *C) {
 843         str := "91f372a266fe2bf2823cb8ec7fda31ce+3+Unknown+Kzzzzz+Afoobar"
 844         l, err := MakeLocator(str)
 845         c.Check(err, Equals, nil)
 846         c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
 847         c.Check(l.Size, Equals, 3)
 848         c.Check(l.Hints, DeepEquals, []string{"3", "Unknown", "Kzzzzz", "Afoobar"})
 849         c.Check(l.String(), Equals, str)
 850 }
 851
 852 func (s *StandaloneSuite) TestMakeLocatorInvalidInput(c *C) {
 853         _, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31c")
 854         c.Check(err, Equals, InvalidLocatorError)
 855 }