sdk/go/keepclient/keepclient_test.go

   1 package keepclient
   2
   3 import (
   4         "crypto/md5"
   5         "flag"
   6         "fmt"
   7         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
   8         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
   9         "git.curoverse.com/arvados.git/sdk/go/streamer"
  10         . "gopkg.in/check.v1"
  11         "io"
  12         "io/ioutil"
  13         "log"
  14         "net"
  15         "net/http"
  16         "os"
  17         "testing"
  18 )
  19
  20 // Gocheck boilerplate
  21 func Test(t *testing.T) {
  22         TestingT(t)
  23 }
  24
  25 // Gocheck boilerplate
  26 var _ = Suite(&ServerRequiredSuite{})
  27 var _ = Suite(&StandaloneSuite{})
  28
  29 var no_server = flag.Bool("no-server", false, "Skip 'ServerRequireSuite'")
  30
  31 // Tests that require the Keep server running
  32 type ServerRequiredSuite struct{}
  33
  34 // Standalone tests
  35 type StandaloneSuite struct{}
  36
  37 func pythonDir() string {
  38         cwd, _ := os.Getwd()
  39         return fmt.Sprintf("%s/../../python/tests", cwd)
  40 }
  41
  42 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
  43         if *no_server {
  44                 c.Skip("Skipping tests that require server")
  45                 return
  46         }
  47         arvadostest.StartAPI()
  48         arvadostest.StartKeep()
  49 }
  50
  51 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
  52         if *no_server {
  53                 return
  54         }
  55         arvadostest.StopKeep()
  56         arvadostest.StopAPI()
  57 }
  58
  59 func (s *ServerRequiredSuite) TestMakeKeepClient(c *C) {
  60         arv, err := arvadosclient.MakeArvadosClient()
  61         c.Assert(err, Equals, nil)
  62
  63         kc, err := MakeKeepClient(&arv)
  64
  65         c.Assert(err, Equals, nil)
  66         c.Check(len(kc.LocalRoots()), Equals, 2)
  67         for _, root := range kc.LocalRoots() {
  68                 c.Check(root, Matches, "http://localhost:\\d+")
  69         }
  70 }
  71
  72 type StubPutHandler struct {
  73         c              *C
  74         expectPath     string
  75         expectApiToken string
  76         expectBody     string
  77         handled        chan string
  78 }
  79
  80 func (sph StubPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
  81         sph.c.Check(req.URL.Path, Equals, "/"+sph.expectPath)
  82         sph.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sph.expectApiToken))
  83         body, err := ioutil.ReadAll(req.Body)
  84         sph.c.Check(err, Equals, nil)
  85         sph.c.Check(body, DeepEquals, []byte(sph.expectBody))
  86         resp.WriteHeader(200)
  87         sph.handled <- fmt.Sprintf("http://%s", req.Host)
  88 }
  89
  90 func RunFakeKeepServer(st http.Handler) (ks KeepServer) {
  91         var err error
  92         ks.listener, err = net.ListenTCP("tcp", &net.TCPAddr{Port: 0})
  93         if err != nil {
  94                 panic(fmt.Sprintf("Could not listen on any port"))
  95         }
  96         ks.url = fmt.Sprintf("http://%s", ks.listener.Addr().String())
  97         go http.Serve(ks.listener, st)
  98         return
  99 }
 100
 101 func UploadToStubHelper(c *C, st http.Handler, f func(*KeepClient, string,
 102         io.ReadCloser, io.WriteCloser, chan uploadStatus)) {
 103
 104         ks := RunFakeKeepServer(st)
 105         defer ks.listener.Close()
 106
 107         arv, _ := arvadosclient.MakeArvadosClient()
 108         arv.ApiToken = "abc123"
 109
 110         kc, _ := MakeKeepClient(&arv)
 111
 112         reader, writer := io.Pipe()
 113         upload_status := make(chan uploadStatus)
 114
 115         f(kc, ks.url, reader, writer, upload_status)
 116 }
 117
 118 func (s *StandaloneSuite) TestUploadToStubKeepServer(c *C) {
 119         log.Printf("TestUploadToStubKeepServer")
 120
 121         st := StubPutHandler{
 122                 c,
 123                 "acbd18db4cc2f85cedef654fccc4a4d8",
 124                 "abc123",
 125                 "foo",
 126                 make(chan string)}
 127
 128         UploadToStubHelper(c, st,
 129                 func(kc *KeepClient, url string, reader io.ReadCloser,
 130                         writer io.WriteCloser, upload_status chan uploadStatus) {
 131
 132                         go kc.uploadToKeepServer(url, st.expectPath, reader, upload_status, int64(len("foo")), "TestUploadToStubKeepServer")
 133
 134                         writer.Write([]byte("foo"))
 135                         writer.Close()
 136
 137                         <-st.handled
 138                         status := <-upload_status
 139                         c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, ""})
 140                 })
 141
 142         log.Printf("TestUploadToStubKeepServer done")
 143 }
 144
 145 func (s *StandaloneSuite) TestUploadToStubKeepServerBufferReader(c *C) {
 146         log.Printf("TestUploadToStubKeepServerBufferReader")
 147
 148         st := StubPutHandler{
 149                 c,
 150                 "acbd18db4cc2f85cedef654fccc4a4d8",
 151                 "abc123",
 152                 "foo",
 153                 make(chan string)}
 154
 155         UploadToStubHelper(c, st,
 156                 func(kc *KeepClient, url string, reader io.ReadCloser,
 157                         writer io.WriteCloser, upload_status chan uploadStatus) {
 158
 159                         tr := streamer.AsyncStreamFromReader(512, reader)
 160                         defer tr.Close()
 161
 162                         br1 := tr.MakeStreamReader()
 163
 164                         go kc.uploadToKeepServer(url, st.expectPath, br1, upload_status, 3, "TestUploadToStubKeepServerBufferReader")
 165
 166                         writer.Write([]byte("foo"))
 167                         writer.Close()
 168
 169                         <-st.handled
 170
 171                         status := <-upload_status
 172                         c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, ""})
 173                 })
 174
 175         log.Printf("TestUploadToStubKeepServerBufferReader done")
 176 }
 177
 178 type FailHandler struct {
 179         handled chan string
 180 }
 181
 182 func (fh FailHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 183         resp.WriteHeader(500)
 184         fh.handled <- fmt.Sprintf("http://%s", req.Host)
 185 }
 186
 187 func (s *StandaloneSuite) TestFailedUploadToStubKeepServer(c *C) {
 188         log.Printf("TestFailedUploadToStubKeepServer")
 189
 190         st := FailHandler{
 191                 make(chan string)}
 192
 193         hash := "acbd18db4cc2f85cedef654fccc4a4d8"
 194
 195         UploadToStubHelper(c, st,
 196                 func(kc *KeepClient, url string, reader io.ReadCloser,
 197                         writer io.WriteCloser, upload_status chan uploadStatus) {
 198
 199                         go kc.uploadToKeepServer(url, hash, reader, upload_status, 3, "TestFailedUploadToStubKeepServer")
 200
 201                         writer.Write([]byte("foo"))
 202                         writer.Close()
 203
 204                         <-st.handled
 205
 206                         status := <-upload_status
 207                         c.Check(status.url, Equals, fmt.Sprintf("%s/%s", url, hash))
 208                         c.Check(status.statusCode, Equals, 500)
 209                 })
 210         log.Printf("TestFailedUploadToStubKeepServer done")
 211 }
 212
 213 type KeepServer struct {
 214         listener net.Listener
 215         url      string
 216 }
 217
 218 func RunSomeFakeKeepServers(st http.Handler, n int) (ks []KeepServer) {
 219         ks = make([]KeepServer, n)
 220
 221         for i := 0; i < n; i += 1 {
 222                 ks[i] = RunFakeKeepServer(st)
 223         }
 224
 225         return ks
 226 }
 227
 228 func (s *StandaloneSuite) TestPutB(c *C) {
 229         log.Printf("TestPutB")
 230
 231         hash := Md5String("foo")
 232
 233         st := StubPutHandler{
 234                 c,
 235                 hash,
 236                 "abc123",
 237                 "foo",
 238                 make(chan string, 5)}
 239
 240         arv, _ := arvadosclient.MakeArvadosClient()
 241         kc, _ := MakeKeepClient(&arv)
 242
 243         kc.Want_replicas = 2
 244         arv.ApiToken = "abc123"
 245         localRoots := make(map[string]string)
 246         writableRoots := make(map[string]string)
 247
 248         ks := RunSomeFakeKeepServers(st, 5)
 249
 250         for i, k := range ks {
 251                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 252                 writableRoots[k.url] = ""
 253                 defer k.listener.Close()
 254         }
 255
 256         kc.SetServiceRoots(localRoots, nil, writableRoots)
 257
 258         kc.PutB([]byte("foo"))
 259
 260         shuff := NewRootSorter(
 261                 kc.LocalRoots(), Md5String("foo")).GetSortedRoots()
 262
 263         s1 := <-st.handled
 264         s2 := <-st.handled
 265         c.Check((s1 == shuff[0] && s2 == shuff[1]) ||
 266                 (s1 == shuff[1] && s2 == shuff[0]),
 267                 Equals,
 268                 true)
 269
 270         log.Printf("TestPutB done")
 271 }
 272
 273 func (s *StandaloneSuite) TestPutHR(c *C) {
 274         log.Printf("TestPutHR")
 275
 276         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 277
 278         st := StubPutHandler{
 279                 c,
 280                 hash,
 281                 "abc123",
 282                 "foo",
 283                 make(chan string, 5)}
 284
 285         arv, _ := arvadosclient.MakeArvadosClient()
 286         kc, _ := MakeKeepClient(&arv)
 287
 288         kc.Want_replicas = 2
 289         arv.ApiToken = "abc123"
 290         localRoots := make(map[string]string)
 291         writableRoots := make(map[string]string)
 292
 293         ks := RunSomeFakeKeepServers(st, 5)
 294
 295         for i, k := range ks {
 296                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 297                 writableRoots[k.url] = ""
 298                 defer k.listener.Close()
 299         }
 300
 301         kc.SetServiceRoots(localRoots, nil, writableRoots)
 302
 303         reader, writer := io.Pipe()
 304
 305         go func() {
 306                 writer.Write([]byte("foo"))
 307                 writer.Close()
 308         }()
 309
 310         kc.PutHR(hash, reader, 3)
 311
 312         shuff := NewRootSorter(kc.LocalRoots(), hash).GetSortedRoots()
 313         log.Print(shuff)
 314
 315         s1 := <-st.handled
 316         s2 := <-st.handled
 317
 318         c.Check((s1 == shuff[0] && s2 == shuff[1]) ||
 319                 (s1 == shuff[1] && s2 == shuff[0]),
 320                 Equals,
 321                 true)
 322
 323         log.Printf("TestPutHR done")
 324 }
 325
 326 func (s *StandaloneSuite) TestPutWithFail(c *C) {
 327         log.Printf("TestPutWithFail")
 328
 329         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 330
 331         st := StubPutHandler{
 332                 c,
 333                 hash,
 334                 "abc123",
 335                 "foo",
 336                 make(chan string, 4)}
 337
 338         fh := FailHandler{
 339                 make(chan string, 1)}
 340
 341         arv, err := arvadosclient.MakeArvadosClient()
 342         kc, _ := MakeKeepClient(&arv)
 343
 344         kc.Want_replicas = 2
 345         arv.ApiToken = "abc123"
 346         localRoots := make(map[string]string)
 347         writableRoots := make(map[string]string)
 348
 349         ks1 := RunSomeFakeKeepServers(st, 4)
 350         ks2 := RunSomeFakeKeepServers(fh, 1)
 351
 352         for i, k := range ks1 {
 353                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 354                 writableRoots[k.url] = ""
 355                 defer k.listener.Close()
 356         }
 357         for i, k := range ks2 {
 358                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
 359                 writableRoots[k.url] = ""
 360                 defer k.listener.Close()
 361         }
 362
 363         kc.SetServiceRoots(localRoots, nil, writableRoots)
 364
 365         shuff := NewRootSorter(
 366                 kc.LocalRoots(), Md5String("foo")).GetSortedRoots()
 367
 368         phash, replicas, err := kc.PutB([]byte("foo"))
 369
 370         <-fh.handled
 371
 372         c.Check(err, Equals, nil)
 373         c.Check(phash, Equals, "")
 374         c.Check(replicas, Equals, 2)
 375
 376         s1 := <-st.handled
 377         s2 := <-st.handled
 378
 379         c.Check((s1 == shuff[1] && s2 == shuff[2]) ||
 380                 (s1 == shuff[2] && s2 == shuff[1]),
 381                 Equals,
 382                 true)
 383 }
 384
 385 func (s *StandaloneSuite) TestPutWithTooManyFail(c *C) {
 386         log.Printf("TestPutWithTooManyFail")
 387
 388         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 389
 390         st := StubPutHandler{
 391                 c,
 392                 hash,
 393                 "abc123",
 394                 "foo",
 395                 make(chan string, 1)}
 396
 397         fh := FailHandler{
 398                 make(chan string, 4)}
 399
 400         arv, err := arvadosclient.MakeArvadosClient()
 401         kc, _ := MakeKeepClient(&arv)
 402
 403         kc.Want_replicas = 2
 404         arv.ApiToken = "abc123"
 405         localRoots := make(map[string]string)
 406         writableRoots := make(map[string]string)
 407
 408         ks1 := RunSomeFakeKeepServers(st, 1)
 409         ks2 := RunSomeFakeKeepServers(fh, 4)
 410
 411         for i, k := range ks1 {
 412                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 413                 writableRoots[k.url] = ""
 414                 defer k.listener.Close()
 415         }
 416         for i, k := range ks2 {
 417                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
 418                 writableRoots[k.url] = ""
 419                 defer k.listener.Close()
 420         }
 421
 422         kc.SetServiceRoots(localRoots, nil, writableRoots)
 423
 424         _, replicas, err := kc.PutB([]byte("foo"))
 425
 426         c.Check(err, Equals, InsufficientReplicasError)
 427         c.Check(replicas, Equals, 1)
 428         c.Check(<-st.handled, Equals, ks1[0].url)
 429
 430         log.Printf("TestPutWithTooManyFail done")
 431 }
 432
 433 type StubGetHandler struct {
 434         c              *C
 435         expectPath     string
 436         expectApiToken string
 437         httpStatus     int
 438         body           []byte
 439 }
 440
 441 func (sgh StubGetHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 442         sgh.c.Check(req.URL.Path, Equals, "/"+sgh.expectPath)
 443         sgh.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sgh.expectApiToken))
 444         resp.WriteHeader(sgh.httpStatus)
 445         resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(sgh.body)))
 446         resp.Write(sgh.body)
 447 }
 448
 449 func (s *StandaloneSuite) TestGet(c *C) {
 450         log.Printf("TestGet")
 451
 452         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 453
 454         st := StubGetHandler{
 455                 c,
 456                 hash,
 457                 "abc123",
 458                 http.StatusOK,
 459                 []byte("foo")}
 460
 461         ks := RunFakeKeepServer(st)
 462         defer ks.listener.Close()
 463
 464         arv, err := arvadosclient.MakeArvadosClient()
 465         kc, _ := MakeKeepClient(&arv)
 466         arv.ApiToken = "abc123"
 467         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, map[string]string{ks.url: ""})
 468
 469         r, n, url2, err := kc.Get(hash)
 470         defer r.Close()
 471         c.Check(err, Equals, nil)
 472         c.Check(n, Equals, int64(3))
 473         c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks.url, hash))
 474
 475         content, err2 := ioutil.ReadAll(r)
 476         c.Check(err2, Equals, nil)
 477         c.Check(content, DeepEquals, []byte("foo"))
 478
 479         log.Printf("TestGet done")
 480 }
 481
 482 func (s *StandaloneSuite) TestGetFail(c *C) {
 483         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 484
 485         st := FailHandler{make(chan string, 1)}
 486
 487         ks := RunFakeKeepServer(st)
 488         defer ks.listener.Close()
 489
 490         arv, err := arvadosclient.MakeArvadosClient()
 491         kc, _ := MakeKeepClient(&arv)
 492         arv.ApiToken = "abc123"
 493         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, map[string]string{ks.url: ""})
 494
 495         r, n, url2, err := kc.Get(hash)
 496         c.Check(err, Equals, BlockNotFound)
 497         c.Check(n, Equals, int64(0))
 498         c.Check(url2, Equals, "")
 499         c.Check(r, Equals, nil)
 500 }
 501
 502 func (s *StandaloneSuite) TestGetWithServiceHint(c *C) {
 503         uuid := "zzzzz-bi6l4-123451234512345"
 504         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 505
 506         // This one shouldn't be used:
 507         ks0 := RunFakeKeepServer(StubGetHandler{
 508                 c,
 509                 "error if used",
 510                 "abc123",
 511                 http.StatusOK,
 512                 []byte("foo")})
 513         defer ks0.listener.Close()
 514         // This one should be used:
 515         ks := RunFakeKeepServer(StubGetHandler{
 516                 c,
 517                 hash + "+K@" + uuid,
 518                 "abc123",
 519                 http.StatusOK,
 520                 []byte("foo")})
 521         defer ks.listener.Close()
 522
 523         arv, err := arvadosclient.MakeArvadosClient()
 524         kc, _ := MakeKeepClient(&arv)
 525         arv.ApiToken = "abc123"
 526         kc.SetServiceRoots(
 527                 map[string]string{"x": ks0.url},
 528                 map[string]string{uuid: ks.url},
 529                 map[string]string{ks0.url: "", ks.url: ""})
 530
 531         r, n, uri, err := kc.Get(hash + "+K@" + uuid)
 532         defer r.Close()
 533         c.Check(err, Equals, nil)
 534         c.Check(n, Equals, int64(3))
 535         c.Check(uri, Equals, fmt.Sprintf("%s/%s", ks.url, hash+"+K@"+uuid))
 536
 537         content, err := ioutil.ReadAll(r)
 538         c.Check(err, Equals, nil)
 539         c.Check(content, DeepEquals, []byte("foo"))
 540 }
 541
 542 // Use a service hint to fetch from a local disk service, overriding
 543 // rendezvous probe order.
 544 func (s *StandaloneSuite) TestGetWithLocalServiceHint(c *C) {
 545         uuid := "zzzzz-bi6l4-zzzzzzzzzzzzzzz"
 546         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 547
 548         // This one shouldn't be used, although it appears first in
 549         // rendezvous probe order:
 550         ks0 := RunFakeKeepServer(StubGetHandler{
 551                 c,
 552                 "error if used",
 553                 "abc123",
 554                 http.StatusOK,
 555                 []byte("foo")})
 556         defer ks0.listener.Close()
 557         // This one should be used:
 558         ks := RunFakeKeepServer(StubGetHandler{
 559                 c,
 560                 hash + "+K@" + uuid,
 561                 "abc123",
 562                 http.StatusOK,
 563                 []byte("foo")})
 564         defer ks.listener.Close()
 565
 566         arv, err := arvadosclient.MakeArvadosClient()
 567         kc, _ := MakeKeepClient(&arv)
 568         arv.ApiToken = "abc123"
 569         kc.SetServiceRoots(
 570                 map[string]string{
 571                         "zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url,
 572                         "zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url,
 573                         "zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url,
 574                         uuid: ks.url},
 575                 map[string]string{
 576                         "zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url,
 577                         "zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url,
 578                         "zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url,
 579                         uuid: ks.url},
 580                 map[string]string{ks.url: ""},
 581         )
 582
 583         r, n, uri, err := kc.Get(hash + "+K@" + uuid)
 584         defer r.Close()
 585         c.Check(err, Equals, nil)
 586         c.Check(n, Equals, int64(3))
 587         c.Check(uri, Equals, fmt.Sprintf("%s/%s", ks.url, hash+"+K@"+uuid))
 588
 589         content, err := ioutil.ReadAll(r)
 590         c.Check(err, Equals, nil)
 591         c.Check(content, DeepEquals, []byte("foo"))
 592 }
 593
 594 func (s *StandaloneSuite) TestGetWithServiceHintFailoverToLocals(c *C) {
 595         uuid := "zzzzz-bi6l4-123451234512345"
 596         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 597
 598         ksLocal := RunFakeKeepServer(StubGetHandler{
 599                 c,
 600                 hash + "+K@" + uuid,
 601                 "abc123",
 602                 http.StatusOK,
 603                 []byte("foo")})
 604         defer ksLocal.listener.Close()
 605         ksGateway := RunFakeKeepServer(StubGetHandler{
 606                 c,
 607                 hash + "+K@" + uuid,
 608                 "abc123",
 609                 http.StatusInternalServerError,
 610                 []byte("Error")})
 611         defer ksGateway.listener.Close()
 612
 613         arv, err := arvadosclient.MakeArvadosClient()
 614         kc, _ := MakeKeepClient(&arv)
 615         arv.ApiToken = "abc123"
 616         kc.SetServiceRoots(
 617                 map[string]string{"zzzzz-bi6l4-keepdisk0000000": ksLocal.url},
 618                 map[string]string{uuid: ksGateway.url},
 619                 map[string]string{ksLocal.url: "", ksGateway.url: ""})
 620
 621         r, n, uri, err := kc.Get(hash + "+K@" + uuid)
 622         c.Assert(err, Equals, nil)
 623         defer r.Close()
 624         c.Check(n, Equals, int64(3))
 625         c.Check(uri, Equals, fmt.Sprintf("%s/%s", ksLocal.url, hash+"+K@"+uuid))
 626
 627         content, err := ioutil.ReadAll(r)
 628         c.Check(err, Equals, nil)
 629         c.Check(content, DeepEquals, []byte("foo"))
 630 }
 631
 632 type BarHandler struct {
 633         handled chan string
 634 }
 635
 636 func (this BarHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 637         resp.Write([]byte("bar"))
 638         this.handled <- fmt.Sprintf("http://%s", req.Host)
 639 }
 640
 641 func (s *StandaloneSuite) TestChecksum(c *C) {
 642         foohash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 643         barhash := fmt.Sprintf("%x", md5.Sum([]byte("bar")))
 644
 645         st := BarHandler{make(chan string, 1)}
 646
 647         ks := RunFakeKeepServer(st)
 648         defer ks.listener.Close()
 649
 650         arv, err := arvadosclient.MakeArvadosClient()
 651         kc, _ := MakeKeepClient(&arv)
 652         arv.ApiToken = "abc123"
 653         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, map[string]string{ks.url: ""})
 654
 655         r, n, _, err := kc.Get(barhash)
 656         _, err = ioutil.ReadAll(r)
 657         c.Check(n, Equals, int64(3))
 658         c.Check(err, Equals, nil)
 659
 660         <-st.handled
 661
 662         r, n, _, err = kc.Get(foohash)
 663         _, err = ioutil.ReadAll(r)
 664         c.Check(n, Equals, int64(3))
 665         c.Check(err, Equals, BadChecksum)
 666
 667         <-st.handled
 668 }
 669
 670 func (s *StandaloneSuite) TestGetWithFailures(c *C) {
 671         content := []byte("waz")
 672         hash := fmt.Sprintf("%x", md5.Sum(content))
 673
 674         fh := FailHandler{
 675                 make(chan string, 4)}
 676
 677         st := StubGetHandler{
 678                 c,
 679                 hash,
 680                 "abc123",
 681                 http.StatusOK,
 682                 content}
 683
 684         arv, err := arvadosclient.MakeArvadosClient()
 685         kc, _ := MakeKeepClient(&arv)
 686         arv.ApiToken = "abc123"
 687         localRoots := make(map[string]string)
 688         writableRoots := make(map[string]string)
 689
 690         ks1 := RunSomeFakeKeepServers(st, 1)
 691         ks2 := RunSomeFakeKeepServers(fh, 4)
 692
 693         for i, k := range ks1 {
 694                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 695                 writableRoots[k.url] = ""
 696                 defer k.listener.Close()
 697         }
 698         for i, k := range ks2 {
 699                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
 700                 writableRoots[k.url] = ""
 701                 defer k.listener.Close()
 702         }
 703
 704         kc.SetServiceRoots(localRoots, nil, writableRoots)
 705
 706         // This test works only if one of the failing services is
 707         // attempted before the succeeding service. Otherwise,
 708         // <-fh.handled below will just hang! (Probe order depends on
 709         // the choice of block content "waz" and the UUIDs of the fake
 710         // servers, so we just tried different strings until we found
 711         // an example that passes this Assert.)
 712         c.Assert(NewRootSorter(localRoots, hash).GetSortedRoots()[0], Not(Equals), ks1[0].url)
 713
 714         r, n, url2, err := kc.Get(hash)
 715
 716         <-fh.handled
 717         c.Check(err, Equals, nil)
 718         c.Check(n, Equals, int64(3))
 719         c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks1[0].url, hash))
 720
 721         read_content, err2 := ioutil.ReadAll(r)
 722         c.Check(err2, Equals, nil)
 723         c.Check(read_content, DeepEquals, content)
 724 }
 725
 726 func (s *ServerRequiredSuite) TestPutGetHead(c *C) {
 727         content := []byte("TestPutGetHead")
 728
 729         arv, err := arvadosclient.MakeArvadosClient()
 730         kc, err := MakeKeepClient(&arv)
 731         c.Assert(err, Equals, nil)
 732
 733         hash := fmt.Sprintf("%x", md5.Sum(content))
 734
 735         {
 736                 n, _, err := kc.Ask(hash)
 737                 c.Check(err, Equals, BlockNotFound)
 738                 c.Check(n, Equals, int64(0))
 739         }
 740         {
 741                 hash2, replicas, err := kc.PutB(content)
 742                 c.Check(hash2, Equals, fmt.Sprintf("%s+%d", hash, len(content)))
 743                 c.Check(replicas, Equals, 2)
 744                 c.Check(err, Equals, nil)
 745         }
 746         {
 747                 r, n, url2, err := kc.Get(hash)
 748                 c.Check(err, Equals, nil)
 749                 c.Check(n, Equals, int64(len(content)))
 750                 c.Check(url2, Matches, fmt.Sprintf("http://localhost:\\d+/%s", hash))
 751
 752                 read_content, err2 := ioutil.ReadAll(r)
 753                 c.Check(err2, Equals, nil)
 754                 c.Check(read_content, DeepEquals, content)
 755         }
 756         {
 757                 n, url2, err := kc.Ask(hash)
 758                 c.Check(err, Equals, nil)
 759                 c.Check(n, Equals, int64(len(content)))
 760                 c.Check(url2, Matches, fmt.Sprintf("http://localhost:\\d+/%s", hash))
 761         }
 762 }
 763
 764 type StubProxyHandler struct {
 765         handled chan string
 766 }
 767
 768 func (this StubProxyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 769         resp.Header().Set("X-Keep-Replicas-Stored", "2")
 770         this.handled <- fmt.Sprintf("http://%s", req.Host)
 771 }
 772
 773 func (s *StandaloneSuite) TestPutProxy(c *C) {
 774         log.Printf("TestPutProxy")
 775
 776         st := StubProxyHandler{make(chan string, 1)}
 777
 778         arv, err := arvadosclient.MakeArvadosClient()
 779         kc, _ := MakeKeepClient(&arv)
 780
 781         kc.Want_replicas = 2
 782         kc.Using_proxy = true
 783         arv.ApiToken = "abc123"
 784         localRoots := make(map[string]string)
 785         writableRoots := make(map[string]string)
 786
 787         ks1 := RunSomeFakeKeepServers(st, 1)
 788
 789         for i, k := range ks1 {
 790                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 791                 writableRoots[k.url] = ""
 792                 defer k.listener.Close()
 793         }
 794
 795         kc.SetServiceRoots(localRoots, nil, writableRoots)
 796
 797         _, replicas, err := kc.PutB([]byte("foo"))
 798         <-st.handled
 799
 800         c.Check(err, Equals, nil)
 801         c.Check(replicas, Equals, 2)
 802
 803         log.Printf("TestPutProxy done")
 804 }
 805
 806 func (s *StandaloneSuite) TestPutProxyInsufficientReplicas(c *C) {
 807         log.Printf("TestPutProxy")
 808
 809         st := StubProxyHandler{make(chan string, 1)}
 810
 811         arv, err := arvadosclient.MakeArvadosClient()
 812         kc, _ := MakeKeepClient(&arv)
 813
 814         kc.Want_replicas = 3
 815         kc.Using_proxy = true
 816         arv.ApiToken = "abc123"
 817         localRoots := make(map[string]string)
 818         writableRoots := make(map[string]string)
 819
 820         ks1 := RunSomeFakeKeepServers(st, 1)
 821
 822         for i, k := range ks1 {
 823                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 824                 writableRoots[k.url] = ""
 825                 defer k.listener.Close()
 826         }
 827         kc.SetServiceRoots(localRoots, nil, writableRoots)
 828
 829         _, replicas, err := kc.PutB([]byte("foo"))
 830         <-st.handled
 831
 832         c.Check(err, Equals, InsufficientReplicasError)
 833         c.Check(replicas, Equals, 2)
 834
 835         log.Printf("TestPutProxy done")
 836 }
 837
 838 func (s *StandaloneSuite) TestMakeLocator(c *C) {
 839         l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce+3+Aabcde@12345678")
 840         c.Check(err, Equals, nil)
 841         c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
 842         c.Check(l.Size, Equals, 3)
 843         c.Check(l.Hints, DeepEquals, []string{"3", "Aabcde@12345678"})
 844 }
 845
 846 func (s *StandaloneSuite) TestMakeLocatorNoHints(c *C) {
 847         l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce")
 848         c.Check(err, Equals, nil)
 849         c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
 850         c.Check(l.Size, Equals, -1)
 851         c.Check(l.Hints, DeepEquals, []string{})
 852 }
 853
 854 func (s *StandaloneSuite) TestMakeLocatorNoSizeHint(c *C) {
 855         l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce+Aabcde@12345678")
 856         c.Check(err, Equals, nil)
 857         c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
 858         c.Check(l.Size, Equals, -1)
 859         c.Check(l.Hints, DeepEquals, []string{"Aabcde@12345678"})
 860 }
 861
 862 func (s *StandaloneSuite) TestMakeLocatorPreservesUnrecognizedHints(c *C) {
 863         str := "91f372a266fe2bf2823cb8ec7fda31ce+3+Unknown+Kzzzzz+Afoobar"
 864         l, err := MakeLocator(str)
 865         c.Check(err, Equals, nil)
 866         c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
 867         c.Check(l.Size, Equals, 3)
 868         c.Check(l.Hints, DeepEquals, []string{"3", "Unknown", "Kzzzzz", "Afoobar"})
 869         c.Check(l.String(), Equals, str)
 870 }
 871
 872 func (s *StandaloneSuite) TestMakeLocatorInvalidInput(c *C) {
 873         _, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31c")
 874         c.Check(err, Equals, InvalidLocatorError)
 875 }
 876
 877 func (s *StandaloneSuite) TestPutBWant2ReplicasWithOnlyOneWritableRoots(c *C) {
 878         log.Printf("TestPutWant2ReplicasWithOnlyOneWritableRoots")
 879
 880         hash := Md5String("foo")
 881
 882         st := StubPutHandler{
 883                 c,
 884                 hash,
 885                 "abc123",
 886                 "foo",
 887                 make(chan string, 5)}
 888
 889         arv, _ := arvadosclient.MakeArvadosClient()
 890         kc, _ := MakeKeepClient(&arv)
 891
 892         kc.Want_replicas = 2
 893         arv.ApiToken = "abc123"
 894         localRoots := make(map[string]string)
 895         writableRoots := make(map[string]string)
 896
 897         ks := RunSomeFakeKeepServers(st, 5)
 898
 899         for i, k := range ks {
 900                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 901                 if i == 0 {
 902                         writableRoots[k.url] = ""
 903                 }
 904                 defer k.listener.Close()
 905         }
 906
 907         kc.SetServiceRoots(localRoots, nil, writableRoots)
 908
 909         _, replicas, err := kc.PutB([]byte("foo"))
 910
 911         c.Check(err, Equals, InsufficientReplicasError)
 912         c.Check(replicas, Equals, 1)
 913
 914         c.Check(<-st.handled, Equals, localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", 0)])
 915
 916         log.Printf("TestPutWant2ReplicasWithOnlyOneWritableRoots done")
 917 }
 918
 919 func (s *StandaloneSuite) TestPutBWithNoWritableRoots(c *C) {
 920         log.Printf("TestPutBWithNoWritableRoots")
 921
 922         hash := Md5String("foo")
 923
 924         st := StubPutHandler{
 925                 c,
 926                 hash,
 927                 "abc123",
 928                 "foo",
 929                 make(chan string, 5)}
 930
 931         arv, _ := arvadosclient.MakeArvadosClient()
 932         kc, _ := MakeKeepClient(&arv)
 933
 934         kc.Want_replicas = 2
 935         arv.ApiToken = "abc123"
 936         localRoots := make(map[string]string)
 937         writableRoots := make(map[string]string)
 938
 939         ks := RunSomeFakeKeepServers(st, 5)
 940
 941         for i, k := range ks {
 942                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 943                 defer k.listener.Close()
 944         }
 945
 946         kc.SetServiceRoots(localRoots, nil, writableRoots)
 947
 948         _, replicas, err := kc.PutB([]byte("foo"))
 949
 950         c.Check(err, Equals, InsufficientReplicasError)
 951         c.Check(replicas, Equals, 0)
 952
 953         log.Printf("TestPutBWithNoWritableRoots done")
 954 }