sdk/go/keepclient/keepclient_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package keepclient
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "errors"
  11         "fmt"
  12         "io"
  13         "io/ioutil"
  14         "log"
  15         "net"
  16         "net/http"
  17         "os"
  18         "strings"
  19         "testing"
  20         "time"
  21
  22         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  23         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  24         . "gopkg.in/check.v1"
  25 )
  26
  27 // Gocheck boilerplate
  28 func Test(t *testing.T) {
  29         TestingT(t)
  30 }
  31
  32 // Gocheck boilerplate
  33 var _ = Suite(&ServerRequiredSuite{})
  34 var _ = Suite(&StandaloneSuite{})
  35
  36 // Tests that require the Keep server running
  37 type ServerRequiredSuite struct{}
  38
  39 // Standalone tests
  40 type StandaloneSuite struct{}
  41
  42 func (s *StandaloneSuite) SetUpTest(c *C) {
  43         RefreshServiceDiscovery()
  44 }
  45
  46 func pythonDir() string {
  47         cwd, _ := os.Getwd()
  48         return fmt.Sprintf("%s/../../python/tests", cwd)
  49 }
  50
  51 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
  52         arvadostest.StartAPI()
  53         arvadostest.StartKeep(2, false)
  54 }
  55
  56 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
  57         arvadostest.StopKeep(2)
  58         arvadostest.StopAPI()
  59 }
  60
  61 func (s *ServerRequiredSuite) SetUpTest(c *C) {
  62         RefreshServiceDiscovery()
  63 }
  64
  65 func (s *ServerRequiredSuite) TestMakeKeepClient(c *C) {
  66         arv, err := arvadosclient.MakeArvadosClient()
  67         c.Assert(err, Equals, nil)
  68
  69         kc, err := MakeKeepClient(arv)
  70
  71         c.Assert(err, Equals, nil)
  72         c.Check(len(kc.LocalRoots()), Equals, 2)
  73         for _, root := range kc.LocalRoots() {
  74                 c.Check(root, Matches, "http://localhost:\\d+")
  75         }
  76 }
  77
  78 func (s *ServerRequiredSuite) TestDefaultReplications(c *C) {
  79         arv, err := arvadosclient.MakeArvadosClient()
  80         c.Assert(err, Equals, nil)
  81
  82         kc, err := MakeKeepClient(arv)
  83         c.Check(err, IsNil)
  84         c.Assert(kc.Want_replicas, Equals, 2)
  85
  86         arv.DiscoveryDoc["defaultCollectionReplication"] = 3.0
  87         kc, err = MakeKeepClient(arv)
  88         c.Check(err, IsNil)
  89         c.Assert(kc.Want_replicas, Equals, 3)
  90
  91         arv.DiscoveryDoc["defaultCollectionReplication"] = 1.0
  92         kc, err = MakeKeepClient(arv)
  93         c.Check(err, IsNil)
  94         c.Assert(kc.Want_replicas, Equals, 1)
  95 }
  96
  97 type StubPutHandler struct {
  98         c                    *C
  99         expectPath           string
 100         expectAPIToken       string
 101         expectBody           string
 102         expectStorageClass   string
 103         returnStorageClasses string
 104         handled              chan string
 105 }
 106
 107 func (sph StubPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 108         sph.c.Check(req.URL.Path, Equals, "/"+sph.expectPath)
 109         sph.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sph.expectAPIToken))
 110         sph.c.Check(req.Header.Get("X-Keep-Storage-Classes"), Equals, sph.expectStorageClass)
 111         body, err := ioutil.ReadAll(req.Body)
 112         sph.c.Check(err, Equals, nil)
 113         sph.c.Check(body, DeepEquals, []byte(sph.expectBody))
 114         resp.Header().Set("X-Keep-Replicas-Stored", "1")
 115         if sph.returnStorageClasses != "" {
 116                 resp.Header().Set("X-Keep-Storage-Classes-Confirmed", sph.returnStorageClasses)
 117         }
 118         resp.WriteHeader(200)
 119         sph.handled <- fmt.Sprintf("http://%s", req.Host)
 120 }
 121
 122 func RunFakeKeepServer(st http.Handler) (ks KeepServer) {
 123         var err error
 124         // If we don't explicitly bind it to localhost, ks.listener.Addr() will
 125         // bind to 0.0.0.0 or [::] which is not a valid address for Dial()
 126         ks.listener, err = net.ListenTCP("tcp", &net.TCPAddr{IP: []byte{127, 0, 0, 1}, Port: 0})
 127         if err != nil {
 128                 panic(fmt.Sprintf("Could not listen on any port"))
 129         }
 130         ks.url = fmt.Sprintf("http://%s", ks.listener.Addr().String())
 131         go http.Serve(ks.listener, st)
 132         return
 133 }
 134
 135 func UploadToStubHelper(c *C, st http.Handler, f func(*KeepClient, string,
 136         io.ReadCloser, io.WriteCloser, chan uploadStatus)) {
 137
 138         ks := RunFakeKeepServer(st)
 139         defer ks.listener.Close()
 140
 141         arv, _ := arvadosclient.MakeArvadosClient()
 142         arv.ApiToken = "abc123"
 143
 144         kc, _ := MakeKeepClient(arv)
 145
 146         reader, writer := io.Pipe()
 147         uploadStatusChan := make(chan uploadStatus)
 148
 149         f(kc, ks.url, reader, writer, uploadStatusChan)
 150 }
 151
 152 func (s *StandaloneSuite) TestUploadToStubKeepServer(c *C) {
 153         log.Printf("TestUploadToStubKeepServer")
 154
 155         st := StubPutHandler{
 156                 c,
 157                 "acbd18db4cc2f85cedef654fccc4a4d8",
 158                 "abc123",
 159                 "foo",
 160                 "", "default=1",
 161                 make(chan string)}
 162
 163         UploadToStubHelper(c, st,
 164                 func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, uploadStatusChan chan uploadStatus) {
 165                         go kc.uploadToKeepServer(url, st.expectPath, nil, reader, uploadStatusChan, int64(len("foo")), kc.getRequestID())
 166
 167                         writer.Write([]byte("foo"))
 168                         writer.Close()
 169
 170                         <-st.handled
 171                         status := <-uploadStatusChan
 172                         c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, map[string]int{"default": 1}, ""})
 173                 })
 174 }
 175
 176 func (s *StandaloneSuite) TestUploadToStubKeepServerBufferReader(c *C) {
 177         st := StubPutHandler{
 178                 c,
 179                 "acbd18db4cc2f85cedef654fccc4a4d8",
 180                 "abc123",
 181                 "foo",
 182                 "", "default=1",
 183                 make(chan string)}
 184
 185         UploadToStubHelper(c, st,
 186                 func(kc *KeepClient, url string, _ io.ReadCloser, _ io.WriteCloser, uploadStatusChan chan uploadStatus) {
 187                         go kc.uploadToKeepServer(url, st.expectPath, nil, bytes.NewBuffer([]byte("foo")), uploadStatusChan, 3, kc.getRequestID())
 188
 189                         <-st.handled
 190
 191                         status := <-uploadStatusChan
 192                         c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, map[string]int{"default": 1}, ""})
 193                 })
 194 }
 195
 196 func (s *StandaloneSuite) TestUploadWithStorageClasses(c *C) {
 197         for _, trial := range []struct {
 198                 respHeader string
 199                 expectMap  map[string]int
 200         }{
 201                 {"", nil},
 202                 {"foo=1", map[string]int{"foo": 1}},
 203                 {" foo=1 , bar=2 ", map[string]int{"foo": 1, "bar": 2}},
 204                 {" =foo=1 ", nil},
 205                 {"foo", nil},
 206         } {
 207                 st := StubPutHandler{
 208                         c,
 209                         "acbd18db4cc2f85cedef654fccc4a4d8",
 210                         "abc123",
 211                         "foo",
 212                         "", trial.respHeader,
 213                         make(chan string)}
 214
 215                 UploadToStubHelper(c, st,
 216                         func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, uploadStatusChan chan uploadStatus) {
 217                                 go kc.uploadToKeepServer(url, st.expectPath, nil, reader, uploadStatusChan, int64(len("foo")), kc.getRequestID())
 218
 219                                 writer.Write([]byte("foo"))
 220                                 writer.Close()
 221
 222                                 <-st.handled
 223                                 status := <-uploadStatusChan
 224                                 c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, trial.expectMap, ""})
 225                         })
 226         }
 227 }
 228
 229 type FailHandler struct {
 230         handled chan string
 231 }
 232
 233 func (fh FailHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 234         resp.WriteHeader(500)
 235         fh.handled <- fmt.Sprintf("http://%s", req.Host)
 236 }
 237
 238 type FailThenSucceedHandler struct {
 239         handled        chan string
 240         count          int
 241         successhandler http.Handler
 242         reqIDs         []string
 243 }
 244
 245 func (fh *FailThenSucceedHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 246         fh.reqIDs = append(fh.reqIDs, req.Header.Get("X-Request-Id"))
 247         if fh.count == 0 {
 248                 resp.WriteHeader(500)
 249                 fh.count++
 250                 fh.handled <- fmt.Sprintf("http://%s", req.Host)
 251         } else {
 252                 fh.successhandler.ServeHTTP(resp, req)
 253         }
 254 }
 255
 256 type Error404Handler struct {
 257         handled chan string
 258 }
 259
 260 func (fh Error404Handler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 261         resp.WriteHeader(404)
 262         fh.handled <- fmt.Sprintf("http://%s", req.Host)
 263 }
 264
 265 func (s *StandaloneSuite) TestFailedUploadToStubKeepServer(c *C) {
 266         st := FailHandler{
 267                 make(chan string)}
 268
 269         hash := "acbd18db4cc2f85cedef654fccc4a4d8"
 270
 271         UploadToStubHelper(c, st,
 272                 func(kc *KeepClient, url string, reader io.ReadCloser,
 273                         writer io.WriteCloser, uploadStatusChan chan uploadStatus) {
 274
 275                         go kc.uploadToKeepServer(url, hash, nil, reader, uploadStatusChan, 3, kc.getRequestID())
 276
 277                         writer.Write([]byte("foo"))
 278                         writer.Close()
 279
 280                         <-st.handled
 281
 282                         status := <-uploadStatusChan
 283                         c.Check(status.url, Equals, fmt.Sprintf("%s/%s", url, hash))
 284                         c.Check(status.statusCode, Equals, 500)
 285                 })
 286 }
 287
 288 type KeepServer struct {
 289         listener net.Listener
 290         url      string
 291 }
 292
 293 func RunSomeFakeKeepServers(st http.Handler, n int) (ks []KeepServer) {
 294         ks = make([]KeepServer, n)
 295
 296         for i := 0; i < n; i++ {
 297                 ks[i] = RunFakeKeepServer(st)
 298         }
 299
 300         return ks
 301 }
 302
 303 func (s *StandaloneSuite) TestPutB(c *C) {
 304         hash := Md5String("foo")
 305
 306         st := StubPutHandler{
 307                 c,
 308                 hash,
 309                 "abc123",
 310                 "foo",
 311                 "",
 312                 "",
 313                 make(chan string, 5)}
 314
 315         arv, _ := arvadosclient.MakeArvadosClient()
 316         kc, _ := MakeKeepClient(arv)
 317
 318         kc.Want_replicas = 2
 319         arv.ApiToken = "abc123"
 320         localRoots := make(map[string]string)
 321         writableLocalRoots := make(map[string]string)
 322
 323         ks := RunSomeFakeKeepServers(st, 5)
 324
 325         for i, k := range ks {
 326                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 327                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 328                 defer k.listener.Close()
 329         }
 330
 331         kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
 332
 333         kc.PutB([]byte("foo"))
 334
 335         shuff := NewRootSorter(
 336                 kc.LocalRoots(), Md5String("foo")).GetSortedRoots()
 337
 338         s1 := <-st.handled
 339         s2 := <-st.handled
 340         c.Check((s1 == shuff[0] && s2 == shuff[1]) ||
 341                 (s1 == shuff[1] && s2 == shuff[0]),
 342                 Equals,
 343                 true)
 344 }
 345
 346 func (s *StandaloneSuite) TestPutHR(c *C) {
 347         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 348
 349         st := StubPutHandler{
 350                 c,
 351                 hash,
 352                 "abc123",
 353                 "foo",
 354                 "",
 355                 "",
 356                 make(chan string, 5)}
 357
 358         arv, _ := arvadosclient.MakeArvadosClient()
 359         kc, _ := MakeKeepClient(arv)
 360
 361         kc.Want_replicas = 2
 362         arv.ApiToken = "abc123"
 363         localRoots := make(map[string]string)
 364         writableLocalRoots := make(map[string]string)
 365
 366         ks := RunSomeFakeKeepServers(st, 5)
 367
 368         for i, k := range ks {
 369                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 370                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 371                 defer k.listener.Close()
 372         }
 373
 374         kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
 375
 376         reader, writer := io.Pipe()
 377
 378         go func() {
 379                 writer.Write([]byte("foo"))
 380                 writer.Close()
 381         }()
 382
 383         kc.PutHR(hash, reader, 3)
 384
 385         shuff := NewRootSorter(kc.LocalRoots(), hash).GetSortedRoots()
 386
 387         s1 := <-st.handled
 388         s2 := <-st.handled
 389
 390         c.Check((s1 == shuff[0] && s2 == shuff[1]) ||
 391                 (s1 == shuff[1] && s2 == shuff[0]),
 392                 Equals,
 393                 true)
 394 }
 395
 396 func (s *StandaloneSuite) TestPutWithFail(c *C) {
 397         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 398
 399         st := StubPutHandler{
 400                 c,
 401                 hash,
 402                 "abc123",
 403                 "foo",
 404                 "",
 405                 "",
 406                 make(chan string, 4)}
 407
 408         fh := FailHandler{
 409                 make(chan string, 1)}
 410
 411         arv, err := arvadosclient.MakeArvadosClient()
 412         c.Check(err, IsNil)
 413         kc, _ := MakeKeepClient(arv)
 414
 415         kc.Want_replicas = 2
 416         arv.ApiToken = "abc123"
 417         localRoots := make(map[string]string)
 418         writableLocalRoots := make(map[string]string)
 419
 420         ks1 := RunSomeFakeKeepServers(st, 4)
 421         ks2 := RunSomeFakeKeepServers(fh, 1)
 422
 423         for i, k := range ks1 {
 424                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 425                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 426                 defer k.listener.Close()
 427         }
 428         for i, k := range ks2 {
 429                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
 430                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
 431                 defer k.listener.Close()
 432         }
 433
 434         kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
 435
 436         shuff := NewRootSorter(
 437                 kc.LocalRoots(), Md5String("foo")).GetSortedRoots()
 438         c.Logf("%+v", shuff)
 439
 440         phash, replicas, err := kc.PutB([]byte("foo"))
 441
 442         <-fh.handled
 443
 444         c.Check(err, Equals, nil)
 445         c.Check(phash, Equals, "")
 446         c.Check(replicas, Equals, 2)
 447
 448         s1 := <-st.handled
 449         s2 := <-st.handled
 450
 451         c.Check((s1 == shuff[1] && s2 == shuff[2]) ||
 452                 (s1 == shuff[2] && s2 == shuff[1]),
 453                 Equals,
 454                 true)
 455 }
 456
 457 func (s *StandaloneSuite) TestPutWithTooManyFail(c *C) {
 458         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 459
 460         st := StubPutHandler{
 461                 c,
 462                 hash,
 463                 "abc123",
 464                 "foo",
 465                 "",
 466                 "",
 467                 make(chan string, 1)}
 468
 469         fh := FailHandler{
 470                 make(chan string, 4)}
 471
 472         arv, err := arvadosclient.MakeArvadosClient()
 473         c.Check(err, IsNil)
 474         kc, _ := MakeKeepClient(arv)
 475
 476         kc.Want_replicas = 2
 477         kc.Retries = 0
 478         arv.ApiToken = "abc123"
 479         localRoots := make(map[string]string)
 480         writableLocalRoots := make(map[string]string)
 481
 482         ks1 := RunSomeFakeKeepServers(st, 1)
 483         ks2 := RunSomeFakeKeepServers(fh, 4)
 484
 485         for i, k := range ks1 {
 486                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 487                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 488                 defer k.listener.Close()
 489         }
 490         for i, k := range ks2 {
 491                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
 492                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
 493                 defer k.listener.Close()
 494         }
 495
 496         kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
 497
 498         _, replicas, err := kc.PutB([]byte("foo"))
 499
 500         c.Check(err, FitsTypeOf, InsufficientReplicasError(errors.New("")))
 501         c.Check(replicas, Equals, 1)
 502         c.Check(<-st.handled, Equals, ks1[0].url)
 503 }
 504
 505 type StubGetHandler struct {
 506         c              *C
 507         expectPath     string
 508         expectAPIToken string
 509         httpStatus     int
 510         body           []byte
 511 }
 512
 513 func (sgh StubGetHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 514         sgh.c.Check(req.URL.Path, Equals, "/"+sgh.expectPath)
 515         sgh.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sgh.expectAPIToken))
 516         resp.WriteHeader(sgh.httpStatus)
 517         resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(sgh.body)))
 518         resp.Write(sgh.body)
 519 }
 520
 521 func (s *StandaloneSuite) TestGet(c *C) {
 522         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 523
 524         st := StubGetHandler{
 525                 c,
 526                 hash,
 527                 "abc123",
 528                 http.StatusOK,
 529                 []byte("foo")}
 530
 531         ks := RunFakeKeepServer(st)
 532         defer ks.listener.Close()
 533
 534         arv, err := arvadosclient.MakeArvadosClient()
 535         c.Check(err, IsNil)
 536         kc, _ := MakeKeepClient(arv)
 537         arv.ApiToken = "abc123"
 538         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
 539
 540         r, n, url2, err := kc.Get(hash)
 541         defer r.Close()
 542         c.Check(err, Equals, nil)
 543         c.Check(n, Equals, int64(3))
 544         c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks.url, hash))
 545
 546         content, err2 := ioutil.ReadAll(r)
 547         c.Check(err2, Equals, nil)
 548         c.Check(content, DeepEquals, []byte("foo"))
 549 }
 550
 551 func (s *StandaloneSuite) TestGet404(c *C) {
 552         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 553
 554         st := Error404Handler{make(chan string, 1)}
 555
 556         ks := RunFakeKeepServer(st)
 557         defer ks.listener.Close()
 558
 559         arv, err := arvadosclient.MakeArvadosClient()
 560         c.Check(err, IsNil)
 561         kc, _ := MakeKeepClient(arv)
 562         arv.ApiToken = "abc123"
 563         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
 564
 565         r, n, url2, err := kc.Get(hash)
 566         c.Check(err, Equals, BlockNotFound)
 567         c.Check(n, Equals, int64(0))
 568         c.Check(url2, Equals, "")
 569         c.Check(r, Equals, nil)
 570 }
 571
 572 func (s *StandaloneSuite) TestGetEmptyBlock(c *C) {
 573         st := Error404Handler{make(chan string, 1)}
 574
 575         ks := RunFakeKeepServer(st)
 576         defer ks.listener.Close()
 577
 578         arv, err := arvadosclient.MakeArvadosClient()
 579         c.Check(err, IsNil)
 580         kc, _ := MakeKeepClient(arv)
 581         arv.ApiToken = "abc123"
 582         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
 583
 584         r, n, url2, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e+0")
 585         c.Check(err, IsNil)
 586         c.Check(n, Equals, int64(0))
 587         c.Check(url2, Equals, "")
 588         c.Assert(r, NotNil)
 589         buf, err := ioutil.ReadAll(r)
 590         c.Check(err, IsNil)
 591         c.Check(buf, DeepEquals, []byte{})
 592 }
 593
 594 func (s *StandaloneSuite) TestGetFail(c *C) {
 595         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 596
 597         st := FailHandler{make(chan string, 1)}
 598
 599         ks := RunFakeKeepServer(st)
 600         defer ks.listener.Close()
 601
 602         arv, err := arvadosclient.MakeArvadosClient()
 603         c.Check(err, IsNil)
 604         kc, _ := MakeKeepClient(arv)
 605         arv.ApiToken = "abc123"
 606         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
 607         kc.Retries = 0
 608
 609         r, n, url2, err := kc.Get(hash)
 610         errNotFound, _ := err.(*ErrNotFound)
 611         c.Check(errNotFound, NotNil)
 612         c.Check(strings.Contains(errNotFound.Error(), "HTTP 500"), Equals, true)
 613         c.Check(errNotFound.Temporary(), Equals, true)
 614         c.Check(n, Equals, int64(0))
 615         c.Check(url2, Equals, "")
 616         c.Check(r, Equals, nil)
 617 }
 618
 619 func (s *StandaloneSuite) TestGetFailRetry(c *C) {
 620         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 621
 622         st := &FailThenSucceedHandler{
 623                 handled: make(chan string, 1),
 624                 successhandler: StubGetHandler{
 625                         c,
 626                         hash,
 627                         "abc123",
 628                         http.StatusOK,
 629                         []byte("foo")}}
 630
 631         ks := RunFakeKeepServer(st)
 632         defer ks.listener.Close()
 633
 634         arv, err := arvadosclient.MakeArvadosClient()
 635         c.Check(err, IsNil)
 636         kc, _ := MakeKeepClient(arv)
 637         arv.ApiToken = "abc123"
 638         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
 639
 640         r, n, url2, err := kc.Get(hash)
 641         defer r.Close()
 642         c.Check(err, Equals, nil)
 643         c.Check(n, Equals, int64(3))
 644         c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks.url, hash))
 645
 646         content, err2 := ioutil.ReadAll(r)
 647         c.Check(err2, Equals, nil)
 648         c.Check(content, DeepEquals, []byte("foo"))
 649
 650         c.Logf("%q", st.reqIDs)
 651         c.Assert(len(st.reqIDs) > 1, Equals, true)
 652         for _, reqid := range st.reqIDs {
 653                 c.Check(reqid, Not(Equals), "")
 654                 c.Check(reqid, Equals, st.reqIDs[0])
 655         }
 656 }
 657
 658 func (s *StandaloneSuite) TestGetNetError(c *C) {
 659         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 660
 661         arv, err := arvadosclient.MakeArvadosClient()
 662         c.Check(err, IsNil)
 663         kc, _ := MakeKeepClient(arv)
 664         arv.ApiToken = "abc123"
 665         kc.SetServiceRoots(map[string]string{"x": "http://localhost:62222"}, nil, nil)
 666
 667         r, n, url2, err := kc.Get(hash)
 668         errNotFound, _ := err.(*ErrNotFound)
 669         c.Check(errNotFound, NotNil)
 670         c.Check(strings.Contains(errNotFound.Error(), "connection refused"), Equals, true)
 671         c.Check(errNotFound.Temporary(), Equals, true)
 672         c.Check(n, Equals, int64(0))
 673         c.Check(url2, Equals, "")
 674         c.Check(r, Equals, nil)
 675 }
 676
 677 func (s *StandaloneSuite) TestGetWithServiceHint(c *C) {
 678         uuid := "zzzzz-bi6l4-123451234512345"
 679         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 680
 681         // This one shouldn't be used:
 682         ks0 := RunFakeKeepServer(StubGetHandler{
 683                 c,
 684                 "error if used",
 685                 "abc123",
 686                 http.StatusOK,
 687                 []byte("foo")})
 688         defer ks0.listener.Close()
 689         // This one should be used:
 690         ks := RunFakeKeepServer(StubGetHandler{
 691                 c,
 692                 hash + "+K@" + uuid,
 693                 "abc123",
 694                 http.StatusOK,
 695                 []byte("foo")})
 696         defer ks.listener.Close()
 697
 698         arv, err := arvadosclient.MakeArvadosClient()
 699         c.Check(err, IsNil)
 700         kc, _ := MakeKeepClient(arv)
 701         arv.ApiToken = "abc123"
 702         kc.SetServiceRoots(
 703                 map[string]string{"x": ks0.url},
 704                 nil,
 705                 map[string]string{uuid: ks.url})
 706
 707         r, n, uri, err := kc.Get(hash + "+K@" + uuid)
 708         defer r.Close()
 709         c.Check(err, Equals, nil)
 710         c.Check(n, Equals, int64(3))
 711         c.Check(uri, Equals, fmt.Sprintf("%s/%s", ks.url, hash+"+K@"+uuid))
 712
 713         content, err := ioutil.ReadAll(r)
 714         c.Check(err, Equals, nil)
 715         c.Check(content, DeepEquals, []byte("foo"))
 716 }
 717
 718 // Use a service hint to fetch from a local disk service, overriding
 719 // rendezvous probe order.
 720 func (s *StandaloneSuite) TestGetWithLocalServiceHint(c *C) {
 721         uuid := "zzzzz-bi6l4-zzzzzzzzzzzzzzz"
 722         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 723
 724         // This one shouldn't be used, although it appears first in
 725         // rendezvous probe order:
 726         ks0 := RunFakeKeepServer(StubGetHandler{
 727                 c,
 728                 "error if used",
 729                 "abc123",
 730                 http.StatusOK,
 731                 []byte("foo")})
 732         defer ks0.listener.Close()
 733         // This one should be used:
 734         ks := RunFakeKeepServer(StubGetHandler{
 735                 c,
 736                 hash + "+K@" + uuid,
 737                 "abc123",
 738                 http.StatusOK,
 739                 []byte("foo")})
 740         defer ks.listener.Close()
 741
 742         arv, err := arvadosclient.MakeArvadosClient()
 743         c.Check(err, IsNil)
 744         kc, _ := MakeKeepClient(arv)
 745         arv.ApiToken = "abc123"
 746         kc.SetServiceRoots(
 747                 map[string]string{
 748                         "zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url,
 749                         "zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url,
 750                         "zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url,
 751                         uuid:                          ks.url},
 752                 nil,
 753                 map[string]string{
 754                         "zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url,
 755                         "zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url,
 756                         "zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url,
 757                         uuid:                          ks.url},
 758         )
 759
 760         r, n, uri, err := kc.Get(hash + "+K@" + uuid)
 761         defer r.Close()
 762         c.Check(err, Equals, nil)
 763         c.Check(n, Equals, int64(3))
 764         c.Check(uri, Equals, fmt.Sprintf("%s/%s", ks.url, hash+"+K@"+uuid))
 765
 766         content, err := ioutil.ReadAll(r)
 767         c.Check(err, Equals, nil)
 768         c.Check(content, DeepEquals, []byte("foo"))
 769 }
 770
 771 func (s *StandaloneSuite) TestGetWithServiceHintFailoverToLocals(c *C) {
 772         uuid := "zzzzz-bi6l4-123451234512345"
 773         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 774
 775         ksLocal := RunFakeKeepServer(StubGetHandler{
 776                 c,
 777                 hash + "+K@" + uuid,
 778                 "abc123",
 779                 http.StatusOK,
 780                 []byte("foo")})
 781         defer ksLocal.listener.Close()
 782         ksGateway := RunFakeKeepServer(StubGetHandler{
 783                 c,
 784                 hash + "+K@" + uuid,
 785                 "abc123",
 786                 http.StatusInternalServerError,
 787                 []byte("Error")})
 788         defer ksGateway.listener.Close()
 789
 790         arv, err := arvadosclient.MakeArvadosClient()
 791         c.Check(err, IsNil)
 792         kc, _ := MakeKeepClient(arv)
 793         arv.ApiToken = "abc123"
 794         kc.SetServiceRoots(
 795                 map[string]string{"zzzzz-bi6l4-keepdisk0000000": ksLocal.url},
 796                 nil,
 797                 map[string]string{uuid: ksGateway.url})
 798
 799         r, n, uri, err := kc.Get(hash + "+K@" + uuid)
 800         c.Assert(err, Equals, nil)
 801         defer r.Close()
 802         c.Check(n, Equals, int64(3))
 803         c.Check(uri, Equals, fmt.Sprintf("%s/%s", ksLocal.url, hash+"+K@"+uuid))
 804
 805         content, err := ioutil.ReadAll(r)
 806         c.Check(err, Equals, nil)
 807         c.Check(content, DeepEquals, []byte("foo"))
 808 }
 809
 810 type BarHandler struct {
 811         handled chan string
 812 }
 813
 814 func (h BarHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 815         resp.Write([]byte("bar"))
 816         h.handled <- fmt.Sprintf("http://%s", req.Host)
 817 }
 818
 819 func (s *StandaloneSuite) TestChecksum(c *C) {
 820         foohash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 821         barhash := fmt.Sprintf("%x", md5.Sum([]byte("bar")))
 822
 823         st := BarHandler{make(chan string, 1)}
 824
 825         ks := RunFakeKeepServer(st)
 826         defer ks.listener.Close()
 827
 828         arv, err := arvadosclient.MakeArvadosClient()
 829         c.Check(err, IsNil)
 830         kc, _ := MakeKeepClient(arv)
 831         arv.ApiToken = "abc123"
 832         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
 833
 834         r, n, _, err := kc.Get(barhash)
 835         c.Check(err, IsNil)
 836         _, err = ioutil.ReadAll(r)
 837         c.Check(n, Equals, int64(3))
 838         c.Check(err, Equals, nil)
 839
 840         <-st.handled
 841
 842         r, n, _, err = kc.Get(foohash)
 843         c.Check(err, IsNil)
 844         _, err = ioutil.ReadAll(r)
 845         c.Check(n, Equals, int64(3))
 846         c.Check(err, Equals, BadChecksum)
 847
 848         <-st.handled
 849 }
 850
 851 func (s *StandaloneSuite) TestGetWithFailures(c *C) {
 852         content := []byte("waz")
 853         hash := fmt.Sprintf("%x", md5.Sum(content))
 854
 855         fh := Error404Handler{
 856                 make(chan string, 4)}
 857
 858         st := StubGetHandler{
 859                 c,
 860                 hash,
 861                 "abc123",
 862                 http.StatusOK,
 863                 content}
 864
 865         arv, err := arvadosclient.MakeArvadosClient()
 866         c.Check(err, IsNil)
 867         kc, _ := MakeKeepClient(arv)
 868         arv.ApiToken = "abc123"
 869         localRoots := make(map[string]string)
 870         writableLocalRoots := make(map[string]string)
 871
 872         ks1 := RunSomeFakeKeepServers(st, 1)
 873         ks2 := RunSomeFakeKeepServers(fh, 4)
 874
 875         for i, k := range ks1 {
 876                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 877                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 878                 defer k.listener.Close()
 879         }
 880         for i, k := range ks2 {
 881                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
 882                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
 883                 defer k.listener.Close()
 884         }
 885
 886         kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
 887         kc.Retries = 0
 888
 889         // This test works only if one of the failing services is
 890         // attempted before the succeeding service. Otherwise,
 891         // <-fh.handled below will just hang! (Probe order depends on
 892         // the choice of block content "waz" and the UUIDs of the fake
 893         // servers, so we just tried different strings until we found
 894         // an example that passes this Assert.)
 895         c.Assert(NewRootSorter(localRoots, hash).GetSortedRoots()[0], Not(Equals), ks1[0].url)
 896
 897         r, n, url2, err := kc.Get(hash)
 898
 899         <-fh.handled
 900         c.Check(err, Equals, nil)
 901         c.Check(n, Equals, int64(3))
 902         c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks1[0].url, hash))
 903
 904         readContent, err2 := ioutil.ReadAll(r)
 905         c.Check(err2, Equals, nil)
 906         c.Check(readContent, DeepEquals, content)
 907 }
 908
 909 func (s *ServerRequiredSuite) TestPutGetHead(c *C) {
 910         content := []byte("TestPutGetHead")
 911
 912         arv, err := arvadosclient.MakeArvadosClient()
 913         c.Check(err, IsNil)
 914         kc, err := MakeKeepClient(arv)
 915         c.Assert(err, Equals, nil)
 916
 917         hash := fmt.Sprintf("%x", md5.Sum(content))
 918
 919         {
 920                 n, _, err := kc.Ask(hash)
 921                 c.Check(err, Equals, BlockNotFound)
 922                 c.Check(n, Equals, int64(0))
 923         }
 924         {
 925                 hash2, replicas, err := kc.PutB(content)
 926                 c.Check(hash2, Matches, fmt.Sprintf(`%s\+%d\b.*`, hash, len(content)))
 927                 c.Check(replicas, Equals, 2)
 928                 c.Check(err, Equals, nil)
 929         }
 930         {
 931                 r, n, url2, err := kc.Get(hash)
 932                 c.Check(err, Equals, nil)
 933                 c.Check(n, Equals, int64(len(content)))
 934                 c.Check(url2, Matches, fmt.Sprintf("http://localhost:\\d+/%s", hash))
 935
 936                 readContent, err2 := ioutil.ReadAll(r)
 937                 c.Check(err2, Equals, nil)
 938                 c.Check(readContent, DeepEquals, content)
 939         }
 940         {
 941                 n, url2, err := kc.Ask(hash)
 942                 c.Check(err, Equals, nil)
 943                 c.Check(n, Equals, int64(len(content)))
 944                 c.Check(url2, Matches, fmt.Sprintf("http://localhost:\\d+/%s", hash))
 945         }
 946         {
 947                 loc, err := kc.LocalLocator(hash)
 948                 c.Check(err, Equals, nil)
 949                 c.Assert(len(loc) >= 32, Equals, true)
 950                 c.Check(loc[:32], Equals, hash[:32])
 951         }
 952         {
 953                 content := []byte("the perth county conspiracy")
 954                 loc, err := kc.LocalLocator(fmt.Sprintf("%x+%d+Rzaaaa-abcde@12345", md5.Sum(content), len(content)))
 955                 c.Check(loc, Equals, "")
 956                 c.Check(err, ErrorMatches, `.*HEAD .*\+R.*`)
 957                 c.Check(err, ErrorMatches, `.*HTTP 400.*`)
 958         }
 959 }
 960
 961 type StubProxyHandler struct {
 962         handled chan string
 963 }
 964
 965 func (h StubProxyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 966         resp.Header().Set("X-Keep-Replicas-Stored", "2")
 967         h.handled <- fmt.Sprintf("http://%s", req.Host)
 968 }
 969
 970 func (s *StandaloneSuite) TestPutProxy(c *C) {
 971         st := StubProxyHandler{make(chan string, 1)}
 972
 973         arv, err := arvadosclient.MakeArvadosClient()
 974         c.Check(err, IsNil)
 975         kc, _ := MakeKeepClient(arv)
 976
 977         kc.Want_replicas = 2
 978         arv.ApiToken = "abc123"
 979         localRoots := make(map[string]string)
 980         writableLocalRoots := make(map[string]string)
 981
 982         ks1 := RunSomeFakeKeepServers(st, 1)
 983
 984         for i, k := range ks1 {
 985                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 986                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 987                 defer k.listener.Close()
 988         }
 989
 990         kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
 991
 992         _, replicas, err := kc.PutB([]byte("foo"))
 993         <-st.handled
 994
 995         c.Check(err, Equals, nil)
 996         c.Check(replicas, Equals, 2)
 997 }
 998
 999 func (s *StandaloneSuite) TestPutProxyInsufficientReplicas(c *C) {
1000         st := StubProxyHandler{make(chan string, 1)}
1001
1002         arv, err := arvadosclient.MakeArvadosClient()
1003         c.Check(err, IsNil)
1004         kc, _ := MakeKeepClient(arv)
1005
1006         kc.Want_replicas = 3
1007         arv.ApiToken = "abc123"
1008         localRoots := make(map[string]string)
1009         writableLocalRoots := make(map[string]string)
1010
1011         ks1 := RunSomeFakeKeepServers(st, 1)
1012
1013         for i, k := range ks1 {
1014                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
1015                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
1016                 defer k.listener.Close()
1017         }
1018         kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
1019
1020         _, replicas, err := kc.PutB([]byte("foo"))
1021         <-st.handled
1022
1023         c.Check(err, FitsTypeOf, InsufficientReplicasError(errors.New("")))
1024         c.Check(replicas, Equals, 2)
1025 }
1026
1027 func (s *StandaloneSuite) TestMakeLocator(c *C) {
1028         l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce+3+Aabcde@12345678")
1029         c.Check(err, Equals, nil)
1030         c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
1031         c.Check(l.Size, Equals, 3)
1032         c.Check(l.Hints, DeepEquals, []string{"3", "Aabcde@12345678"})
1033 }
1034
1035 func (s *StandaloneSuite) TestMakeLocatorNoHints(c *C) {
1036         l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce")
1037         c.Check(err, Equals, nil)
1038         c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
1039         c.Check(l.Size, Equals, -1)
1040         c.Check(l.Hints, DeepEquals, []string{})
1041 }
1042
1043 func (s *StandaloneSuite) TestMakeLocatorNoSizeHint(c *C) {
1044         l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce+Aabcde@12345678")
1045         c.Check(err, Equals, nil)
1046         c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
1047         c.Check(l.Size, Equals, -1)
1048         c.Check(l.Hints, DeepEquals, []string{"Aabcde@12345678"})
1049 }
1050
1051 func (s *StandaloneSuite) TestMakeLocatorPreservesUnrecognizedHints(c *C) {
1052         str := "91f372a266fe2bf2823cb8ec7fda31ce+3+Unknown+Kzzzzz+Afoobar"
1053         l, err := MakeLocator(str)
1054         c.Check(err, Equals, nil)
1055         c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
1056         c.Check(l.Size, Equals, 3)
1057         c.Check(l.Hints, DeepEquals, []string{"3", "Unknown", "Kzzzzz", "Afoobar"})
1058         c.Check(l.String(), Equals, str)
1059 }
1060
1061 func (s *StandaloneSuite) TestMakeLocatorInvalidInput(c *C) {
1062         _, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31c")
1063         c.Check(err, Equals, InvalidLocatorError)
1064 }
1065
1066 func (s *StandaloneSuite) TestPutBWant2ReplicasWithOnlyOneWritableLocalRoot(c *C) {
1067         hash := Md5String("foo")
1068
1069         st := StubPutHandler{
1070                 c,
1071                 hash,
1072                 "abc123",
1073                 "foo",
1074                 "",
1075                 "",
1076                 make(chan string, 5)}
1077
1078         arv, _ := arvadosclient.MakeArvadosClient()
1079         kc, _ := MakeKeepClient(arv)
1080
1081         kc.Want_replicas = 2
1082         arv.ApiToken = "abc123"
1083         localRoots := make(map[string]string)
1084         writableLocalRoots := make(map[string]string)
1085
1086         ks := RunSomeFakeKeepServers(st, 5)
1087
1088         for i, k := range ks {
1089                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
1090                 if i == 0 {
1091                         writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
1092                 }
1093                 defer k.listener.Close()
1094         }
1095
1096         kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
1097
1098         _, replicas, err := kc.PutB([]byte("foo"))
1099
1100         c.Check(err, FitsTypeOf, InsufficientReplicasError(errors.New("")))
1101         c.Check(replicas, Equals, 1)
1102
1103         c.Check(<-st.handled, Equals, localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", 0)])
1104 }
1105
1106 func (s *StandaloneSuite) TestPutBWithNoWritableLocalRoots(c *C) {
1107         hash := Md5String("foo")
1108
1109         st := StubPutHandler{
1110                 c,
1111                 hash,
1112                 "abc123",
1113                 "foo",
1114                 "",
1115                 "",
1116                 make(chan string, 5)}
1117
1118         arv, _ := arvadosclient.MakeArvadosClient()
1119         kc, _ := MakeKeepClient(arv)
1120
1121         kc.Want_replicas = 2
1122         arv.ApiToken = "abc123"
1123         localRoots := make(map[string]string)
1124         writableLocalRoots := make(map[string]string)
1125
1126         ks := RunSomeFakeKeepServers(st, 5)
1127
1128         for i, k := range ks {
1129                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
1130                 defer k.listener.Close()
1131         }
1132
1133         kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
1134
1135         _, replicas, err := kc.PutB([]byte("foo"))
1136
1137         c.Check(err, FitsTypeOf, InsufficientReplicasError(errors.New("")))
1138         c.Check(replicas, Equals, 0)
1139 }
1140
1141 type StubGetIndexHandler struct {
1142         c              *C
1143         expectPath     string
1144         expectAPIToken string
1145         httpStatus     int
1146         body           []byte
1147 }
1148
1149 func (h StubGetIndexHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
1150         h.c.Check(req.URL.Path, Equals, h.expectPath)
1151         h.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", h.expectAPIToken))
1152         resp.WriteHeader(h.httpStatus)
1153         resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(h.body)))
1154         resp.Write(h.body)
1155 }
1156
1157 func (s *StandaloneSuite) TestGetIndexWithNoPrefix(c *C) {
1158         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
1159
1160         st := StubGetIndexHandler{
1161                 c,
1162                 "/index",
1163                 "abc123",
1164                 http.StatusOK,
1165                 []byte(hash + "+3 1443559274\n\n")}
1166
1167         ks := RunFakeKeepServer(st)
1168         defer ks.listener.Close()
1169
1170         arv, err := arvadosclient.MakeArvadosClient()
1171         c.Assert(err, IsNil)
1172         kc, err := MakeKeepClient(arv)
1173         c.Assert(err, IsNil)
1174         arv.ApiToken = "abc123"
1175         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
1176
1177         r, err := kc.GetIndex("x", "")
1178         c.Check(err, IsNil)
1179
1180         content, err2 := ioutil.ReadAll(r)
1181         c.Check(err2, Equals, nil)
1182         c.Check(content, DeepEquals, st.body[0:len(st.body)-1])
1183 }
1184
1185 func (s *StandaloneSuite) TestGetIndexWithPrefix(c *C) {
1186         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
1187
1188         st := StubGetIndexHandler{
1189                 c,
1190                 "/index/" + hash[0:3],
1191                 "abc123",
1192                 http.StatusOK,
1193                 []byte(hash + "+3 1443559274\n\n")}
1194
1195         ks := RunFakeKeepServer(st)
1196         defer ks.listener.Close()
1197
1198         arv, err := arvadosclient.MakeArvadosClient()
1199         c.Check(err, IsNil)
1200         kc, _ := MakeKeepClient(arv)
1201         arv.ApiToken = "abc123"
1202         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
1203
1204         r, err := kc.GetIndex("x", hash[0:3])
1205         c.Assert(err, Equals, nil)
1206
1207         content, err2 := ioutil.ReadAll(r)
1208         c.Check(err2, Equals, nil)
1209         c.Check(content, DeepEquals, st.body[0:len(st.body)-1])
1210 }
1211
1212 func (s *StandaloneSuite) TestGetIndexIncomplete(c *C) {
1213         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
1214
1215         st := StubGetIndexHandler{
1216                 c,
1217                 "/index/" + hash[0:3],
1218                 "abc123",
1219                 http.StatusOK,
1220                 []byte(hash)}
1221
1222         ks := RunFakeKeepServer(st)
1223         defer ks.listener.Close()
1224
1225         arv, err := arvadosclient.MakeArvadosClient()
1226         c.Check(err, IsNil)
1227         kc, _ := MakeKeepClient(arv)
1228         arv.ApiToken = "abc123"
1229         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
1230
1231         _, err = kc.GetIndex("x", hash[0:3])
1232         c.Check(err, Equals, ErrIncompleteIndex)
1233 }
1234
1235 func (s *StandaloneSuite) TestGetIndexWithNoSuchServer(c *C) {
1236         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
1237
1238         st := StubGetIndexHandler{
1239                 c,
1240                 "/index/" + hash[0:3],
1241                 "abc123",
1242                 http.StatusOK,
1243                 []byte(hash)}
1244
1245         ks := RunFakeKeepServer(st)
1246         defer ks.listener.Close()
1247
1248         arv, err := arvadosclient.MakeArvadosClient()
1249         c.Check(err, IsNil)
1250         kc, _ := MakeKeepClient(arv)
1251         arv.ApiToken = "abc123"
1252         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
1253
1254         _, err = kc.GetIndex("y", hash[0:3])
1255         c.Check(err, Equals, ErrNoSuchKeepServer)
1256 }
1257
1258 func (s *StandaloneSuite) TestGetIndexWithNoSuchPrefix(c *C) {
1259         st := StubGetIndexHandler{
1260                 c,
1261                 "/index/abcd",
1262                 "abc123",
1263                 http.StatusOK,
1264                 []byte("\n")}
1265
1266         ks := RunFakeKeepServer(st)
1267         defer ks.listener.Close()
1268
1269         arv, err := arvadosclient.MakeArvadosClient()
1270         c.Check(err, IsNil)
1271         kc, _ := MakeKeepClient(arv)
1272         arv.ApiToken = "abc123"
1273         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
1274
1275         r, err := kc.GetIndex("x", "abcd")
1276         c.Check(err, Equals, nil)
1277
1278         content, err2 := ioutil.ReadAll(r)
1279         c.Check(err2, Equals, nil)
1280         c.Check(content, DeepEquals, st.body[0:len(st.body)-1])
1281 }
1282
1283 func (s *StandaloneSuite) TestPutBRetry(c *C) {
1284         st := &FailThenSucceedHandler{
1285                 handled: make(chan string, 1),
1286                 successhandler: StubPutHandler{
1287                         c,
1288                         Md5String("foo"),
1289                         "abc123",
1290                         "foo",
1291                         "",
1292                         "",
1293                         make(chan string, 5)}}
1294
1295         arv, _ := arvadosclient.MakeArvadosClient()
1296         kc, _ := MakeKeepClient(arv)
1297
1298         kc.Want_replicas = 2
1299         arv.ApiToken = "abc123"
1300         localRoots := make(map[string]string)
1301         writableLocalRoots := make(map[string]string)
1302
1303         ks := RunSomeFakeKeepServers(st, 2)
1304
1305         for i, k := range ks {
1306                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
1307                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
1308                 defer k.listener.Close()
1309         }
1310
1311         kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
1312
1313         hash, replicas, err := kc.PutB([]byte("foo"))
1314
1315         c.Check(err, Equals, nil)
1316         c.Check(hash, Equals, "")
1317         c.Check(replicas, Equals, 2)
1318 }
1319
1320 func (s *ServerRequiredSuite) TestMakeKeepClientWithNonDiskTypeService(c *C) {
1321         arv, err := arvadosclient.MakeArvadosClient()
1322         c.Assert(err, Equals, nil)
1323
1324         // Add an additional "testblobstore" keepservice
1325         blobKeepService := make(arvadosclient.Dict)
1326         err = arv.Create("keep_services",
1327                 arvadosclient.Dict{"keep_service": arvadosclient.Dict{
1328                         "service_host": "localhost",
1329                         "service_port": "21321",
1330                         "service_type": "testblobstore"}},
1331                 &blobKeepService)
1332         c.Assert(err, Equals, nil)
1333         defer func() { arv.Delete("keep_services", blobKeepService["uuid"].(string), nil, nil) }()
1334         RefreshServiceDiscovery()
1335
1336         // Make a keepclient and ensure that the testblobstore is included
1337         kc, err := MakeKeepClient(arv)
1338         c.Assert(err, Equals, nil)
1339
1340         // verify kc.LocalRoots
1341         c.Check(len(kc.LocalRoots()), Equals, 3)
1342         for _, root := range kc.LocalRoots() {
1343                 c.Check(root, Matches, "http://localhost:\\d+")
1344         }
1345         c.Assert(kc.LocalRoots()[blobKeepService["uuid"].(string)], Not(Equals), "")
1346
1347         // verify kc.GatewayRoots
1348         c.Check(len(kc.GatewayRoots()), Equals, 3)
1349         for _, root := range kc.GatewayRoots() {
1350                 c.Check(root, Matches, "http://localhost:\\d+")
1351         }
1352         c.Assert(kc.GatewayRoots()[blobKeepService["uuid"].(string)], Not(Equals), "")
1353
1354         // verify kc.WritableLocalRoots
1355         c.Check(len(kc.WritableLocalRoots()), Equals, 3)
1356         for _, root := range kc.WritableLocalRoots() {
1357                 c.Check(root, Matches, "http://localhost:\\d+")
1358         }
1359         c.Assert(kc.WritableLocalRoots()[blobKeepService["uuid"].(string)], Not(Equals), "")
1360
1361         c.Assert(kc.replicasPerService, Equals, 0)
1362         c.Assert(kc.foundNonDiskSvc, Equals, true)
1363         c.Assert(kc.httpClient().(*http.Client).Timeout, Equals, 300*time.Second)
1364 }