sdk/go/keepclient/keepclient_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4
   5 package keepclient
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "crypto/md5"
  11         "fmt"
  12         "io"
  13         "io/ioutil"
  14         "log"
  15         "net"
  16         "net/http"
  17         "os"
  18         "strings"
  19         "sync"
  20         "testing"
  21         "time"
  22
  23         "git.arvados.org/arvados.git/sdk/go/arvados"
  24         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  25         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  26         . "gopkg.in/check.v1"
  27         check "gopkg.in/check.v1"
  28 )
  29
  30 // Gocheck boilerplate
  31 func Test(t *testing.T) {
  32         TestingT(t)
  33 }
  34
  35 // Gocheck boilerplate
  36 var _ = Suite(&ServerRequiredSuite{})
  37 var _ = Suite(&StandaloneSuite{})
  38
  39 // Tests that require the Keep server running
  40 type ServerRequiredSuite struct{}
  41
  42 // Standalone tests
  43 type StandaloneSuite struct{}
  44
  45 func (s *StandaloneSuite) SetUpTest(c *C) {
  46         RefreshServiceDiscovery()
  47 }
  48
  49 func pythonDir() string {
  50         cwd, _ := os.Getwd()
  51         return fmt.Sprintf("%s/../../python/tests", cwd)
  52 }
  53
  54 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
  55         arvadostest.StartAPI()
  56         arvadostest.StartKeep(2, false)
  57 }
  58
  59 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
  60         arvadostest.StopKeep(2)
  61         arvadostest.StopAPI()
  62 }
  63
  64 func (s *ServerRequiredSuite) SetUpTest(c *C) {
  65         RefreshServiceDiscovery()
  66 }
  67
  68 func (s *ServerRequiredSuite) TestMakeKeepClient(c *C) {
  69         arv, err := arvadosclient.MakeArvadosClient()
  70         c.Assert(err, Equals, nil)
  71
  72         kc, err := MakeKeepClient(arv)
  73
  74         c.Assert(err, Equals, nil)
  75         c.Check(len(kc.LocalRoots()), Equals, 2)
  76         for _, root := range kc.LocalRoots() {
  77                 c.Check(root, Matches, "http://localhost:\\d+")
  78         }
  79 }
  80
  81 func (s *ServerRequiredSuite) TestDefaultReplications(c *C) {
  82         arv, err := arvadosclient.MakeArvadosClient()
  83         c.Assert(err, Equals, nil)
  84
  85         kc, err := MakeKeepClient(arv)
  86         c.Check(err, IsNil)
  87         c.Assert(kc.Want_replicas, Equals, 2)
  88
  89         arv.DiscoveryDoc["defaultCollectionReplication"] = 3.0
  90         kc, err = MakeKeepClient(arv)
  91         c.Check(err, IsNil)
  92         c.Assert(kc.Want_replicas, Equals, 3)
  93
  94         arv.DiscoveryDoc["defaultCollectionReplication"] = 1.0
  95         kc, err = MakeKeepClient(arv)
  96         c.Check(err, IsNil)
  97         c.Assert(kc.Want_replicas, Equals, 1)
  98 }
  99
 100 type StubPutHandler struct {
 101         c                    *C
 102         expectPath           string
 103         expectAPIToken       string
 104         expectBody           string
 105         expectStorageClass   string
 106         returnStorageClasses string
 107         handled              chan string
 108         requests             []*http.Request
 109         mtx                  sync.Mutex
 110 }
 111
 112 func (sph *StubPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 113         sph.mtx.Lock()
 114         sph.requests = append(sph.requests, req)
 115         sph.mtx.Unlock()
 116         sph.c.Check(req.URL.Path, Equals, "/"+sph.expectPath)
 117         sph.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sph.expectAPIToken))
 118         if sph.expectStorageClass != "*" {
 119                 sph.c.Check(req.Header.Get("X-Keep-Storage-Classes"), Equals, sph.expectStorageClass)
 120         }
 121         body, err := ioutil.ReadAll(req.Body)
 122         sph.c.Check(err, Equals, nil)
 123         sph.c.Check(body, DeepEquals, []byte(sph.expectBody))
 124         resp.Header().Set("X-Keep-Replicas-Stored", "1")
 125         if sph.returnStorageClasses != "" {
 126                 resp.Header().Set("X-Keep-Storage-Classes-Confirmed", sph.returnStorageClasses)
 127         }
 128         resp.WriteHeader(200)
 129         sph.handled <- fmt.Sprintf("http://%s", req.Host)
 130 }
 131
 132 func RunFakeKeepServer(st http.Handler) (ks KeepServer) {
 133         var err error
 134         // If we don't explicitly bind it to localhost, ks.listener.Addr() will
 135         // bind to 0.0.0.0 or [::] which is not a valid address for Dial()
 136         ks.listener, err = net.ListenTCP("tcp", &net.TCPAddr{IP: []byte{127, 0, 0, 1}, Port: 0})
 137         if err != nil {
 138                 panic(fmt.Sprintf("Could not listen on any port"))
 139         }
 140         ks.url = fmt.Sprintf("http://%s", ks.listener.Addr().String())
 141         go http.Serve(ks.listener, st)
 142         return
 143 }
 144
 145 func UploadToStubHelper(c *C, st http.Handler, f func(*KeepClient, string,
 146         io.ReadCloser, io.WriteCloser, chan uploadStatus)) {
 147
 148         ks := RunFakeKeepServer(st)
 149         defer ks.listener.Close()
 150
 151         arv, _ := arvadosclient.MakeArvadosClient()
 152         arv.ApiToken = "abc123"
 153
 154         kc, _ := MakeKeepClient(arv)
 155
 156         reader, writer := io.Pipe()
 157         uploadStatusChan := make(chan uploadStatus)
 158
 159         f(kc, ks.url, reader, writer, uploadStatusChan)
 160 }
 161
 162 func (s *StandaloneSuite) TestUploadToStubKeepServer(c *C) {
 163         log.Printf("TestUploadToStubKeepServer")
 164
 165         st := &StubPutHandler{
 166                 c:                    c,
 167                 expectPath:           "acbd18db4cc2f85cedef654fccc4a4d8",
 168                 expectAPIToken:       "abc123",
 169                 expectBody:           "foo",
 170                 expectStorageClass:   "",
 171                 returnStorageClasses: "default=1",
 172                 handled:              make(chan string),
 173         }
 174
 175         UploadToStubHelper(c, st,
 176                 func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, uploadStatusChan chan uploadStatus) {
 177                         go kc.uploadToKeepServer(url, st.expectPath, nil, reader, uploadStatusChan, len("foo"), kc.getRequestID())
 178
 179                         writer.Write([]byte("foo"))
 180                         writer.Close()
 181
 182                         <-st.handled
 183                         status := <-uploadStatusChan
 184                         c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, map[string]int{"default": 1}, ""})
 185                 })
 186 }
 187
 188 func (s *StandaloneSuite) TestUploadToStubKeepServerBufferReader(c *C) {
 189         st := &StubPutHandler{
 190                 c:                    c,
 191                 expectPath:           "acbd18db4cc2f85cedef654fccc4a4d8",
 192                 expectAPIToken:       "abc123",
 193                 expectBody:           "foo",
 194                 expectStorageClass:   "",
 195                 returnStorageClasses: "default=1",
 196                 handled:              make(chan string),
 197         }
 198
 199         UploadToStubHelper(c, st,
 200                 func(kc *KeepClient, url string, _ io.ReadCloser, _ io.WriteCloser, uploadStatusChan chan uploadStatus) {
 201                         go kc.uploadToKeepServer(url, st.expectPath, nil, bytes.NewBuffer([]byte("foo")), uploadStatusChan, 3, kc.getRequestID())
 202
 203                         <-st.handled
 204
 205                         status := <-uploadStatusChan
 206                         c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, map[string]int{"default": 1}, ""})
 207                 })
 208 }
 209
 210 func (s *StandaloneSuite) TestUploadWithStorageClasses(c *C) {
 211         for _, trial := range []struct {
 212                 respHeader string
 213                 expectMap  map[string]int
 214         }{
 215                 {"", nil},
 216                 {"foo=1", map[string]int{"foo": 1}},
 217                 {" foo=1 , bar=2 ", map[string]int{"foo": 1, "bar": 2}},
 218                 {" =foo=1 ", nil},
 219                 {"foo", nil},
 220         } {
 221                 st := &StubPutHandler{
 222                         c:                    c,
 223                         expectPath:           "acbd18db4cc2f85cedef654fccc4a4d8",
 224                         expectAPIToken:       "abc123",
 225                         expectBody:           "foo",
 226                         expectStorageClass:   "",
 227                         returnStorageClasses: trial.respHeader,
 228                         handled:              make(chan string),
 229                 }
 230
 231                 UploadToStubHelper(c, st,
 232                         func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, uploadStatusChan chan uploadStatus) {
 233                                 go kc.uploadToKeepServer(url, st.expectPath, nil, reader, uploadStatusChan, len("foo"), kc.getRequestID())
 234
 235                                 writer.Write([]byte("foo"))
 236                                 writer.Close()
 237
 238                                 <-st.handled
 239                                 status := <-uploadStatusChan
 240                                 c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, trial.expectMap, ""})
 241                         })
 242         }
 243 }
 244
 245 func (s *StandaloneSuite) TestPutWithStorageClasses(c *C) {
 246         nServers := 5
 247         for _, trial := range []struct {
 248                 replicas      int
 249                 clientClasses []string
 250                 putClasses    []string // putClasses takes precedence over clientClasses
 251                 minRequests   int
 252                 maxRequests   int
 253                 success       bool
 254         }{
 255                 {1, []string{"class1"}, nil, 1, 1, true},
 256                 {2, []string{"class1"}, nil, 1, 2, true},
 257                 {3, []string{"class1"}, nil, 2, 3, true},
 258                 {1, []string{"class1", "class2"}, nil, 1, 1, true},
 259                 {3, nil, []string{"class1"}, 2, 3, true},
 260                 {1, nil, []string{"class1", "class2"}, 1, 1, true},
 261                 {1, []string{"class404"}, []string{"class1", "class2"}, 1, 1, true},
 262                 {1, []string{"class1"}, []string{"class404", "class2"}, nServers, nServers, false},
 263                 {nServers*2 + 1, []string{"class1"}, nil, nServers, nServers, false},
 264                 {1, []string{"class404"}, nil, nServers, nServers, false},
 265                 {1, []string{"class1", "class404"}, nil, nServers, nServers, false},
 266                 {1, nil, []string{"class1", "class404"}, nServers, nServers, false},
 267         } {
 268                 c.Logf("%+v", trial)
 269                 st := &StubPutHandler{
 270                         c:                    c,
 271                         expectPath:           "acbd18db4cc2f85cedef654fccc4a4d8",
 272                         expectAPIToken:       "abc123",
 273                         expectBody:           "foo",
 274                         expectStorageClass:   "*",
 275                         returnStorageClasses: "class1=2, class2=2",
 276                         handled:              make(chan string, 100),
 277                 }
 278                 ks := RunSomeFakeKeepServers(st, nServers)
 279                 arv, _ := arvadosclient.MakeArvadosClient()
 280                 kc, _ := MakeKeepClient(arv)
 281                 kc.Want_replicas = trial.replicas
 282                 kc.StorageClasses = trial.clientClasses
 283                 arv.ApiToken = "abc123"
 284                 localRoots := make(map[string]string)
 285                 writableLocalRoots := make(map[string]string)
 286                 for i, k := range ks {
 287                         localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 288                         writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 289                         defer k.listener.Close()
 290                 }
 291                 kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
 292
 293                 _, err := kc.BlockWrite(context.Background(), arvados.BlockWriteOptions{
 294                         Data:           []byte("foo"),
 295                         StorageClasses: trial.putClasses,
 296                 })
 297                 if trial.success {
 298                         c.Check(err, check.IsNil)
 299                 } else {
 300                         c.Check(err, check.NotNil)
 301                 }
 302                 c.Check(len(st.handled) >= trial.minRequests, check.Equals, true, check.Commentf("len(st.handled)==%d, trial.minRequests==%d", len(st.handled), trial.minRequests))
 303                 c.Check(len(st.handled) <= trial.maxRequests, check.Equals, true, check.Commentf("len(st.handled)==%d, trial.maxRequests==%d", len(st.handled), trial.maxRequests))
 304                 if !trial.success && trial.replicas == 1 && c.Check(len(st.requests) >= 2, check.Equals, true) {
 305                         // Max concurrency should be 1. First request
 306                         // should have succeeded for class1. Second
 307                         // request should only ask for class404.
 308                         c.Check(st.requests[1].Header.Get("X-Keep-Storage-Classes"), check.Equals, "class404")
 309                 }
 310         }
 311 }
 312
 313 type FailHandler struct {
 314         handled chan string
 315 }
 316
 317 func (fh FailHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 318         resp.WriteHeader(500)
 319         fh.handled <- fmt.Sprintf("http://%s", req.Host)
 320 }
 321
 322 type FailThenSucceedHandler struct {
 323         handled        chan string
 324         count          int
 325         successhandler http.Handler
 326         reqIDs         []string
 327 }
 328
 329 func (fh *FailThenSucceedHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 330         fh.reqIDs = append(fh.reqIDs, req.Header.Get("X-Request-Id"))
 331         if fh.count == 0 {
 332                 resp.WriteHeader(500)
 333                 fh.count++
 334                 fh.handled <- fmt.Sprintf("http://%s", req.Host)
 335         } else {
 336                 fh.successhandler.ServeHTTP(resp, req)
 337         }
 338 }
 339
 340 type Error404Handler struct {
 341         handled chan string
 342 }
 343
 344 func (fh Error404Handler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 345         resp.WriteHeader(404)
 346         fh.handled <- fmt.Sprintf("http://%s", req.Host)
 347 }
 348
 349 func (s *StandaloneSuite) TestFailedUploadToStubKeepServer(c *C) {
 350         st := FailHandler{
 351                 make(chan string)}
 352
 353         hash := "acbd18db4cc2f85cedef654fccc4a4d8"
 354
 355         UploadToStubHelper(c, st,
 356                 func(kc *KeepClient, url string, reader io.ReadCloser,
 357                         writer io.WriteCloser, uploadStatusChan chan uploadStatus) {
 358
 359                         go kc.uploadToKeepServer(url, hash, nil, reader, uploadStatusChan, 3, kc.getRequestID())
 360
 361                         writer.Write([]byte("foo"))
 362                         writer.Close()
 363
 364                         <-st.handled
 365
 366                         status := <-uploadStatusChan
 367                         c.Check(status.url, Equals, fmt.Sprintf("%s/%s", url, hash))
 368                         c.Check(status.statusCode, Equals, 500)
 369                 })
 370 }
 371
 372 type KeepServer struct {
 373         listener net.Listener
 374         url      string
 375 }
 376
 377 func RunSomeFakeKeepServers(st http.Handler, n int) (ks []KeepServer) {
 378         ks = make([]KeepServer, n)
 379
 380         for i := 0; i < n; i++ {
 381                 ks[i] = RunFakeKeepServer(st)
 382         }
 383
 384         return ks
 385 }
 386
 387 func (s *StandaloneSuite) TestPutB(c *C) {
 388         hash := Md5String("foo")
 389
 390         st := &StubPutHandler{
 391                 c:                    c,
 392                 expectPath:           hash,
 393                 expectAPIToken:       "abc123",
 394                 expectBody:           "foo",
 395                 expectStorageClass:   "",
 396                 returnStorageClasses: "",
 397                 handled:              make(chan string, 5),
 398         }
 399
 400         arv, _ := arvadosclient.MakeArvadosClient()
 401         kc, _ := MakeKeepClient(arv)
 402
 403         kc.Want_replicas = 2
 404         arv.ApiToken = "abc123"
 405         localRoots := make(map[string]string)
 406         writableLocalRoots := make(map[string]string)
 407
 408         ks := RunSomeFakeKeepServers(st, 5)
 409
 410         for i, k := range ks {
 411                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 412                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 413                 defer k.listener.Close()
 414         }
 415
 416         kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
 417
 418         kc.PutB([]byte("foo"))
 419
 420         shuff := NewRootSorter(
 421                 kc.LocalRoots(), Md5String("foo")).GetSortedRoots()
 422
 423         s1 := <-st.handled
 424         s2 := <-st.handled
 425         c.Check((s1 == shuff[0] && s2 == shuff[1]) ||
 426                 (s1 == shuff[1] && s2 == shuff[0]),
 427                 Equals,
 428                 true)
 429 }
 430
 431 func (s *StandaloneSuite) TestPutHR(c *C) {
 432         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 433
 434         st := &StubPutHandler{
 435                 c:                    c,
 436                 expectPath:           hash,
 437                 expectAPIToken:       "abc123",
 438                 expectBody:           "foo",
 439                 expectStorageClass:   "",
 440                 returnStorageClasses: "",
 441                 handled:              make(chan string, 5),
 442         }
 443
 444         arv, _ := arvadosclient.MakeArvadosClient()
 445         kc, _ := MakeKeepClient(arv)
 446
 447         kc.Want_replicas = 2
 448         arv.ApiToken = "abc123"
 449         localRoots := make(map[string]string)
 450         writableLocalRoots := make(map[string]string)
 451
 452         ks := RunSomeFakeKeepServers(st, 5)
 453
 454         for i, k := range ks {
 455                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 456                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 457                 defer k.listener.Close()
 458         }
 459
 460         kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
 461
 462         reader, writer := io.Pipe()
 463
 464         go func() {
 465                 writer.Write([]byte("foo"))
 466                 writer.Close()
 467         }()
 468
 469         kc.PutHR(hash, reader, 3)
 470
 471         shuff := NewRootSorter(kc.LocalRoots(), hash).GetSortedRoots()
 472
 473         s1 := <-st.handled
 474         s2 := <-st.handled
 475
 476         c.Check((s1 == shuff[0] && s2 == shuff[1]) ||
 477                 (s1 == shuff[1] && s2 == shuff[0]),
 478                 Equals,
 479                 true)
 480 }
 481
 482 func (s *StandaloneSuite) TestPutWithFail(c *C) {
 483         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 484
 485         st := &StubPutHandler{
 486                 c:                    c,
 487                 expectPath:           hash,
 488                 expectAPIToken:       "abc123",
 489                 expectBody:           "foo",
 490                 expectStorageClass:   "",
 491                 returnStorageClasses: "",
 492                 handled:              make(chan string, 4),
 493         }
 494
 495         fh := FailHandler{
 496                 make(chan string, 1)}
 497
 498         arv, err := arvadosclient.MakeArvadosClient()
 499         c.Check(err, IsNil)
 500         kc, _ := MakeKeepClient(arv)
 501
 502         kc.Want_replicas = 2
 503         arv.ApiToken = "abc123"
 504         localRoots := make(map[string]string)
 505         writableLocalRoots := make(map[string]string)
 506
 507         ks1 := RunSomeFakeKeepServers(st, 4)
 508         ks2 := RunSomeFakeKeepServers(fh, 1)
 509
 510         for i, k := range ks1 {
 511                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 512                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 513                 defer k.listener.Close()
 514         }
 515         for i, k := range ks2 {
 516                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
 517                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
 518                 defer k.listener.Close()
 519         }
 520
 521         kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
 522
 523         shuff := NewRootSorter(
 524                 kc.LocalRoots(), Md5String("foo")).GetSortedRoots()
 525         c.Logf("%+v", shuff)
 526
 527         phash, replicas, err := kc.PutB([]byte("foo"))
 528
 529         <-fh.handled
 530
 531         c.Check(err, Equals, nil)
 532         c.Check(phash, Equals, "")
 533         c.Check(replicas, Equals, 2)
 534
 535         s1 := <-st.handled
 536         s2 := <-st.handled
 537
 538         c.Check((s1 == shuff[1] && s2 == shuff[2]) ||
 539                 (s1 == shuff[2] && s2 == shuff[1]),
 540                 Equals,
 541                 true)
 542 }
 543
 544 func (s *StandaloneSuite) TestPutWithTooManyFail(c *C) {
 545         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 546
 547         st := &StubPutHandler{
 548                 c:                    c,
 549                 expectPath:           hash,
 550                 expectAPIToken:       "abc123",
 551                 expectBody:           "foo",
 552                 expectStorageClass:   "",
 553                 returnStorageClasses: "",
 554                 handled:              make(chan string, 1),
 555         }
 556
 557         fh := FailHandler{
 558                 make(chan string, 4)}
 559
 560         arv, err := arvadosclient.MakeArvadosClient()
 561         c.Check(err, IsNil)
 562         kc, _ := MakeKeepClient(arv)
 563
 564         kc.Want_replicas = 2
 565         kc.Retries = 0
 566         arv.ApiToken = "abc123"
 567         localRoots := make(map[string]string)
 568         writableLocalRoots := make(map[string]string)
 569
 570         ks1 := RunSomeFakeKeepServers(st, 1)
 571         ks2 := RunSomeFakeKeepServers(fh, 4)
 572
 573         for i, k := range ks1 {
 574                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 575                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 576                 defer k.listener.Close()
 577         }
 578         for i, k := range ks2 {
 579                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
 580                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
 581                 defer k.listener.Close()
 582         }
 583
 584         kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
 585
 586         _, replicas, err := kc.PutB([]byte("foo"))
 587
 588         c.Check(err, FitsTypeOf, InsufficientReplicasError{})
 589         c.Check(replicas, Equals, 1)
 590         c.Check(<-st.handled, Equals, ks1[0].url)
 591 }
 592
 593 type StubGetHandler struct {
 594         c              *C
 595         expectPath     string
 596         expectAPIToken string
 597         httpStatus     int
 598         body           []byte
 599 }
 600
 601 func (sgh StubGetHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 602         sgh.c.Check(req.URL.Path, Equals, "/"+sgh.expectPath)
 603         sgh.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sgh.expectAPIToken))
 604         resp.WriteHeader(sgh.httpStatus)
 605         resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(sgh.body)))
 606         resp.Write(sgh.body)
 607 }
 608
 609 func (s *StandaloneSuite) TestGet(c *C) {
 610         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 611
 612         st := StubGetHandler{
 613                 c,
 614                 hash,
 615                 "abc123",
 616                 http.StatusOK,
 617                 []byte("foo")}
 618
 619         ks := RunFakeKeepServer(st)
 620         defer ks.listener.Close()
 621
 622         arv, err := arvadosclient.MakeArvadosClient()
 623         c.Check(err, IsNil)
 624         kc, _ := MakeKeepClient(arv)
 625         arv.ApiToken = "abc123"
 626         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
 627
 628         r, n, url2, err := kc.Get(hash)
 629         defer r.Close()
 630         c.Check(err, Equals, nil)
 631         c.Check(n, Equals, int64(3))
 632         c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks.url, hash))
 633
 634         content, err2 := ioutil.ReadAll(r)
 635         c.Check(err2, Equals, nil)
 636         c.Check(content, DeepEquals, []byte("foo"))
 637 }
 638
 639 func (s *StandaloneSuite) TestGet404(c *C) {
 640         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 641
 642         st := Error404Handler{make(chan string, 1)}
 643
 644         ks := RunFakeKeepServer(st)
 645         defer ks.listener.Close()
 646
 647         arv, err := arvadosclient.MakeArvadosClient()
 648         c.Check(err, IsNil)
 649         kc, _ := MakeKeepClient(arv)
 650         arv.ApiToken = "abc123"
 651         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
 652
 653         r, n, url2, err := kc.Get(hash)
 654         c.Check(err, Equals, BlockNotFound)
 655         c.Check(n, Equals, int64(0))
 656         c.Check(url2, Equals, "")
 657         c.Check(r, Equals, nil)
 658 }
 659
 660 func (s *StandaloneSuite) TestGetEmptyBlock(c *C) {
 661         st := Error404Handler{make(chan string, 1)}
 662
 663         ks := RunFakeKeepServer(st)
 664         defer ks.listener.Close()
 665
 666         arv, err := arvadosclient.MakeArvadosClient()
 667         c.Check(err, IsNil)
 668         kc, _ := MakeKeepClient(arv)
 669         arv.ApiToken = "abc123"
 670         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
 671
 672         r, n, url2, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e+0")
 673         c.Check(err, IsNil)
 674         c.Check(n, Equals, int64(0))
 675         c.Check(url2, Equals, "")
 676         c.Assert(r, NotNil)
 677         buf, err := ioutil.ReadAll(r)
 678         c.Check(err, IsNil)
 679         c.Check(buf, DeepEquals, []byte{})
 680 }
 681
 682 func (s *StandaloneSuite) TestGetFail(c *C) {
 683         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 684
 685         st := FailHandler{make(chan string, 1)}
 686
 687         ks := RunFakeKeepServer(st)
 688         defer ks.listener.Close()
 689
 690         arv, err := arvadosclient.MakeArvadosClient()
 691         c.Check(err, IsNil)
 692         kc, _ := MakeKeepClient(arv)
 693         arv.ApiToken = "abc123"
 694         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
 695         kc.Retries = 0
 696
 697         r, n, url2, err := kc.Get(hash)
 698         errNotFound, _ := err.(*ErrNotFound)
 699         c.Check(errNotFound, NotNil)
 700         c.Check(strings.Contains(errNotFound.Error(), "HTTP 500"), Equals, true)
 701         c.Check(errNotFound.Temporary(), Equals, true)
 702         c.Check(n, Equals, int64(0))
 703         c.Check(url2, Equals, "")
 704         c.Check(r, Equals, nil)
 705 }
 706
 707 func (s *StandaloneSuite) TestGetFailRetry(c *C) {
 708         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 709
 710         st := &FailThenSucceedHandler{
 711                 handled: make(chan string, 1),
 712                 successhandler: StubGetHandler{
 713                         c,
 714                         hash,
 715                         "abc123",
 716                         http.StatusOK,
 717                         []byte("foo")}}
 718
 719         ks := RunFakeKeepServer(st)
 720         defer ks.listener.Close()
 721
 722         arv, err := arvadosclient.MakeArvadosClient()
 723         c.Check(err, IsNil)
 724         kc, _ := MakeKeepClient(arv)
 725         arv.ApiToken = "abc123"
 726         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
 727
 728         r, n, url2, err := kc.Get(hash)
 729         defer r.Close()
 730         c.Check(err, Equals, nil)
 731         c.Check(n, Equals, int64(3))
 732         c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks.url, hash))
 733
 734         content, err2 := ioutil.ReadAll(r)
 735         c.Check(err2, Equals, nil)
 736         c.Check(content, DeepEquals, []byte("foo"))
 737
 738         c.Logf("%q", st.reqIDs)
 739         c.Assert(len(st.reqIDs) > 1, Equals, true)
 740         for _, reqid := range st.reqIDs {
 741                 c.Check(reqid, Not(Equals), "")
 742                 c.Check(reqid, Equals, st.reqIDs[0])
 743         }
 744 }
 745
 746 func (s *StandaloneSuite) TestGetNetError(c *C) {
 747         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 748
 749         arv, err := arvadosclient.MakeArvadosClient()
 750         c.Check(err, IsNil)
 751         kc, _ := MakeKeepClient(arv)
 752         arv.ApiToken = "abc123"
 753         kc.SetServiceRoots(map[string]string{"x": "http://localhost:62222"}, nil, nil)
 754
 755         r, n, url2, err := kc.Get(hash)
 756         errNotFound, _ := err.(*ErrNotFound)
 757         c.Check(errNotFound, NotNil)
 758         c.Check(strings.Contains(errNotFound.Error(), "connection refused"), Equals, true)
 759         c.Check(errNotFound.Temporary(), Equals, true)
 760         c.Check(n, Equals, int64(0))
 761         c.Check(url2, Equals, "")
 762         c.Check(r, Equals, nil)
 763 }
 764
 765 func (s *StandaloneSuite) TestGetWithServiceHint(c *C) {
 766         uuid := "zzzzz-bi6l4-123451234512345"
 767         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 768
 769         // This one shouldn't be used:
 770         ks0 := RunFakeKeepServer(StubGetHandler{
 771                 c,
 772                 "error if used",
 773                 "abc123",
 774                 http.StatusOK,
 775                 []byte("foo")})
 776         defer ks0.listener.Close()
 777         // This one should be used:
 778         ks := RunFakeKeepServer(StubGetHandler{
 779                 c,
 780                 hash + "+K@" + uuid,
 781                 "abc123",
 782                 http.StatusOK,
 783                 []byte("foo")})
 784         defer ks.listener.Close()
 785
 786         arv, err := arvadosclient.MakeArvadosClient()
 787         c.Check(err, IsNil)
 788         kc, _ := MakeKeepClient(arv)
 789         arv.ApiToken = "abc123"
 790         kc.SetServiceRoots(
 791                 map[string]string{"x": ks0.url},
 792                 nil,
 793                 map[string]string{uuid: ks.url})
 794
 795         r, n, uri, err := kc.Get(hash + "+K@" + uuid)
 796         defer r.Close()
 797         c.Check(err, Equals, nil)
 798         c.Check(n, Equals, int64(3))
 799         c.Check(uri, Equals, fmt.Sprintf("%s/%s", ks.url, hash+"+K@"+uuid))
 800
 801         content, err := ioutil.ReadAll(r)
 802         c.Check(err, Equals, nil)
 803         c.Check(content, DeepEquals, []byte("foo"))
 804 }
 805
 806 // Use a service hint to fetch from a local disk service, overriding
 807 // rendezvous probe order.
 808 func (s *StandaloneSuite) TestGetWithLocalServiceHint(c *C) {
 809         uuid := "zzzzz-bi6l4-zzzzzzzzzzzzzzz"
 810         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 811
 812         // This one shouldn't be used, although it appears first in
 813         // rendezvous probe order:
 814         ks0 := RunFakeKeepServer(StubGetHandler{
 815                 c,
 816                 "error if used",
 817                 "abc123",
 818                 http.StatusOK,
 819                 []byte("foo")})
 820         defer ks0.listener.Close()
 821         // This one should be used:
 822         ks := RunFakeKeepServer(StubGetHandler{
 823                 c,
 824                 hash + "+K@" + uuid,
 825                 "abc123",
 826                 http.StatusOK,
 827                 []byte("foo")})
 828         defer ks.listener.Close()
 829
 830         arv, err := arvadosclient.MakeArvadosClient()
 831         c.Check(err, IsNil)
 832         kc, _ := MakeKeepClient(arv)
 833         arv.ApiToken = "abc123"
 834         kc.SetServiceRoots(
 835                 map[string]string{
 836                         "zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url,
 837                         "zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url,
 838                         "zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url,
 839                         uuid:                          ks.url},
 840                 nil,
 841                 map[string]string{
 842                         "zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url,
 843                         "zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url,
 844                         "zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url,
 845                         uuid:                          ks.url},
 846         )
 847
 848         r, n, uri, err := kc.Get(hash + "+K@" + uuid)
 849         defer r.Close()
 850         c.Check(err, Equals, nil)
 851         c.Check(n, Equals, int64(3))
 852         c.Check(uri, Equals, fmt.Sprintf("%s/%s", ks.url, hash+"+K@"+uuid))
 853
 854         content, err := ioutil.ReadAll(r)
 855         c.Check(err, Equals, nil)
 856         c.Check(content, DeepEquals, []byte("foo"))
 857 }
 858
 859 func (s *StandaloneSuite) TestGetWithServiceHintFailoverToLocals(c *C) {
 860         uuid := "zzzzz-bi6l4-123451234512345"
 861         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 862
 863         ksLocal := RunFakeKeepServer(StubGetHandler{
 864                 c,
 865                 hash + "+K@" + uuid,
 866                 "abc123",
 867                 http.StatusOK,
 868                 []byte("foo")})
 869         defer ksLocal.listener.Close()
 870         ksGateway := RunFakeKeepServer(StubGetHandler{
 871                 c,
 872                 hash + "+K@" + uuid,
 873                 "abc123",
 874                 http.StatusInternalServerError,
 875                 []byte("Error")})
 876         defer ksGateway.listener.Close()
 877
 878         arv, err := arvadosclient.MakeArvadosClient()
 879         c.Check(err, IsNil)
 880         kc, _ := MakeKeepClient(arv)
 881         arv.ApiToken = "abc123"
 882         kc.SetServiceRoots(
 883                 map[string]string{"zzzzz-bi6l4-keepdisk0000000": ksLocal.url},
 884                 nil,
 885                 map[string]string{uuid: ksGateway.url})
 886
 887         r, n, uri, err := kc.Get(hash + "+K@" + uuid)
 888         c.Assert(err, Equals, nil)
 889         defer r.Close()
 890         c.Check(n, Equals, int64(3))
 891         c.Check(uri, Equals, fmt.Sprintf("%s/%s", ksLocal.url, hash+"+K@"+uuid))
 892
 893         content, err := ioutil.ReadAll(r)
 894         c.Check(err, Equals, nil)
 895         c.Check(content, DeepEquals, []byte("foo"))
 896 }
 897
 898 type BarHandler struct {
 899         handled chan string
 900 }
 901
 902 func (h BarHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 903         resp.Write([]byte("bar"))
 904         h.handled <- fmt.Sprintf("http://%s", req.Host)
 905 }
 906
 907 func (s *StandaloneSuite) TestChecksum(c *C) {
 908         foohash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 909         barhash := fmt.Sprintf("%x", md5.Sum([]byte("bar")))
 910
 911         st := BarHandler{make(chan string, 1)}
 912
 913         ks := RunFakeKeepServer(st)
 914         defer ks.listener.Close()
 915
 916         arv, err := arvadosclient.MakeArvadosClient()
 917         c.Check(err, IsNil)
 918         kc, _ := MakeKeepClient(arv)
 919         arv.ApiToken = "abc123"
 920         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
 921
 922         r, n, _, err := kc.Get(barhash)
 923         c.Check(err, IsNil)
 924         _, err = ioutil.ReadAll(r)
 925         c.Check(n, Equals, int64(3))
 926         c.Check(err, Equals, nil)
 927
 928         <-st.handled
 929
 930         r, n, _, err = kc.Get(foohash)
 931         c.Check(err, IsNil)
 932         _, err = ioutil.ReadAll(r)
 933         c.Check(n, Equals, int64(3))
 934         c.Check(err, Equals, BadChecksum)
 935
 936         <-st.handled
 937 }
 938
 939 func (s *StandaloneSuite) TestGetWithFailures(c *C) {
 940         content := []byte("waz")
 941         hash := fmt.Sprintf("%x", md5.Sum(content))
 942
 943         fh := Error404Handler{
 944                 make(chan string, 4)}
 945
 946         st := StubGetHandler{
 947                 c,
 948                 hash,
 949                 "abc123",
 950                 http.StatusOK,
 951                 content}
 952
 953         arv, err := arvadosclient.MakeArvadosClient()
 954         c.Check(err, IsNil)
 955         kc, _ := MakeKeepClient(arv)
 956         arv.ApiToken = "abc123"
 957         localRoots := make(map[string]string)
 958         writableLocalRoots := make(map[string]string)
 959
 960         ks1 := RunSomeFakeKeepServers(st, 1)
 961         ks2 := RunSomeFakeKeepServers(fh, 4)
 962
 963         for i, k := range ks1 {
 964                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 965                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
 966                 defer k.listener.Close()
 967         }
 968         for i, k := range ks2 {
 969                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
 970                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
 971                 defer k.listener.Close()
 972         }
 973
 974         kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
 975         kc.Retries = 0
 976
 977         // This test works only if one of the failing services is
 978         // attempted before the succeeding service. Otherwise,
 979         // <-fh.handled below will just hang! (Probe order depends on
 980         // the choice of block content "waz" and the UUIDs of the fake
 981         // servers, so we just tried different strings until we found
 982         // an example that passes this Assert.)
 983         c.Assert(NewRootSorter(localRoots, hash).GetSortedRoots()[0], Not(Equals), ks1[0].url)
 984
 985         r, n, url2, err := kc.Get(hash)
 986
 987         <-fh.handled
 988         c.Check(err, Equals, nil)
 989         c.Check(n, Equals, int64(3))
 990         c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks1[0].url, hash))
 991
 992         readContent, err2 := ioutil.ReadAll(r)
 993         c.Check(err2, Equals, nil)
 994         c.Check(readContent, DeepEquals, content)
 995 }
 996
 997 func (s *ServerRequiredSuite) TestPutGetHead(c *C) {
 998         content := []byte("TestPutGetHead")
 999
1000         arv, err := arvadosclient.MakeArvadosClient()
1001         c.Check(err, IsNil)
1002         kc, err := MakeKeepClient(arv)
1003         c.Assert(err, Equals, nil)
1004
1005         hash := fmt.Sprintf("%x", md5.Sum(content))
1006
1007         {
1008                 n, _, err := kc.Ask(hash)
1009                 c.Check(err, Equals, BlockNotFound)
1010                 c.Check(n, Equals, int64(0))
1011         }
1012         {
1013                 hash2, replicas, err := kc.PutB(content)
1014                 c.Check(hash2, Matches, fmt.Sprintf(`%s\+%d\b.*`, hash, len(content)))
1015                 c.Check(replicas, Equals, 2)
1016                 c.Check(err, Equals, nil)
1017         }
1018         {
1019                 r, n, url2, err := kc.Get(hash)
1020                 c.Check(err, Equals, nil)
1021                 c.Check(n, Equals, int64(len(content)))
1022                 c.Check(url2, Matches, fmt.Sprintf("http://localhost:\\d+/%s", hash))
1023
1024                 readContent, err2 := ioutil.ReadAll(r)
1025                 c.Check(err2, Equals, nil)
1026                 c.Check(readContent, DeepEquals, content)
1027         }
1028         {
1029                 n, url2, err := kc.Ask(hash)
1030                 c.Check(err, Equals, nil)
1031                 c.Check(n, Equals, int64(len(content)))
1032                 c.Check(url2, Matches, fmt.Sprintf("http://localhost:\\d+/%s", hash))
1033         }
1034         {
1035                 loc, err := kc.LocalLocator(hash)
1036                 c.Check(err, Equals, nil)
1037                 c.Assert(len(loc) >= 32, Equals, true)
1038                 c.Check(loc[:32], Equals, hash[:32])
1039         }
1040         {
1041                 content := []byte("the perth county conspiracy")
1042                 loc, err := kc.LocalLocator(fmt.Sprintf("%x+%d+Rzaaaa-abcde@12345", md5.Sum(content), len(content)))
1043                 c.Check(loc, Equals, "")
1044                 c.Check(err, ErrorMatches, `.*HEAD .*\+R.*`)
1045                 c.Check(err, ErrorMatches, `.*HTTP 400.*`)
1046         }
1047 }
1048
1049 type StubProxyHandler struct {
1050         handled chan string
1051 }
1052
1053 func (h StubProxyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
1054         resp.Header().Set("X-Keep-Replicas-Stored", "2")
1055         h.handled <- fmt.Sprintf("http://%s", req.Host)
1056 }
1057
1058 func (s *StandaloneSuite) TestPutProxy(c *C) {
1059         st := StubProxyHandler{make(chan string, 1)}
1060
1061         arv, err := arvadosclient.MakeArvadosClient()
1062         c.Check(err, IsNil)
1063         kc, _ := MakeKeepClient(arv)
1064
1065         kc.Want_replicas = 2
1066         arv.ApiToken = "abc123"
1067         localRoots := make(map[string]string)
1068         writableLocalRoots := make(map[string]string)
1069
1070         ks1 := RunSomeFakeKeepServers(st, 1)
1071
1072         for i, k := range ks1 {
1073                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
1074                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
1075                 defer k.listener.Close()
1076         }
1077
1078         kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
1079
1080         _, replicas, err := kc.PutB([]byte("foo"))
1081         <-st.handled
1082
1083         c.Check(err, Equals, nil)
1084         c.Check(replicas, Equals, 2)
1085 }
1086
1087 func (s *StandaloneSuite) TestPutProxyInsufficientReplicas(c *C) {
1088         st := StubProxyHandler{make(chan string, 1)}
1089
1090         arv, err := arvadosclient.MakeArvadosClient()
1091         c.Check(err, IsNil)
1092         kc, _ := MakeKeepClient(arv)
1093
1094         kc.Want_replicas = 3
1095         arv.ApiToken = "abc123"
1096         localRoots := make(map[string]string)
1097         writableLocalRoots := make(map[string]string)
1098
1099         ks1 := RunSomeFakeKeepServers(st, 1)
1100
1101         for i, k := range ks1 {
1102                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
1103                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
1104                 defer k.listener.Close()
1105         }
1106         kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
1107
1108         _, replicas, err := kc.PutB([]byte("foo"))
1109         <-st.handled
1110
1111         c.Check(err, FitsTypeOf, InsufficientReplicasError{})
1112         c.Check(replicas, Equals, 2)
1113 }
1114
1115 func (s *StandaloneSuite) TestMakeLocator(c *C) {
1116         l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce+3+Aabcde@12345678")
1117         c.Check(err, Equals, nil)
1118         c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
1119         c.Check(l.Size, Equals, 3)
1120         c.Check(l.Hints, DeepEquals, []string{"3", "Aabcde@12345678"})
1121 }
1122
1123 func (s *StandaloneSuite) TestMakeLocatorNoHints(c *C) {
1124         l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce")
1125         c.Check(err, Equals, nil)
1126         c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
1127         c.Check(l.Size, Equals, -1)
1128         c.Check(l.Hints, DeepEquals, []string{})
1129 }
1130
1131 func (s *StandaloneSuite) TestMakeLocatorNoSizeHint(c *C) {
1132         l, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31ce+Aabcde@12345678")
1133         c.Check(err, Equals, nil)
1134         c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
1135         c.Check(l.Size, Equals, -1)
1136         c.Check(l.Hints, DeepEquals, []string{"Aabcde@12345678"})
1137 }
1138
1139 func (s *StandaloneSuite) TestMakeLocatorPreservesUnrecognizedHints(c *C) {
1140         str := "91f372a266fe2bf2823cb8ec7fda31ce+3+Unknown+Kzzzzz+Afoobar"
1141         l, err := MakeLocator(str)
1142         c.Check(err, Equals, nil)
1143         c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
1144         c.Check(l.Size, Equals, 3)
1145         c.Check(l.Hints, DeepEquals, []string{"3", "Unknown", "Kzzzzz", "Afoobar"})
1146         c.Check(l.String(), Equals, str)
1147 }
1148
1149 func (s *StandaloneSuite) TestMakeLocatorInvalidInput(c *C) {
1150         _, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31c")
1151         c.Check(err, Equals, InvalidLocatorError)
1152 }
1153
1154 func (s *StandaloneSuite) TestPutBWant2ReplicasWithOnlyOneWritableLocalRoot(c *C) {
1155         hash := Md5String("foo")
1156
1157         st := &StubPutHandler{
1158                 c:                    c,
1159                 expectPath:           hash,
1160                 expectAPIToken:       "abc123",
1161                 expectBody:           "foo",
1162                 expectStorageClass:   "",
1163                 returnStorageClasses: "",
1164                 handled:              make(chan string, 5),
1165         }
1166
1167         arv, _ := arvadosclient.MakeArvadosClient()
1168         kc, _ := MakeKeepClient(arv)
1169
1170         kc.Want_replicas = 2
1171         arv.ApiToken = "abc123"
1172         localRoots := make(map[string]string)
1173         writableLocalRoots := make(map[string]string)
1174
1175         ks := RunSomeFakeKeepServers(st, 5)
1176
1177         for i, k := range ks {
1178                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
1179                 if i == 0 {
1180                         writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
1181                 }
1182                 defer k.listener.Close()
1183         }
1184
1185         kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
1186
1187         _, replicas, err := kc.PutB([]byte("foo"))
1188
1189         c.Check(err, FitsTypeOf, InsufficientReplicasError{})
1190         c.Check(replicas, Equals, 1)
1191
1192         c.Check(<-st.handled, Equals, localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", 0)])
1193 }
1194
1195 func (s *StandaloneSuite) TestPutBWithNoWritableLocalRoots(c *C) {
1196         hash := Md5String("foo")
1197
1198         st := &StubPutHandler{
1199                 c:                    c,
1200                 expectPath:           hash,
1201                 expectAPIToken:       "abc123",
1202                 expectBody:           "foo",
1203                 expectStorageClass:   "",
1204                 returnStorageClasses: "",
1205                 handled:              make(chan string, 5),
1206         }
1207
1208         arv, _ := arvadosclient.MakeArvadosClient()
1209         kc, _ := MakeKeepClient(arv)
1210
1211         kc.Want_replicas = 2
1212         arv.ApiToken = "abc123"
1213         localRoots := make(map[string]string)
1214         writableLocalRoots := make(map[string]string)
1215
1216         ks := RunSomeFakeKeepServers(st, 5)
1217
1218         for i, k := range ks {
1219                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
1220                 defer k.listener.Close()
1221         }
1222
1223         kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
1224
1225         _, replicas, err := kc.PutB([]byte("foo"))
1226
1227         c.Check(err, FitsTypeOf, InsufficientReplicasError{})
1228         c.Check(replicas, Equals, 0)
1229 }
1230
1231 type StubGetIndexHandler struct {
1232         c              *C
1233         expectPath     string
1234         expectAPIToken string
1235         httpStatus     int
1236         body           []byte
1237 }
1238
1239 func (h StubGetIndexHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
1240         h.c.Check(req.URL.Path, Equals, h.expectPath)
1241         h.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", h.expectAPIToken))
1242         resp.WriteHeader(h.httpStatus)
1243         resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(h.body)))
1244         resp.Write(h.body)
1245 }
1246
1247 func (s *StandaloneSuite) TestGetIndexWithNoPrefix(c *C) {
1248         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
1249
1250         st := StubGetIndexHandler{
1251                 c,
1252                 "/index",
1253                 "abc123",
1254                 http.StatusOK,
1255                 []byte(hash + "+3 1443559274\n\n")}
1256
1257         ks := RunFakeKeepServer(st)
1258         defer ks.listener.Close()
1259
1260         arv, err := arvadosclient.MakeArvadosClient()
1261         c.Assert(err, IsNil)
1262         kc, err := MakeKeepClient(arv)
1263         c.Assert(err, IsNil)
1264         arv.ApiToken = "abc123"
1265         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
1266
1267         r, err := kc.GetIndex("x", "")
1268         c.Check(err, IsNil)
1269
1270         content, err2 := ioutil.ReadAll(r)
1271         c.Check(err2, Equals, nil)
1272         c.Check(content, DeepEquals, st.body[0:len(st.body)-1])
1273 }
1274
1275 func (s *StandaloneSuite) TestGetIndexWithPrefix(c *C) {
1276         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
1277
1278         st := StubGetIndexHandler{
1279                 c,
1280                 "/index/" + hash[0:3],
1281                 "abc123",
1282                 http.StatusOK,
1283                 []byte(hash + "+3 1443559274\n\n")}
1284
1285         ks := RunFakeKeepServer(st)
1286         defer ks.listener.Close()
1287
1288         arv, err := arvadosclient.MakeArvadosClient()
1289         c.Check(err, IsNil)
1290         kc, _ := MakeKeepClient(arv)
1291         arv.ApiToken = "abc123"
1292         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
1293
1294         r, err := kc.GetIndex("x", hash[0:3])
1295         c.Assert(err, Equals, nil)
1296
1297         content, err2 := ioutil.ReadAll(r)
1298         c.Check(err2, Equals, nil)
1299         c.Check(content, DeepEquals, st.body[0:len(st.body)-1])
1300 }
1301
1302 func (s *StandaloneSuite) TestGetIndexIncomplete(c *C) {
1303         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
1304
1305         st := StubGetIndexHandler{
1306                 c,
1307                 "/index/" + hash[0:3],
1308                 "abc123",
1309                 http.StatusOK,
1310                 []byte(hash)}
1311
1312         ks := RunFakeKeepServer(st)
1313         defer ks.listener.Close()
1314
1315         arv, err := arvadosclient.MakeArvadosClient()
1316         c.Check(err, IsNil)
1317         kc, _ := MakeKeepClient(arv)
1318         arv.ApiToken = "abc123"
1319         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
1320
1321         _, err = kc.GetIndex("x", hash[0:3])
1322         c.Check(err, Equals, ErrIncompleteIndex)
1323 }
1324
1325 func (s *StandaloneSuite) TestGetIndexWithNoSuchServer(c *C) {
1326         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
1327
1328         st := StubGetIndexHandler{
1329                 c,
1330                 "/index/" + hash[0:3],
1331                 "abc123",
1332                 http.StatusOK,
1333                 []byte(hash)}
1334
1335         ks := RunFakeKeepServer(st)
1336         defer ks.listener.Close()
1337
1338         arv, err := arvadosclient.MakeArvadosClient()
1339         c.Check(err, IsNil)
1340         kc, _ := MakeKeepClient(arv)
1341         arv.ApiToken = "abc123"
1342         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
1343
1344         _, err = kc.GetIndex("y", hash[0:3])
1345         c.Check(err, Equals, ErrNoSuchKeepServer)
1346 }
1347
1348 func (s *StandaloneSuite) TestGetIndexWithNoSuchPrefix(c *C) {
1349         st := StubGetIndexHandler{
1350                 c,
1351                 "/index/abcd",
1352                 "abc123",
1353                 http.StatusOK,
1354                 []byte("\n")}
1355
1356         ks := RunFakeKeepServer(st)
1357         defer ks.listener.Close()
1358
1359         arv, err := arvadosclient.MakeArvadosClient()
1360         c.Check(err, IsNil)
1361         kc, _ := MakeKeepClient(arv)
1362         arv.ApiToken = "abc123"
1363         kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
1364
1365         r, err := kc.GetIndex("x", "abcd")
1366         c.Check(err, Equals, nil)
1367
1368         content, err2 := ioutil.ReadAll(r)
1369         c.Check(err2, Equals, nil)
1370         c.Check(content, DeepEquals, st.body[0:len(st.body)-1])
1371 }
1372
1373 func (s *StandaloneSuite) TestPutBRetry(c *C) {
1374         st := &FailThenSucceedHandler{
1375                 handled: make(chan string, 1),
1376                 successhandler: &StubPutHandler{
1377                         c:                    c,
1378                         expectPath:           Md5String("foo"),
1379                         expectAPIToken:       "abc123",
1380                         expectBody:           "foo",
1381                         expectStorageClass:   "",
1382                         returnStorageClasses: "",
1383                         handled:              make(chan string, 5),
1384                 },
1385         }
1386
1387         arv, _ := arvadosclient.MakeArvadosClient()
1388         kc, _ := MakeKeepClient(arv)
1389
1390         kc.Want_replicas = 2
1391         arv.ApiToken = "abc123"
1392         localRoots := make(map[string]string)
1393         writableLocalRoots := make(map[string]string)
1394
1395         ks := RunSomeFakeKeepServers(st, 2)
1396
1397         for i, k := range ks {
1398                 localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
1399                 writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
1400                 defer k.listener.Close()
1401         }
1402
1403         kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
1404
1405         hash, replicas, err := kc.PutB([]byte("foo"))
1406
1407         c.Check(err, Equals, nil)
1408         c.Check(hash, Equals, "")
1409         c.Check(replicas, Equals, 2)
1410 }
1411
1412 func (s *ServerRequiredSuite) TestMakeKeepClientWithNonDiskTypeService(c *C) {
1413         arv, err := arvadosclient.MakeArvadosClient()
1414         c.Assert(err, Equals, nil)
1415
1416         // Add an additional "testblobstore" keepservice
1417         blobKeepService := make(arvadosclient.Dict)
1418         err = arv.Create("keep_services",
1419                 arvadosclient.Dict{"keep_service": arvadosclient.Dict{
1420                         "service_host": "localhost",
1421                         "service_port": "21321",
1422                         "service_type": "testblobstore"}},
1423                 &blobKeepService)
1424         c.Assert(err, Equals, nil)
1425         defer func() { arv.Delete("keep_services", blobKeepService["uuid"].(string), nil, nil) }()
1426         RefreshServiceDiscovery()
1427
1428         // Make a keepclient and ensure that the testblobstore is included
1429         kc, err := MakeKeepClient(arv)
1430         c.Assert(err, Equals, nil)
1431
1432         // verify kc.LocalRoots
1433         c.Check(len(kc.LocalRoots()), Equals, 3)
1434         for _, root := range kc.LocalRoots() {
1435                 c.Check(root, Matches, "http://localhost:\\d+")
1436         }
1437         c.Assert(kc.LocalRoots()[blobKeepService["uuid"].(string)], Not(Equals), "")
1438
1439         // verify kc.GatewayRoots
1440         c.Check(len(kc.GatewayRoots()), Equals, 3)
1441         for _, root := range kc.GatewayRoots() {
1442                 c.Check(root, Matches, "http://localhost:\\d+")
1443         }
1444         c.Assert(kc.GatewayRoots()[blobKeepService["uuid"].(string)], Not(Equals), "")
1445
1446         // verify kc.WritableLocalRoots
1447         c.Check(len(kc.WritableLocalRoots()), Equals, 3)
1448         for _, root := range kc.WritableLocalRoots() {
1449                 c.Check(root, Matches, "http://localhost:\\d+")
1450         }
1451         c.Assert(kc.WritableLocalRoots()[blobKeepService["uuid"].(string)], Not(Equals), "")
1452
1453         c.Assert(kc.replicasPerService, Equals, 0)
1454         c.Assert(kc.foundNonDiskSvc, Equals, true)
1455         c.Assert(kc.httpClient().(*http.Client).Timeout, Equals, 300*time.Second)
1456 }