14804: Exposes the bug with a test.

Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima@veritasgenetics.com>

diff --git a/services/keepstore/handler_test.go b/services/keepstore/handler_test.go
index c37a4d112fb8b86aaa076431f08524930ce83d0b..ac75cf10f810a419f2fc687e30b12fbb79080299 100644 (file)
--- a/services/keepstore/handler_test.go
+++ b/services/keepstore/handler_test.go
@@ -151,6 +151,23 @@ func TestGetHandler(t *testing.T) {
        ExpectStatusCode(t,
                "Authenticated request, expired locator",
                ExpiredError.HTTPCode, response)
+
+       // Authenticated request, signed locator
+       // => 503 Server busy (transient error)
+
+       // Set up the block owning volume to respond with errors
+       vols[0].(*MockVolume).Bad = true
+       vols[0].(*MockVolume).BadVolumeError = VolumeBusyError
+       response = IssueRequest(&RequestTester{
+               method:   "GET",
+               uri:      signedLocator,
+               apiToken: knownToken,
+       })
+       // A transient error from one volume while the other doesn't find the block
+       // should make the service return a 503 so that clients can retry.
+       ExpectStatusCode(t,
+               "Volume backend busy",
+               503, response)
 }
 
 // Test PutBlockHandler on the following situations:

diff --git a/services/keepstore/keepstore.go b/services/keepstore/keepstore.go
index 6ae414bf931ce9164f7beefcc0d9be294da6e9c5..a6c8cd99545c24fdc2a56f6c2ff1866682a6ed6d 100644 (file)
--- a/services/keepstore/keepstore.go
+++ b/services/keepstore/keepstore.go
@@ -50,6 +50,7 @@ var (
        DiskHashError       = &KeepError{500, "Hash mismatch in stored data"}
        ExpiredError        = &KeepError{401, "Expired permission signature"}
        NotFoundError       = &KeepError{404, "Not Found"}
+       VolumeBusyError     = &KeepError{503, "Volume backend busy"}
        GenericError        = &KeepError{500, "Fail"}
        FullError           = &KeepError{503, "Full"}
        SizeRequiredError   = &KeepError{411, "Missing Content-Length"}

diff --git a/services/keepstore/keepstore_test.go b/services/keepstore/keepstore_test.go
index 26d49946a4555647f88e41658d2f7a2f949c830b..d1d380466ba5983d4a7752c95ff47cf3e9312a75 100644 (file)
--- a/services/keepstore/keepstore_test.go
+++ b/services/keepstore/keepstore_test.go
@@ -7,6 +7,7 @@ package main
 import (
        "bytes"
        "context"
+       "errors"
        "fmt"
        "io/ioutil"
        "os"
@@ -165,6 +166,7 @@ func TestPutBlockOneVol(t *testing.T) {
 
        vols := KeepVM.AllWritable()
        vols[0].(*MockVolume).Bad = true
+       vols[0].(*MockVolume).BadVolumeError = errors.New("Bad volume")
 
        // Check that PutBlock stores the data as expected.
        if n, err := PutBlock(context.Background(), TestBlock, TestHash); err != nil || n < 1 {

diff --git a/services/keepstore/volume_test.go b/services/keepstore/volume_test.go
index 43ddd090cc1cfd22419e80aa86f1e838ffebd479..046f3fac2e0c8c27081c22fea69a0aae7f02acda 100644 (file)
--- a/services/keepstore/volume_test.go
+++ b/services/keepstore/volume_test.go
@@ -40,7 +40,8 @@ type MockVolume struct {
        Timestamps map[string]time.Time
 
        // Bad volumes return an error for every operation.
-       Bad bool
+       Bad            bool
+       BadVolumeError error
 
        // Touchable volumes' Touch() method succeeds for a locator
        // that has been Put().
@@ -104,7 +105,7 @@ func (v *MockVolume) Compare(ctx context.Context, loc string, buf []byte) error
        v.gotCall("Compare")
        <-v.Gate
        if v.Bad {
-               return errors.New("Bad volume")
+               return v.BadVolumeError
        } else if block, ok := v.Store[loc]; ok {
                if fmt.Sprintf("%x", md5.Sum(block)) != loc {
                        return DiskHashError
@@ -122,7 +123,7 @@ func (v *MockVolume) Get(ctx context.Context, loc string, buf []byte) (int, erro
        v.gotCall("Get")
        <-v.Gate
        if v.Bad {
-               return 0, errors.New("Bad volume")
+               return 0, v.BadVolumeError
        } else if block, ok := v.Store[loc]; ok {
                copy(buf[:len(block)], block)
                return len(block), nil
@@ -134,7 +135,7 @@ func (v *MockVolume) Put(ctx context.Context, loc string, block []byte) error {
        v.gotCall("Put")
        <-v.Gate
        if v.Bad {
-               return errors.New("Bad volume")
+               return v.BadVolumeError
        }
        if v.Readonly {
                return MethodDisabledError
@@ -162,7 +163,7 @@ func (v *MockVolume) Mtime(loc string) (time.Time, error) {
        var mtime time.Time
        var err error
        if v.Bad {
-               err = errors.New("Bad volume")
+               err = v.BadVolumeError
        } else if t, ok := v.Timestamps[loc]; ok {
                mtime = t
        } else {