10682: Track Azure backend errors by type.

diff --git a/services/keepstore/azure_blob_volume.go b/services/keepstore/azure_blob_volume.go
index 383759803ccdbe55169f63737651aadfcb11f7c7..220744c25c5ba829de41d330b52332e5864dfc4a 100644 (file)
--- a/services/keepstore/azure_blob_volume.go
+++ b/services/keepstore/azure_blob_volume.go
@@ -106,26 +106,6 @@ type AzureBlobVolume struct {
        bsClient *azureBlobClient
 }
 
-// azureBlobClient wraps storage.BlobStorageClient in order to count
-// I/O and API usage stats.
-type azureBlobClient struct {
-       client *storage.BlobStorageClient
-       stats  azureBlobStats
-}
-
-type azureBlobStats struct {
-       statsTicker
-       Ops            uint64
-       GetOps         uint64
-       GetRangeOps    uint64
-       CreateOps      uint64
-       SetMetadataOps uint64
-       DelOps         uint64
-       ListOps        uint64
-
-       lock sync.Mutex
-}
-
 // Examples implements VolumeWithExamples.
 func (*AzureBlobVolume) Examples() []Volume {
        return []Volume{
@@ -652,6 +632,36 @@ func (v *AzureBlobVolume) InternalStats() interface{} {
        return &v.bsClient.stats
 }
 
+type azureBlobStats struct {
+       statsTicker
+       Ops            uint64
+       GetOps         uint64
+       GetRangeOps    uint64
+       CreateOps      uint64
+       SetMetadataOps uint64
+       DelOps         uint64
+       ListOps        uint64
+}
+
+func (s *azureBlobStats) TickErr(err error) {
+       if err == nil {
+               return
+       }
+       errType := fmt.Sprintf("%T", err)
+       if err, ok := err.(storage.AzureStorageServiceError); ok {
+               errType = errType + fmt.Sprintf(" %d (%s)", err.StatusCode, err.Code)
+       }
+       log.Printf("errType %T, err %s", err, err)
+       s.statsTicker.TickErr(err, errType)
+}
+
+// azureBlobClient wraps storage.BlobStorageClient in order to count
+// I/O and API usage stats.
+type azureBlobClient struct {
+       client *storage.BlobStorageClient
+       stats  azureBlobStats
+}
+
 func (c *azureBlobClient) ContainerExists(cname string) (bool, error) {
        c.stats.Tick(&c.stats.Ops)
        ok, err := c.client.ContainerExists(cname)

diff --git a/services/keepstore/azure_blob_volume_test.go b/services/keepstore/azure_blob_volume_test.go
index 7bccafcaa1963fd2f5e2bd5921da1e05b49f1143..4b015a9962d170e47c9f8fdd401d6c63bdd569ea 100644 (file)
--- a/services/keepstore/azure_blob_volume_test.go
+++ b/services/keepstore/azure_blob_volume_test.go
@@ -495,10 +495,10 @@ func TestAzureBlobVolumeRangeFenceposts(t *testing.T) {
                }
                gotHash := fmt.Sprintf("%x", md5.Sum(gotData))
                if gotLen != size {
-                       t.Error("length mismatch: got %d != %d", gotLen, size)
+                       t.Errorf("length mismatch: got %d != %d", gotLen, size)
                }
                if gotHash != hash {
-                       t.Error("hash mismatch: got %s != %s", gotHash, hash)
+                       t.Errorf("hash mismatch: got %s != %s", gotHash, hash)
                }
        }
 }
@@ -684,6 +684,7 @@ func (s *StubbedAzureBlobSuite) TestStats(c *check.C) {
        c.Check(err, check.NotNil)
        c.Check(stats(), check.Matches, `.*"Ops":[^0],.*`)
        c.Check(stats(), check.Matches, `.*"Errors":[^0],.*`)
+       c.Check(stats(), check.Matches, `.*"storage\.AzureStorageServiceError 404 \(404 Not Found\)":[^0].*`)
        c.Check(stats(), check.Matches, `.*"InBytes":0,.*`)
 
        err = s.volume.Put(context.Background(), loc, []byte("foo"))

diff --git a/services/keepstore/s3_volume.go b/services/keepstore/s3_volume.go
index c1d21051a8ebd9676481229d22e6a3670a4550a9..d34b8772c5eb90d8a5ac3ad89d3a2cee6a1f5d9c 100644 (file)
--- a/services/keepstore/s3_volume.go
+++ b/services/keepstore/s3_volume.go
@@ -872,35 +872,35 @@ type s3bucket struct {
 func (b *s3bucket) GetReader(path string) (io.ReadCloser, error) {
        rdr, err := b.Bucket.GetReader(path)
        b.stats.Tick(&b.stats.Ops, &b.stats.GetOps)
-       b.stats.tickErr(err)
+       b.stats.TickErr(err)
        return NewCountingReader(rdr, b.stats.TickInBytes), err
 }
 
 func (b *s3bucket) Head(path string, headers map[string][]string) (*http.Response, error) {
        resp, err := b.Bucket.Head(path, headers)
        b.stats.Tick(&b.stats.Ops, &b.stats.HeadOps)
-       b.stats.tickErr(err)
+       b.stats.TickErr(err)
        return resp, err
 }
 
 func (b *s3bucket) PutReader(path string, r io.Reader, length int64, contType string, perm s3.ACL, options s3.Options) error {
        err := b.Bucket.PutReader(path, NewCountingReader(r, b.stats.TickOutBytes), length, contType, perm, options)
        b.stats.Tick(&b.stats.Ops, &b.stats.PutOps)
-       b.stats.tickErr(err)
+       b.stats.TickErr(err)
        return err
 }
 
 func (b *s3bucket) Put(path string, data []byte, contType string, perm s3.ACL, options s3.Options) error {
        err := b.Bucket.PutReader(path, NewCountingReader(bytes.NewBuffer(data), b.stats.TickOutBytes), int64(len(data)), contType, perm, options)
        b.stats.Tick(&b.stats.Ops, &b.stats.PutOps)
-       b.stats.tickErr(err)
+       b.stats.TickErr(err)
        return err
 }
 
 func (b *s3bucket) Del(path string) error {
        err := b.Bucket.Del(path)
        b.stats.Tick(&b.stats.Ops, &b.stats.DelOps)
-       b.stats.tickErr(err)
+       b.stats.TickErr(err)
        return err
 }
 
@@ -912,25 +912,15 @@ type s3bucketStats struct {
        HeadOps uint64
        DelOps  uint64
        ListOps uint64
-
-       ErrorCodes map[string]uint64 `json:",omitempty"`
-
-       lock sync.Mutex
 }
 
-func (s *s3bucketStats) tickErr(err error) {
+func (s *s3bucketStats) TickErr(err error) {
        if err == nil {
                return
        }
-       s.TickErr(err)
-       errStr := fmt.Sprintf("%T", err)
+       errType := fmt.Sprintf("%T", err)
        if err, ok := err.(*s3.Error); ok {
-               errStr = errStr + fmt.Sprintf(" %d %s", err.StatusCode, err.Code)
-       }
-       s.lock.Lock()
-       if s.ErrorCodes == nil {
-               s.ErrorCodes = make(map[string]uint64)
+               errType = errType + fmt.Sprintf(" %d %s", err.StatusCode, err.Code)
        }
-       s.ErrorCodes[errStr]++
-       s.lock.Unlock()
+       s.statsTicker.TickErr(err, errType)
 }

diff --git a/services/keepstore/stats.go b/services/keepstore/stats.go
index 02d260c00c5d2b0b1c462799b13e4f7a6b107ccf..e02eb3aeeef8095d0923b16098acc83682cabd0a 100644 (file)
--- a/services/keepstore/stats.go
+++ b/services/keepstore/stats.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+       "sync"
        "sync/atomic"
 )
 
@@ -8,6 +9,9 @@ type statsTicker struct {
        Errors   uint64
        InBytes  uint64
        OutBytes uint64
+
+       ErrorCodes map[string]uint64 `json:",omitempty"`
+       lock       sync.Mutex
 }
 
 // Tick increments each of the given counters by 1 using
@@ -18,11 +22,18 @@ func (s *statsTicker) Tick(counters ...*uint64) {
        }
 }
 
-func (s *statsTicker) TickErr(err error) {
+func (s *statsTicker) TickErr(err error, errType string) {
        if err == nil {
                return
        }
        s.Tick(&s.Errors)
+
+       s.lock.Lock()
+       if s.ErrorCodes == nil {
+               s.ErrorCodes = make(map[string]uint64)
+       }
+       s.ErrorCodes[errType]++
+       s.lock.Unlock()
 }
 
 func (s *statsTicker) TickInBytes(n uint64) {