10484: Track s3 errors by response code.

diff --git a/services/keepstore/s3_volume.go b/services/keepstore/s3_volume.go
index c52f616d1ad98a81872ee41c93aacebd6881d8b6..48ba95b4887ee03e2b245faa1ab96cd2c868cdff 100644 (file)
--- a/services/keepstore/s3_volume.go
+++ b/services/keepstore/s3_volume.go
@@ -165,6 +165,10 @@ type bucketStats struct {
        DelOps   uint64
        InBytes  uint64
        OutBytes uint64
+
+       ErrorCodes map[string]uint64 `json:",omitempty"`
+
+       lock sync.Mutex
 }
 
 // Examples implements VolumeWithExamples.
@@ -851,8 +855,18 @@ func (v *S3Volume) tick(counters ...*uint64) {
 }
 
 func (v *S3Volume) tickErr(err error) error {
-       if err != nil {
-               atomic.AddUint64(&v.bucketStats.Errors, 1)
+       if err == nil {
+               return nil
+       }
+       atomic.AddUint64(&v.bucketStats.Errors, 1)
+       if err, ok := err.(*s3.Error); ok {
+               errStr := fmt.Sprintf("%d %s", err.StatusCode, err.Code)
+               v.bucketStats.lock.Lock()
+               if v.bucketStats.ErrorCodes == nil {
+                       v.bucketStats.ErrorCodes = make(map[string]uint64)
+               }
+               v.bucketStats.ErrorCodes[errStr]++
+               v.bucketStats.lock.Unlock()
        }
        return err
 }