From: Tom Clegg <tom@curoverse.com>
Date: Thu, 10 Nov 2016 20:21:26 +0000 (-0500)
Subject: 10484: Track s3 errors by response code.
X-Git-Tag: 1.1.0~591^2~4
X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/515a58c0ef8634fca2397a8609f868524a42132c

10484: Track s3 errors by response code.
---

diff --git a/services/keepstore/s3_volume.go b/services/keepstore/s3_volume.go
index c52f616d1a..48ba95b488 100644
--- a/services/keepstore/s3_volume.go
+++ b/services/keepstore/s3_volume.go
@@ -165,6 +165,10 @@ type bucketStats struct {
 	DelOps   uint64
 	InBytes  uint64
 	OutBytes uint64
+
+	ErrorCodes map[string]uint64 `json:",omitempty"`
+
+	lock sync.Mutex
 }
 
 // Examples implements VolumeWithExamples.
@@ -851,8 +855,18 @@ func (v *S3Volume) tick(counters ...*uint64) {
 }
 
 func (v *S3Volume) tickErr(err error) error {
-	if err != nil {
-		atomic.AddUint64(&v.bucketStats.Errors, 1)
+	if err == nil {
+		return nil
+	}
+	atomic.AddUint64(&v.bucketStats.Errors, 1)
+	if err, ok := err.(*s3.Error); ok {
+		errStr := fmt.Sprintf("%d %s", err.StatusCode, err.Code)
+		v.bucketStats.lock.Lock()
+		if v.bucketStats.ErrorCodes == nil {
+			v.bucketStats.ErrorCodes = make(map[string]uint64)
+		}
+		v.bucketStats.ErrorCodes[errStr]++
+		v.bucketStats.lock.Unlock()
 	}
 	return err
 }