X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/64efd1030538d59821ce288a7674e29d49c35744..f4b7e7bf5b1a875db95a42781937c2a690062dd3:/services/keepstore/s3_volume.go diff --git a/services/keepstore/s3_volume.go b/services/keepstore/s3_volume.go index 7f2c3f05ef..4c39dcd5c4 100644 --- a/services/keepstore/s3_volume.go +++ b/services/keepstore/s3_volume.go @@ -1,22 +1,36 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: AGPL-3.0 + package main import ( + "bytes" + "context" + "crypto/sha256" "encoding/base64" "encoding/hex" "flag" "fmt" "io" - "log" + "io/ioutil" "net/http" "os" "regexp" "strings" "sync" + "sync/atomic" "time" "git.curoverse.com/arvados.git/sdk/go/arvados" "github.com/AdRoll/goamz/aws" "github.com/AdRoll/goamz/s3" + "github.com/prometheus/client_golang/prometheus" +) + +const ( + s3DefaultReadTimeout = arvados.Duration(10 * time.Minute) + s3DefaultConnectTimeout = arvados.Duration(time.Minute) ) var ( @@ -33,6 +47,8 @@ var ( s3RaceWindow time.Duration s3ACL = s3.Private + + zeroTime time.Time ) const ( @@ -56,7 +72,7 @@ func (s *s3VolumeAdder) Set(bucketName string) error { if s3AccessKeyFile == "" || s3SecretKeyFile == "" { return fmt.Errorf("-s3-access-key-file and -s3-secret-key-file arguments must given before -s3-bucket-volume") } - if flagSerializeIO { + if deprecated.flagSerializeIO { log.Print("Notice: -serialize is not supported by s3-bucket volumes.") } s.Config.Volumes = append(s.Config.Volumes, &S3Volume{ @@ -68,7 +84,7 @@ func (s *s3VolumeAdder) Set(bucketName string) error { RaceWindow: arvados.Duration(s3RaceWindow), S3Replication: s3Replication, UnsafeDelete: s3UnsafeDelete, - ReadOnly: flagReadonly, + ReadOnly: deprecated.flagReadonly, IndexPageSize: 1000, }) return nil @@ -121,7 +137,7 @@ func init() { &s3UnsafeDelete, "s3-unsafe-delete", false, - "EXPERIMENTAL. Enable deletion (garbage collection), even though there are known race conditions that can cause data loss.") + "EXPERIMENTAL. Enable deletion (garbage collection) even when trash lifetime is zero, even though there are known race conditions that can cause data loss.") } // S3Volume implements Volume using an S3 bucket. @@ -134,11 +150,14 @@ type S3Volume struct { LocationConstraint bool IndexPageSize int S3Replication int + ConnectTimeout arvados.Duration + ReadTimeout arvados.Duration RaceWindow arvados.Duration ReadOnly bool UnsafeDelete bool + StorageClasses []string - bucket *s3.Bucket + bucket *s3bucket startOnce sync.Once } @@ -147,24 +166,28 @@ type S3Volume struct { func (*S3Volume) Examples() []Volume { return []Volume{ &S3Volume{ - AccessKeyFile: "/etc/aws_s3_access_key.txt", - SecretKeyFile: "/etc/aws_s3_secret_key.txt", - Endpoint: "", - Region: "us-east-1", - Bucket: "example-bucket-name", - IndexPageSize: 1000, - S3Replication: 2, - RaceWindow: arvados.Duration(24 * time.Hour), + AccessKeyFile: "/etc/aws_s3_access_key.txt", + SecretKeyFile: "/etc/aws_s3_secret_key.txt", + Endpoint: "", + Region: "us-east-1", + Bucket: "example-bucket-name", + IndexPageSize: 1000, + S3Replication: 2, + RaceWindow: arvados.Duration(24 * time.Hour), + ConnectTimeout: arvados.Duration(time.Minute), + ReadTimeout: arvados.Duration(5 * time.Minute), }, &S3Volume{ - AccessKeyFile: "/etc/gce_s3_access_key.txt", - SecretKeyFile: "/etc/gce_s3_secret_key.txt", - Endpoint: "https://storage.googleapis.com", - Region: "", - Bucket: "example-bucket-name", - IndexPageSize: 1000, - S3Replication: 2, - RaceWindow: arvados.Duration(24 * time.Hour), + AccessKeyFile: "/etc/gce_s3_access_key.txt", + SecretKeyFile: "/etc/gce_s3_secret_key.txt", + Endpoint: "https://storage.googleapis.com", + Region: "", + Bucket: "example-bucket-name", + IndexPageSize: 1000, + S3Replication: 2, + RaceWindow: arvados.Duration(24 * time.Hour), + ConnectTimeout: arvados.Duration(time.Minute), + ReadTimeout: arvados.Duration(5 * time.Minute), }, } } @@ -176,7 +199,7 @@ func (*S3Volume) Type() string { // Start populates private fields and verifies the configuration is // valid. -func (v *S3Volume) Start() error { +func (v *S3Volume) Start(vm *volumeMetricsVecs) error { region, ok := aws.Regions[v.Region] if v.Endpoint == "" { if !ok { @@ -203,13 +226,62 @@ func (v *S3Volume) Start() error { if err != nil { return err } - v.bucket = &s3.Bucket{ - S3: s3.New(auth, region), - Name: v.Bucket, + + // Zero timeouts mean "wait forever", which is a bad + // default. Default to long timeouts instead. + if v.ConnectTimeout == 0 { + v.ConnectTimeout = s3DefaultConnectTimeout + } + if v.ReadTimeout == 0 { + v.ReadTimeout = s3DefaultReadTimeout + } + + client := s3.New(auth, region) + if region.EC2Endpoint.Signer == aws.V4Signature { + // Currently affects only eu-central-1 + client.Signature = aws.V4Signature } + client.ConnectTimeout = time.Duration(v.ConnectTimeout) + client.ReadTimeout = time.Duration(v.ReadTimeout) + v.bucket = &s3bucket{ + Bucket: &s3.Bucket{ + S3: client, + Name: v.Bucket, + }, + } + // Set up prometheus metrics + lbls := prometheus.Labels{"device_id": v.DeviceID()} + v.bucket.stats.opsCounters, v.bucket.stats.errCounters, v.bucket.stats.ioBytes = vm.getCounterVecsFor(lbls) + return nil } +// DeviceID returns a globally unique ID for the storage bucket. +func (v *S3Volume) DeviceID() string { + return "s3://" + v.Endpoint + "/" + v.Bucket +} + +func (v *S3Volume) getReaderWithContext(ctx context.Context, loc string) (rdr io.ReadCloser, err error) { + ready := make(chan bool) + go func() { + rdr, err = v.getReader(loc) + close(ready) + }() + select { + case <-ready: + return + case <-ctx.Done(): + theConfig.debugLogf("s3: abandoning getReader(): %s", ctx.Err()) + go func() { + <-ready + if err == nil { + rdr.Close() + } + }() + return nil, ctx.Err() + } +} + // getReader wraps (Bucket)GetReader. // // In situations where (Bucket)GetReader would fail because the block @@ -221,6 +293,7 @@ func (v *S3Volume) getReader(loc string) (rdr io.ReadCloser, err error) { if err == nil || !os.IsNotExist(err) { return } + _, err = v.bucket.Head("recent/"+loc, nil) err = v.translateError(err) if err != nil { @@ -232,6 +305,7 @@ func (v *S3Volume) getReader(loc string) (rdr io.ReadCloser, err error) { err = os.ErrNotExist return } + rdr, err = v.bucket.GetReader(loc) if err != nil { log.Printf("warning: reading %s after successful fixRace: %s", loc, err) @@ -242,50 +316,150 @@ func (v *S3Volume) getReader(loc string) (rdr io.ReadCloser, err error) { // Get a block: copy the block data into buf, and return the number of // bytes copied. -func (v *S3Volume) Get(loc string, buf []byte) (int, error) { - rdr, err := v.getReader(loc) +func (v *S3Volume) Get(ctx context.Context, loc string, buf []byte) (int, error) { + rdr, err := v.getReaderWithContext(ctx, loc) if err != nil { return 0, err } - defer rdr.Close() - n, err := io.ReadFull(rdr, buf) - switch err { - case nil, io.EOF, io.ErrUnexpectedEOF: - return n, nil - default: - return 0, v.translateError(err) + + var n int + ready := make(chan bool) + go func() { + defer close(ready) + + defer rdr.Close() + n, err = io.ReadFull(rdr, buf) + + switch err { + case nil, io.EOF, io.ErrUnexpectedEOF: + err = nil + default: + err = v.translateError(err) + } + }() + select { + case <-ctx.Done(): + theConfig.debugLogf("s3: interrupting ReadFull() with Close() because %s", ctx.Err()) + rdr.Close() + // Must wait for ReadFull to return, to ensure it + // doesn't write to buf after we return. + theConfig.debugLogf("s3: waiting for ReadFull() to fail") + <-ready + return 0, ctx.Err() + case <-ready: + return n, err } } // Compare the given data with the stored data. -func (v *S3Volume) Compare(loc string, expect []byte) error { - rdr, err := v.getReader(loc) +func (v *S3Volume) Compare(ctx context.Context, loc string, expect []byte) error { + errChan := make(chan error, 1) + go func() { + _, err := v.bucket.Head("recent/"+loc, nil) + errChan <- err + }() + var err error + select { + case <-ctx.Done(): + return ctx.Err() + case err = <-errChan: + } + if err != nil { + // Checking for "loc" itself here would interfere with + // future GET requests. + // + // On AWS, if X doesn't exist, a HEAD or GET request + // for X causes X's non-existence to be cached. Thus, + // if we test for X, then create X and return a + // signature to our client, the client might still get + // 404 from all keepstores when trying to read it. + // + // To avoid this, we avoid doing HEAD X or GET X until + // we know X has been written. + // + // Note that X might exist even though recent/X + // doesn't: for example, the response to HEAD recent/X + // might itself come from a stale cache. In such + // cases, we will return a false negative and + // PutHandler might needlessly create another replica + // on a different volume. That's not ideal, but it's + // better than passing the eventually-consistent + // problem on to our clients. + return v.translateError(err) + } + rdr, err := v.getReaderWithContext(ctx, loc) if err != nil { return err } defer rdr.Close() - return v.translateError(compareReaderWithBuf(rdr, expect, loc[:32])) + return v.translateError(compareReaderWithBuf(ctx, rdr, expect, loc[:32])) } // Put writes a block. -func (v *S3Volume) Put(loc string, block []byte) error { +func (v *S3Volume) Put(ctx context.Context, loc string, block []byte) error { if v.ReadOnly { return MethodDisabledError } var opts s3.Options - if len(block) > 0 { + size := len(block) + if size > 0 { md5, err := hex.DecodeString(loc) if err != nil { return err } opts.ContentMD5 = base64.StdEncoding.EncodeToString(md5) + // In AWS regions that use V4 signatures, we need to + // provide ContentSHA256 up front. Otherwise, the S3 + // library reads the request body (from our buffer) + // into another new buffer in order to compute the + // SHA256 before sending the request -- which would + // mean consuming 128 MiB of memory for the duration + // of a 64 MiB write. + opts.ContentSHA256 = fmt.Sprintf("%x", sha256.Sum256(block)) } - err := v.bucket.Put(loc, block, "application/octet-stream", s3ACL, opts) - if err != nil { + + // Send the block data through a pipe, so that (if we need to) + // we can close the pipe early and abandon our PutReader() + // goroutine, without worrying about PutReader() accessing our + // block buffer after we release it. + bufr, bufw := io.Pipe() + go func() { + io.Copy(bufw, bytes.NewReader(block)) + bufw.Close() + }() + + var err error + ready := make(chan bool) + go func() { + defer func() { + if ctx.Err() != nil { + theConfig.debugLogf("%s: abandoned PutReader goroutine finished with err: %s", v, err) + } + }() + defer close(ready) + err = v.bucket.PutReader(loc, bufr, int64(size), "application/octet-stream", s3ACL, opts) + if err != nil { + return + } + err = v.bucket.PutReader("recent/"+loc, nil, 0, "application/octet-stream", s3ACL, s3.Options{}) + }() + select { + case <-ctx.Done(): + theConfig.debugLogf("%s: taking PutReader's input away: %s", v, ctx.Err()) + // Our pipe might be stuck in Write(), waiting for + // PutReader() to read. If so, un-stick it. This means + // PutReader will get corrupt data, but that's OK: the + // size and MD5 won't match, so the write will fail. + go io.Copy(ioutil.Discard, bufr) + // CloseWithError() will return once pending I/O is done. + bufw.CloseWithError(ctx.Err()) + theConfig.debugLogf("%s: abandoning PutReader goroutine", v) + return ctx.Err() + case <-ready: + // Unblock pipe in case PutReader did not consume it. + io.Copy(ioutil.Discard, bufr) return v.translateError(err) } - err = v.bucket.Put("recent/"+loc, nil, "application/octet-stream", s3ACL, s3.Options{}) - return v.translateError(err) } // Touch sets the timestamp for the given locator to the current time. @@ -301,7 +475,7 @@ func (v *S3Volume) Touch(loc string) error { } else if err != nil { return err } - err = v.bucket.Put("recent/"+loc, nil, "application/octet-stream", s3ACL, s3.Options{}) + err = v.bucket.PutReader("recent/"+loc, nil, 0, "application/octet-stream", s3ACL, s3.Options{}) return v.translateError(err) } @@ -315,7 +489,7 @@ func (v *S3Volume) Mtime(loc string) (time.Time, error) { err = v.translateError(err) if os.IsNotExist(err) { // The data object X exists, but recent/X is missing. - err = v.bucket.Put("recent/"+loc, nil, "application/octet-stream", s3ACL, s3.Options{}) + err = v.bucket.PutReader("recent/"+loc, nil, 0, "application/octet-stream", s3ACL, s3.Options{}) if err != nil { log.Printf("error: creating %q: %s", "recent/"+loc, err) return zeroTime, v.translateError(err) @@ -338,16 +512,18 @@ func (v *S3Volume) Mtime(loc string) (time.Time, error) { func (v *S3Volume) IndexTo(prefix string, writer io.Writer) error { // Use a merge sort to find matching sets of X and recent/X. dataL := s3Lister{ - Bucket: v.bucket, + Bucket: v.bucket.Bucket, Prefix: prefix, PageSize: v.IndexPageSize, + Stats: &v.bucket.stats, } recentL := s3Lister{ - Bucket: v.bucket, + Bucket: v.bucket.Bucket, Prefix: "recent/" + prefix, PageSize: v.IndexPageSize, + Stats: &v.bucket.stats, } - for data, recent := dataL.First(), recentL.First(); data != nil; data = dataL.Next() { + for data, recent := dataL.First(), recentL.First(); data != nil && dataL.Error() == nil; data = dataL.Next() { if data.Key >= "g" { // Conveniently, "recent/*" and "trash/*" are // lexically greater than all hex-encoded data @@ -366,7 +542,7 @@ func (v *S3Volume) IndexTo(prefix string, writer io.Writer) error { stamp := data // Advance to the corresponding recent/X marker, if any - for recent != nil { + for recent != nil && recentL.Error() == nil { if cmp := strings.Compare(recent.Key[7:], data.Key); cmp < 0 { recent = recentL.Next() continue @@ -381,13 +557,16 @@ func (v *S3Volume) IndexTo(prefix string, writer io.Writer) error { break } } + if err := recentL.Error(); err != nil { + return err + } t, err := time.Parse(time.RFC3339, stamp.LastModified) if err != nil { return err } fmt.Fprintf(writer, "%s+%d %d\n", data.Key, data.Size, t.UnixNano()) } - return nil + return dataL.Error() } // Trash a Keep block. @@ -404,7 +583,7 @@ func (v *S3Volume) Trash(loc string) error { if !s3UnsafeDelete { return ErrS3TrashDisabled } - return v.bucket.Del(loc) + return v.translateError(v.bucket.Del(loc)) } err := v.checkRaceWindow(loc) if err != nil { @@ -459,8 +638,10 @@ func (v *S3Volume) safeCopy(dst, src string) error { MetadataDirective: "REPLACE", }, v.bucket.Name+"/"+src) err = v.translateError(err) - if err != nil { + if os.IsNotExist(err) { return err + } else if err != nil { + return fmt.Errorf("PutCopy(%q ← %q): %s", dst, v.bucket.Name+"/"+src, err) } if t, err := time.Parse(time.RFC3339Nano, resp.LastModified); err != nil { return fmt.Errorf("PutCopy succeeded but did not return a timestamp: %q: %s", resp.LastModified, err) @@ -492,7 +673,7 @@ func (v *S3Volume) Untrash(loc string) error { if err != nil { return err } - err = v.bucket.Put("recent/"+loc, nil, "application/octet-stream", s3ACL, s3.Options{}) + err = v.bucket.PutReader("recent/"+loc, nil, 0, "application/octet-stream", s3ACL, s3.Options{}) return v.translateError(err) } @@ -507,9 +688,14 @@ func (v *S3Volume) Status() *VolumeStatus { } } +// InternalStats returns bucket I/O and API call counters. +func (v *S3Volume) InternalStats() interface{} { + return &v.bucket.stats +} + // String implements fmt.Stringer. func (v *S3Volume) String() string { - return fmt.Sprintf("s3-bucket:%+q", v.bucket.Name) + return fmt.Sprintf("s3-bucket:%+q", v.Bucket) } // Writable returns false if all future Put, Mtime, and Delete calls @@ -524,6 +710,11 @@ func (v *S3Volume) Replication() int { return v.S3Replication } +// GetStorageClasses implements Volume +func (v *S3Volume) GetStorageClasses() []string { + return v.StorageClasses +} + var s3KeepBlockRegexp = regexp.MustCompile(`^[0-9a-f]{32}$`) func (v *S3Volume) isKeepBlock(s string) bool { @@ -596,26 +787,21 @@ func (v *S3Volume) translateError(err error) error { func (v *S3Volume) EmptyTrash() { var bytesInTrash, blocksInTrash, bytesDeleted, blocksDeleted int64 - // Use a merge sort to find matching sets of trash/X and recent/X. - trashL := s3Lister{ - Bucket: v.bucket, - Prefix: "trash/", - PageSize: v.IndexPageSize, - } // Define "ready to delete" as "...when EmptyTrash started". startT := time.Now() - for trash := trashL.First(); trash != nil; trash = trashL.Next() { + + emptyOneKey := func(trash *s3.Key) { loc := trash.Key[6:] if !v.isKeepBlock(loc) { - continue + return } - bytesInTrash += trash.Size - blocksInTrash++ + atomic.AddInt64(&bytesInTrash, trash.Size) + atomic.AddInt64(&blocksInTrash, 1) trashT, err := time.Parse(time.RFC3339, trash.LastModified) if err != nil { log.Printf("warning: %s: EmptyTrash: %q: parse %q: %s", v, trash.Key, trash.LastModified, err) - continue + return } recent, err := v.bucket.Head("recent/"+loc, nil) if err != nil && os.IsNotExist(v.translateError(err)) { @@ -624,15 +810,15 @@ func (v *S3Volume) EmptyTrash() { if err != nil { log.Printf("error: %s: EmptyTrash: Untrash(%q): %s", v, loc, err) } - continue + return } else if err != nil { log.Printf("warning: %s: EmptyTrash: HEAD %q: %s", v, "recent/"+loc, err) - continue + return } recentT, err := v.lastModified(recent) if err != nil { log.Printf("warning: %s: EmptyTrash: %q: parse %q: %s", v, "recent/"+loc, recent.Header.Get("Last-Modified"), err) - continue + return } if trashT.Sub(recentT) < theConfig.BlobSignatureTTL.Duration() { if age := startT.Sub(recentT); age >= theConfig.BlobSignatureTTL.Duration()-time.Duration(v.RaceWindow) { @@ -647,37 +833,68 @@ func (v *S3Volume) EmptyTrash() { log.Printf("notice: %s: EmptyTrash: detected old race for %q, calling fixRace + Touch", v, loc) v.fixRace(loc) v.Touch(loc) - continue - } else if _, err := v.bucket.Head(loc, nil); os.IsNotExist(err) { + return + } + _, err := v.bucket.Head(loc, nil) + if os.IsNotExist(err) { log.Printf("notice: %s: EmptyTrash: detected recent race for %q, calling fixRace", v, loc) v.fixRace(loc) - continue + return } else if err != nil { log.Printf("warning: %s: EmptyTrash: HEAD %q: %s", v, loc, err) - continue + return } } if startT.Sub(trashT) < theConfig.TrashLifetime.Duration() { - continue + return } err = v.bucket.Del(trash.Key) if err != nil { log.Printf("warning: %s: EmptyTrash: deleting %q: %s", v, trash.Key, err) - continue + return } - bytesDeleted += trash.Size - blocksDeleted++ + atomic.AddInt64(&bytesDeleted, trash.Size) + atomic.AddInt64(&blocksDeleted, 1) _, err = v.bucket.Head(loc, nil) - if os.IsNotExist(err) { - err = v.bucket.Del("recent/" + loc) - if err != nil { - log.Printf("warning: %s: EmptyTrash: deleting %q: %s", v, "recent/"+loc, err) - } - } else if err != nil { - log.Printf("warning: %s: EmptyTrash: HEAD %q: %s", v, "recent/"+loc, err) + if err == nil { + log.Printf("warning: %s: EmptyTrash: HEAD %q succeeded immediately after deleting %q", v, loc, loc) + return + } + if !os.IsNotExist(v.translateError(err)) { + log.Printf("warning: %s: EmptyTrash: HEAD %q: %s", v, loc, err) + return + } + err = v.bucket.Del("recent/" + loc) + if err != nil { + log.Printf("warning: %s: EmptyTrash: deleting %q: %s", v, "recent/"+loc, err) } } + + var wg sync.WaitGroup + todo := make(chan *s3.Key, theConfig.EmptyTrashWorkers) + for i := 0; i < 1 || i < theConfig.EmptyTrashWorkers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for key := range todo { + emptyOneKey(key) + } + }() + } + + trashL := s3Lister{ + Bucket: v.bucket.Bucket, + Prefix: "trash/", + PageSize: v.IndexPageSize, + Stats: &v.bucket.stats, + } + for trash := trashL.First(); trash != nil; trash = trashL.Next() { + todo <- trash + } + close(todo) + wg.Wait() + if err := trashL.Error(); err != nil { log.Printf("error: %s: EmptyTrash: lister: %s", v, err) } @@ -688,6 +905,7 @@ type s3Lister struct { Bucket *s3.Bucket Prefix string PageSize int + Stats *s3bucketStats nextMarker string buf []s3.Key err error @@ -716,6 +934,8 @@ func (lister *s3Lister) Error() error { } func (lister *s3Lister) getPage() { + lister.Stats.TickOps("list") + lister.Stats.Tick(&lister.Stats.Ops, &lister.Stats.ListOps) resp, err := lister.Bucket.List(lister.Prefix, "", lister.nextMarker, lister.PageSize) lister.nextMarker = "" if err != nil { @@ -742,3 +962,73 @@ func (lister *s3Lister) pop() (k *s3.Key) { } return } + +// s3bucket wraps s3.bucket and counts I/O and API usage stats. +type s3bucket struct { + *s3.Bucket + stats s3bucketStats +} + +func (b *s3bucket) GetReader(path string) (io.ReadCloser, error) { + rdr, err := b.Bucket.GetReader(path) + b.stats.TickOps("get") + b.stats.Tick(&b.stats.Ops, &b.stats.GetOps) + b.stats.TickErr(err) + return NewCountingReader(rdr, b.stats.TickInBytes), err +} + +func (b *s3bucket) Head(path string, headers map[string][]string) (*http.Response, error) { + resp, err := b.Bucket.Head(path, headers) + b.stats.TickOps("head") + b.stats.Tick(&b.stats.Ops, &b.stats.HeadOps) + b.stats.TickErr(err) + return resp, err +} + +func (b *s3bucket) PutReader(path string, r io.Reader, length int64, contType string, perm s3.ACL, options s3.Options) error { + if length == 0 { + // goamz will only send Content-Length: 0 when reader + // is nil due to net.http.Request.ContentLength + // behavior. Otherwise, Content-Length header is + // omitted which will cause some S3 services + // (including AWS and Ceph RadosGW) to fail to create + // empty objects. + r = nil + } else { + r = NewCountingReader(r, b.stats.TickOutBytes) + } + err := b.Bucket.PutReader(path, r, length, contType, perm, options) + b.stats.TickOps("put") + b.stats.Tick(&b.stats.Ops, &b.stats.PutOps) + b.stats.TickErr(err) + return err +} + +func (b *s3bucket) Del(path string) error { + err := b.Bucket.Del(path) + b.stats.TickOps("delete") + b.stats.Tick(&b.stats.Ops, &b.stats.DelOps) + b.stats.TickErr(err) + return err +} + +type s3bucketStats struct { + statsTicker + Ops uint64 + GetOps uint64 + PutOps uint64 + HeadOps uint64 + DelOps uint64 + ListOps uint64 +} + +func (s *s3bucketStats) TickErr(err error) { + if err == nil { + return + } + errType := fmt.Sprintf("%T", err) + if err, ok := err.(*s3.Error); ok { + errType = errType + fmt.Sprintf(" %d %s", err.StatusCode, err.Code) + } + s.statsTicker.TickErr(err, errType) +}