- log.Printf("error: fixRace: %s", err)
- return false
- }
- return true
-}
-
-func (v *S3Volume) translateError(err error) error {
- switch err := err.(type) {
- case *s3.Error:
- if (err.StatusCode == http.StatusNotFound && err.Code == "NoSuchKey") ||
- strings.Contains(err.Error(), "Not Found") {
- return os.ErrNotExist
- }
- // Other 404 errors like NoSuchVersion and
- // NoSuchBucket are different problems which should
- // get called out downstream, so we don't convert them
- // to os.ErrNotExist.
- }
- return err
-}
-
-// EmptyTrash looks for trashed blocks that exceeded TrashLifetime
-// and deletes them from the volume.
-func (v *S3Volume) EmptyTrash() {
- var bytesInTrash, blocksInTrash, bytesDeleted, blocksDeleted int64
-
- // Use a merge sort to find matching sets of trash/X and recent/X.
- trashL := s3Lister{
- Bucket: v.bucket.Bucket,
- Prefix: "trash/",
- PageSize: v.IndexPageSize,
- }
- // Define "ready to delete" as "...when EmptyTrash started".
- startT := time.Now()
- for trash := trashL.First(); trash != nil; trash = trashL.Next() {
- loc := trash.Key[6:]
- if !v.isKeepBlock(loc) {
- continue
- }
- bytesInTrash += trash.Size
- blocksInTrash++
-
- trashT, err := time.Parse(time.RFC3339, trash.LastModified)
- if err != nil {
- log.Printf("warning: %s: EmptyTrash: %q: parse %q: %s", v, trash.Key, trash.LastModified, err)
- continue
- }
- recent, err := v.bucket.Head("recent/"+loc, nil)
- if err != nil && os.IsNotExist(v.translateError(err)) {
- log.Printf("warning: %s: EmptyTrash: found trash marker %q but no %q (%s); calling Untrash", v, trash.Key, "recent/"+loc, err)
- err = v.Untrash(loc)
- if err != nil {
- log.Printf("error: %s: EmptyTrash: Untrash(%q): %s", v, loc, err)
- }
- continue
- } else if err != nil {
- log.Printf("warning: %s: EmptyTrash: HEAD %q: %s", v, "recent/"+loc, err)
- continue
- }
- recentT, err := v.lastModified(recent)
- if err != nil {
- log.Printf("warning: %s: EmptyTrash: %q: parse %q: %s", v, "recent/"+loc, recent.Header.Get("Last-Modified"), err)
- continue
- }
- if trashT.Sub(recentT) < theConfig.BlobSignatureTTL.Duration() {
- if age := startT.Sub(recentT); age >= theConfig.BlobSignatureTTL.Duration()-time.Duration(v.RaceWindow) {
- // recent/loc is too old to protect
- // loc from being Trashed again during
- // the raceWindow that starts if we
- // delete trash/X now.
- //
- // Note this means (TrashCheckInterval
- // < BlobSignatureTTL - raceWindow) is
- // necessary to avoid starvation.
- log.Printf("notice: %s: EmptyTrash: detected old race for %q, calling fixRace + Touch", v, loc)
- v.fixRace(loc)
- v.Touch(loc)
- continue
- }
- _, err := v.bucket.Head(loc, nil)
- if os.IsNotExist(err) {
- log.Printf("notice: %s: EmptyTrash: detected recent race for %q, calling fixRace", v, loc)
- v.fixRace(loc)
- continue
- } else if err != nil {
- log.Printf("warning: %s: EmptyTrash: HEAD %q: %s", v, loc, err)
- continue
- }
- }
- if startT.Sub(trashT) < theConfig.TrashLifetime.Duration() {
- continue
- }
- err = v.bucket.Del(trash.Key)
- if err != nil {
- log.Printf("warning: %s: EmptyTrash: deleting %q: %s", v, trash.Key, err)
- continue
- }
- bytesDeleted += trash.Size
- blocksDeleted++
-
- _, err = v.bucket.Head(loc, nil)
- if os.IsNotExist(err) {
- err = v.bucket.Del("recent/" + loc)
- if err != nil {
- log.Printf("warning: %s: EmptyTrash: deleting %q: %s", v, "recent/"+loc, err)
- }
- } else if err != nil {
- log.Printf("warning: %s: EmptyTrash: HEAD %q: %s", v, "recent/"+loc, err)
- }
- }
- if err := trashL.Error(); err != nil {
- log.Printf("error: %s: EmptyTrash: lister: %s", v, err)
- }
- log.Printf("EmptyTrash stats for %v: Deleted %v bytes in %v blocks. Remaining in trash: %v bytes in %v blocks.", v.String(), bytesDeleted, blocksDeleted, bytesInTrash-bytesDeleted, blocksInTrash-blocksDeleted)
-}
-
-type s3Lister struct {
- Bucket *s3.Bucket
- Prefix string
- PageSize int
- nextMarker string
- buf []s3.Key
- err error
-}
-
-// First fetches the first page and returns the first item. It returns
-// nil if the response is the empty set or an error occurs.
-func (lister *s3Lister) First() *s3.Key {
- lister.getPage()
- return lister.pop()
-}
-
-// Next returns the next item, fetching the next page if necessary. It
-// returns nil if the last available item has already been fetched, or
-// an error occurs.
-func (lister *s3Lister) Next() *s3.Key {
- if len(lister.buf) == 0 && lister.nextMarker != "" {
- lister.getPage()