X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/1127e8884c809a35280d8e57dbe3bc1b8f8818a5..ffe1fe1c77743147ee82aacdc50edde3672cd748:/services/keepstore/s3_volume.go diff --git a/services/keepstore/s3_volume.go b/services/keepstore/s3_volume.go index ad6ff5bf42..1a2a47b0df 100644 --- a/services/keepstore/s3_volume.go +++ b/services/keepstore/s3_volume.go @@ -18,7 +18,9 @@ import ( ) var ( - ErrS3TrashNotAvailable = fmt.Errorf("trash is not possible because -trash-lifetime=0 and -s3-unsafe-delete=false") + // ErrS3TrashDisabled is returned by Trash if that operation + // is impossible with the current config. + ErrS3TrashDisabled = fmt.Errorf("trash function is disabled because -trash-lifetime=0 and -s3-unsafe-delete=false") s3AccessKeyFile string s3SecretKeyFile string @@ -41,9 +43,6 @@ type s3VolumeAdder struct { } func (s *s3VolumeAdder) Set(bucketName string) error { - if trashLifetime != 0 { - return ErrNotImplemented - } if bucketName == "" { return fmt.Errorf("no container name given") } @@ -134,6 +133,7 @@ func init() { "EXPERIMENTAL. Enable deletion (garbage collection), even though there are known race conditions that can cause data loss.") } +// S3Volume implements Volume using an S3 bucket. type S3Volume struct { *s3.Bucket raceWindow time.Duration @@ -158,11 +158,18 @@ func NewS3Volume(auth aws.Auth, region aws.Region, bucket string, raceWindow tim } } +// Check returns an error if the volume is inaccessible (e.g., config +// error). func (v *S3Volume) Check() error { return nil } -func (v *S3Volume) getReaderWithFixRace(loc string) (rdr io.ReadCloser, err error) { +// getReader wraps (Bucket)GetReader. +// +// In situations where (Bucket)GetReader would fail because the block +// disappeared in a Trash race, getReader calls fixRace to recover the +// data, and tries again. +func (v *S3Volume) getReader(loc string) (rdr io.ReadCloser, err error) { rdr, err = v.Bucket.GetReader(loc) err = v.translateError(err) if err == nil || !os.IsNotExist(err) { @@ -187,8 +194,10 @@ func (v *S3Volume) getReaderWithFixRace(loc string) (rdr io.ReadCloser, err erro return } +// Get a block: copy the block data into buf, and return the number of +// bytes copied. func (v *S3Volume) Get(loc string, buf []byte) (int, error) { - rdr, err := v.getReaderWithFixRace(loc) + rdr, err := v.getReader(loc) if err != nil { return 0, err } @@ -202,8 +211,9 @@ func (v *S3Volume) Get(loc string, buf []byte) (int, error) { } } +// Compare the given data with the stored data. func (v *S3Volume) Compare(loc string, expect []byte) error { - rdr, err := v.getReaderWithFixRace(loc) + rdr, err := v.getReader(loc) if err != nil { return err } @@ -211,6 +221,7 @@ func (v *S3Volume) Compare(loc string, expect []byte) error { return v.translateError(compareReaderWithBuf(rdr, expect, loc[:32])) } +// Put writes a block. func (v *S3Volume) Put(loc string, block []byte) error { if v.readonly { return MethodDisabledError @@ -231,6 +242,7 @@ func (v *S3Volume) Put(loc string, block []byte) error { return v.translateError(err) } +// Touch sets the timestamp for the given locator to the current time. func (v *S3Volume) Touch(loc string) error { if v.readonly { return MethodDisabledError @@ -247,6 +259,7 @@ func (v *S3Volume) Touch(loc string) error { return v.translateError(err) } +// Mtime returns the stored timestamp for the given locator. func (v *S3Volume) Mtime(loc string) (time.Time, error) { _, err := v.Bucket.Head(loc, nil) if err != nil { @@ -274,6 +287,8 @@ func (v *S3Volume) Mtime(loc string) (time.Time, error) { return v.lastModified(resp) } +// IndexTo writes a complete list of locators with the given prefix +// for which Get() can retrieve data. func (v *S3Volume) IndexTo(prefix string, writer io.Writer) error { // Use a merge sort to find matching sets of X and recent/X. dataL := s3Lister{ @@ -283,7 +298,7 @@ func (v *S3Volume) IndexTo(prefix string, writer io.Writer) error { } recentL := s3Lister{ Bucket: v.Bucket, - Prefix: "recent/"+prefix, + Prefix: "recent/" + prefix, PageSize: v.indexPageSize, } for data, recent := dataL.First(), recentL.First(); data != nil; data = dataL.Next() { @@ -341,43 +356,57 @@ func (v *S3Volume) Trash(loc string) error { } if trashLifetime == 0 { if !s3UnsafeDelete { - return ErrS3TrashNotAvailable + return ErrS3TrashDisabled } return v.Bucket.Del(loc) } - - // Make sure we're not in the race window. - { - resp, err := v.Bucket.Head("trash/"+loc, nil) - err = v.translateError(err) - if os.IsNotExist(err) { - // OK, trash/X doesn't exist so we're not in - // the race window - } else if err != nil { - // Error looking up trash/X. Unsafe to proceed - // without knowing whether we're in the race - // window - return err - } else if t, err := v.lastModified(resp); err != nil { - // Can't parse timestamp - return err - } else if safeWindow := t.Add(trashLifetime).Sub(time.Now().Add(v.raceWindow)); safeWindow <= 0 { - // We can't count on "touch trash/X" to - // prolong trash/X's lifetime. The new - // timestamp might not become visible until - // now+raceWindow, and EmptyTrash is allowed - // to delete trash/X before then. - return fmt.Errorf("same block is already in trash, and safe window ended %s ago", -safeWindow) - } + err := v.checkRaceWindow(loc) + if err != nil { + return err } - - err := v.safeCopy("trash/"+loc, loc) + err = v.safeCopy("trash/"+loc, loc) if err != nil { return err } return v.translateError(v.Bucket.Del(loc)) } +// checkRaceWindow returns a non-nil error if trash/loc is, or might +// be, in the race window (i.e., it's not safe to trash loc). +func (v *S3Volume) checkRaceWindow(loc string) error { + resp, err := v.Bucket.Head("trash/"+loc, nil) + err = v.translateError(err) + if os.IsNotExist(err) { + // OK, trash/X doesn't exist so we're not in the race + // window + return nil + } else if err != nil { + // Error looking up trash/X. We don't know whether + // we're in the race window + return err + } + t, err := v.lastModified(resp) + if err != nil { + // Can't parse timestamp + return err + } + safeWindow := t.Add(trashLifetime).Sub(time.Now().Add(v.raceWindow)) + if safeWindow <= 0 { + // We can't count on "touch trash/X" to prolong + // trash/X's lifetime. The new timestamp might not + // become visible until now+raceWindow, and EmptyTrash + // is allowed to delete trash/X before then. + return fmt.Errorf("same block is already in trash, and safe window ended %s ago", -safeWindow) + } + // trash/X exists, but it won't be eligible for deletion until + // after now+raceWindow, so it's safe to overwrite it. + return nil +} + +// safeCopy calls PutCopy, and checks the response to make sure the +// copy succeeded and updated the timestamp on the destination object +// (PutCopy returns 200 OK if the request was received, even if the +// copy failed). func (v *S3Volume) safeCopy(dst, src string) error { resp, err := v.Bucket.PutCopy(dst, s3ACL, s3.CopyOptions{ ContentType: "application/octet-stream", @@ -411,10 +440,19 @@ func (v *S3Volume) lastModified(resp *http.Response) (t time.Time, err error) { return } +// Untrash moves block from trash back into store func (v *S3Volume) Untrash(loc string) error { - return v.safeCopy(loc, "trash/"+loc) + err := v.safeCopy(loc, "trash/"+loc) + if err != nil { + return err + } + err = v.Bucket.Put("recent/"+loc, nil, "application/octet-stream", s3ACL, s3.Options{}) + return v.translateError(err) } +// Status returns a *VolumeStatus representing the current in-use +// storage capacity and a fake available capacity that doesn't make +// the volume seem full or nearly-full. func (v *S3Volume) Status() *VolumeStatus { return &VolumeStatus{ DeviceNum: 1, @@ -423,13 +461,19 @@ func (v *S3Volume) Status() *VolumeStatus { } } +// String implements fmt.Stringer. func (v *S3Volume) String() string { return fmt.Sprintf("s3-bucket:%+q", v.Bucket.Name) } +// Writable returns false if all future Put, Mtime, and Delete calls +// are expected to fail. func (v *S3Volume) Writable() bool { return !v.readonly } + +// Replication returns the storage redundancy of the underlying +// device. Configured via command line flag. func (v *S3Volume) Replication() int { return v.replication } @@ -504,6 +548,8 @@ func (v *S3Volume) translateError(err error) error { // EmptyTrash looks for trashed blocks that exceeded trashLifetime // and deletes them from the volume. func (v *S3Volume) EmptyTrash() { + var bytesInTrash, blocksInTrash, bytesDeleted, blocksDeleted int64 + // Use a merge sort to find matching sets of trash/X and recent/X. trashL := s3Lister{ Bucket: v.Bucket, @@ -511,30 +557,61 @@ func (v *S3Volume) EmptyTrash() { PageSize: v.indexPageSize, } // Define "ready to delete" as "...when EmptyTrash started". - now := time.Now() + startT := time.Now() for trash := trashL.First(); trash != nil; trash = trashL.Next() { loc := trash.Key[6:] if !v.isKeepBlock(loc) { continue } - recent, err := v.Bucket.Head("recent/"+loc, nil) + bytesInTrash += trash.Size + blocksInTrash++ + + trashT, err := time.Parse(time.RFC3339, trash.LastModified) if err != nil { - log.Printf("warning: %s: EmptyTrash: cannot delete trash %q with no corresponding recent/* marker", v, trash.Key) + log.Printf("warning: %s: EmptyTrash: %q: parse %q: %s", v, trash.Key, trash.LastModified, err) continue } - trashT, err := time.Parse(time.RFC3339, trash.LastModified) - if err != nil { + recent, err := v.Bucket.Head("recent/"+loc, nil) + if err != nil && os.IsNotExist(v.translateError(err)) { + log.Printf("warning: %s: EmptyTrash: found trash marker %q but no %q (%s); calling Untrash", v, trash.Key, "recent/"+loc, err) + err = v.Untrash(loc) + if err != nil { + log.Printf("error: %s: EmptyTrash: Untrash(%q): %s", v, loc, err) + } + continue + } else if err != nil { + log.Printf("warning: %s: EmptyTrash: HEAD %q: %s", v, "recent/"+loc, err) continue } recentT, err := v.lastModified(recent) if err != nil { + log.Printf("warning: %s: EmptyTrash: %q: parse %q: %s", v, "recent/"+loc, recent.Header.Get("Last-Modified"), err) continue } if trashT.Sub(recentT) < blobSignatureTTL { - v.fixRace(loc) - continue + if age := startT.Sub(recentT); age >= blobSignatureTTL-v.raceWindow { + // recent/loc is too old to protect + // loc from being Trashed again during + // the raceWindow that starts if we + // delete trash/X now. + // + // Note this means (trashCheckInterval + // < blobSignatureTTL - raceWindow) is + // necessary to avoid starvation. + log.Printf("notice: %s: EmptyTrash: detected old race for %q, calling fixRace + Touch", v, loc) + v.fixRace(loc) + v.Touch(loc) + continue + } else if _, err := v.Bucket.Head(loc, nil); os.IsNotExist(err) { + log.Printf("notice: %s: EmptyTrash: detected recent race for %q, calling fixRace", v, loc) + v.fixRace(loc) + continue + } else if err != nil { + log.Printf("warning: %s: EmptyTrash: HEAD %q: %s", v, loc, err) + continue + } } - if now.Sub(trashT) < trashLifetime { + if startT.Sub(trashT) < trashLifetime { continue } err = v.Bucket.Del(trash.Key) @@ -542,9 +619,12 @@ func (v *S3Volume) EmptyTrash() { log.Printf("warning: %s: EmptyTrash: deleting %q: %s", v, trash.Key, err) continue } + bytesDeleted += trash.Size + blocksDeleted++ + _, err = v.Bucket.Head(loc, nil) if os.IsNotExist(err) { - err = v.Bucket.Del("recent/"+loc) + err = v.Bucket.Del("recent/" + loc) if err != nil { log.Printf("warning: %s: EmptyTrash: deleting %q: %s", v, "recent/"+loc, err) } @@ -555,15 +635,16 @@ func (v *S3Volume) EmptyTrash() { if err := trashL.Error(); err != nil { log.Printf("error: %s: EmptyTrash: lister: %s", v, err) } + log.Printf("EmptyTrash stats for %v: Deleted %v bytes in %v blocks. Remaining in trash: %v bytes in %v blocks.", v.String(), bytesDeleted, blocksDeleted, bytesInTrash-bytesDeleted, blocksInTrash-blocksDeleted) } type s3Lister struct { - Bucket *s3.Bucket - Prefix string - PageSize int - nextMarker string - buf []s3.Key - err error + Bucket *s3.Bucket + Prefix string + PageSize int + nextMarker string + buf []s3.Key + err error } // First fetches the first page and returns the first item. It returns