X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/a6d2f88debdfa7bc390b63c1f18a0541987ae0b8..1d656f4f1ec1f890a7677e748bea43a08cfa0b6c:/services/keepstore/s3_volume.go diff --git a/services/keepstore/s3_volume.go b/services/keepstore/s3_volume.go index 5b00d4abc5..caed35b670 100644 --- a/services/keepstore/s3_volume.go +++ b/services/keepstore/s3_volume.go @@ -11,15 +11,17 @@ import ( "os" "regexp" "strings" + "sync" "time" + "git.curoverse.com/arvados.git/sdk/go/arvados" "github.com/AdRoll/goamz/aws" "github.com/AdRoll/goamz/s3" ) var ( - // Returned by Trash if that operation is impossible with the - // current config. + // ErrS3TrashDisabled is returned by Trash if that operation + // is impossible with the current config. ErrS3TrashDisabled = fmt.Errorf("trash function is disabled because -trash-lifetime=0 and -s3-unsafe-delete=false") s3AccessKeyFile string @@ -39,52 +41,36 @@ const ( ) type s3VolumeAdder struct { - *volumeSet + *Config +} + +// String implements flag.Value +func (s *s3VolumeAdder) String() string { + return "-" } func (s *s3VolumeAdder) Set(bucketName string) error { - if trashLifetime != 0 { - return ErrNotImplemented - } if bucketName == "" { return fmt.Errorf("no container name given") } if s3AccessKeyFile == "" || s3SecretKeyFile == "" { return fmt.Errorf("-s3-access-key-file and -s3-secret-key-file arguments must given before -s3-bucket-volume") } - region, ok := aws.Regions[s3RegionName] - if s3Endpoint == "" { - if !ok { - return fmt.Errorf("unrecognized region %+q; try specifying -s3-endpoint instead", s3RegionName) - } - } else { - if ok { - return fmt.Errorf("refusing to use AWS region name %+q with endpoint %+q; "+ - "specify empty endpoint (\"-s3-endpoint=\") or use a different region name", s3RegionName, s3Endpoint) - } - region = aws.Region{ - Name: s3RegionName, - S3Endpoint: s3Endpoint, - } - } - var err error - var auth aws.Auth - auth.AccessKey, err = readKeyFromFile(s3AccessKeyFile) - if err != nil { - return err - } - auth.SecretKey, err = readKeyFromFile(s3SecretKeyFile) - if err != nil { - return err - } - if flagSerializeIO { + if deprecated.flagSerializeIO { log.Print("Notice: -serialize is not supported by s3-bucket volumes.") } - v := NewS3Volume(auth, region, bucketName, s3RaceWindow, flagReadonly, s3Replication) - if err := v.Check(); err != nil { - return err - } - *s.volumeSet = append(*s.volumeSet, v) + s.Config.Volumes = append(s.Config.Volumes, &S3Volume{ + Bucket: bucketName, + AccessKeyFile: s3AccessKeyFile, + SecretKeyFile: s3SecretKeyFile, + Endpoint: s3Endpoint, + Region: s3RegionName, + RaceWindow: arvados.Duration(s3RaceWindow), + S3Replication: s3Replication, + UnsafeDelete: s3UnsafeDelete, + ReadOnly: deprecated.flagReadonly, + IndexPageSize: 1000, + }) return nil } @@ -96,7 +82,9 @@ func s3regions() (okList []string) { } func init() { - flag.Var(&s3VolumeAdder{&volumes}, + VolumeTypes = append(VolumeTypes, func() VolumeWithExamples { return &S3Volume{} }) + + flag.Var(&s3VolumeAdder{theConfig}, "s3-bucket-volume", "Use the given bucket as a storage volume. Can be given multiple times.") flag.StringVar( @@ -113,12 +101,12 @@ func init() { &s3AccessKeyFile, "s3-access-key-file", "", - "File containing the access key used for subsequent -s3-bucket-volume arguments.") + "`File` containing the access key used for subsequent -s3-bucket-volume arguments.") flag.StringVar( &s3SecretKeyFile, "s3-secret-key-file", "", - "File containing the secret key used for subsequent -s3-bucket-volume arguments.") + "`File` containing the secret key used for subsequent -s3-bucket-volume arguments.") flag.DurationVar( &s3RaceWindow, "s3-race-window", @@ -138,32 +126,87 @@ func init() { // S3Volume implements Volume using an S3 bucket. type S3Volume struct { - *s3.Bucket - raceWindow time.Duration - readonly bool - replication int - indexPageSize int -} - -// NewS3Volume returns a new S3Volume using the given auth, region, -// and bucket name. The replication argument specifies the replication -// level to report when writing data. -func NewS3Volume(auth aws.Auth, region aws.Region, bucket string, raceWindow time.Duration, readonly bool, replication int) *S3Volume { - return &S3Volume{ - Bucket: &s3.Bucket{ - S3: s3.New(auth, region), - Name: bucket, + AccessKeyFile string + SecretKeyFile string + Endpoint string + Region string + Bucket string + LocationConstraint bool + IndexPageSize int + S3Replication int + RaceWindow arvados.Duration + ReadOnly bool + UnsafeDelete bool + + bucket *s3.Bucket + + startOnce sync.Once +} + +// Examples implements VolumeWithExamples. +func (*S3Volume) Examples() []Volume { + return []Volume{ + &S3Volume{ + AccessKeyFile: "/etc/aws_s3_access_key.txt", + SecretKeyFile: "/etc/aws_s3_secret_key.txt", + Endpoint: "", + Region: "us-east-1", + Bucket: "example-bucket-name", + IndexPageSize: 1000, + S3Replication: 2, + RaceWindow: arvados.Duration(24 * time.Hour), + }, + &S3Volume{ + AccessKeyFile: "/etc/gce_s3_access_key.txt", + SecretKeyFile: "/etc/gce_s3_secret_key.txt", + Endpoint: "https://storage.googleapis.com", + Region: "", + Bucket: "example-bucket-name", + IndexPageSize: 1000, + S3Replication: 2, + RaceWindow: arvados.Duration(24 * time.Hour), }, - raceWindow: raceWindow, - readonly: readonly, - replication: replication, - indexPageSize: 1000, } } -// Check returns an error if the volume is inaccessible (e.g., config -// error). -func (v *S3Volume) Check() error { +// Type implements Volume. +func (*S3Volume) Type() string { + return "S3" +} + +// Start populates private fields and verifies the configuration is +// valid. +func (v *S3Volume) Start() error { + region, ok := aws.Regions[v.Region] + if v.Endpoint == "" { + if !ok { + return fmt.Errorf("unrecognized region %+q; try specifying -s3-endpoint instead", v.Region) + } + } else if ok { + return fmt.Errorf("refusing to use AWS region name %+q with endpoint %+q; "+ + "specify empty endpoint (\"-s3-endpoint=\") or use a different region name", v.Region, v.Endpoint) + } else { + region = aws.Region{ + Name: v.Region, + S3Endpoint: v.Endpoint, + S3LocationConstraint: v.LocationConstraint, + } + } + + var err error + var auth aws.Auth + auth.AccessKey, err = readKeyFromFile(v.AccessKeyFile) + if err != nil { + return err + } + auth.SecretKey, err = readKeyFromFile(v.SecretKeyFile) + if err != nil { + return err + } + v.bucket = &s3.Bucket{ + S3: s3.New(auth, region), + Name: v.Bucket, + } return nil } @@ -173,12 +216,12 @@ func (v *S3Volume) Check() error { // disappeared in a Trash race, getReader calls fixRace to recover the // data, and tries again. func (v *S3Volume) getReader(loc string) (rdr io.ReadCloser, err error) { - rdr, err = v.Bucket.GetReader(loc) + rdr, err = v.bucket.GetReader(loc) err = v.translateError(err) if err == nil || !os.IsNotExist(err) { return } - _, err = v.Bucket.Head("recent/"+loc, nil) + _, err = v.bucket.Head("recent/"+loc, nil) err = v.translateError(err) if err != nil { // If we can't read recent/X, there's no point in @@ -189,7 +232,7 @@ func (v *S3Volume) getReader(loc string) (rdr io.ReadCloser, err error) { err = os.ErrNotExist return } - rdr, err = v.Bucket.GetReader(loc) + rdr, err = v.bucket.GetReader(loc) if err != nil { log.Printf("warning: reading %s after successful fixRace: %s", loc, err) err = v.translateError(err) @@ -226,7 +269,7 @@ func (v *S3Volume) Compare(loc string, expect []byte) error { // Put writes a block. func (v *S3Volume) Put(loc string, block []byte) error { - if v.readonly { + if v.ReadOnly { return MethodDisabledError } var opts s3.Options @@ -237,20 +280,20 @@ func (v *S3Volume) Put(loc string, block []byte) error { } opts.ContentMD5 = base64.StdEncoding.EncodeToString(md5) } - err := v.Bucket.Put(loc, block, "application/octet-stream", s3ACL, opts) + err := v.bucket.Put(loc, block, "application/octet-stream", s3ACL, opts) if err != nil { return v.translateError(err) } - err = v.Bucket.Put("recent/"+loc, nil, "application/octet-stream", s3ACL, s3.Options{}) + err = v.bucket.Put("recent/"+loc, nil, "application/octet-stream", s3ACL, s3.Options{}) return v.translateError(err) } // Touch sets the timestamp for the given locator to the current time. func (v *S3Volume) Touch(loc string) error { - if v.readonly { + if v.ReadOnly { return MethodDisabledError } - _, err := v.Bucket.Head(loc, nil) + _, err := v.bucket.Head(loc, nil) err = v.translateError(err) if os.IsNotExist(err) && v.fixRace(loc) { // The data object got trashed in a race, but fixRace @@ -258,27 +301,27 @@ func (v *S3Volume) Touch(loc string) error { } else if err != nil { return err } - err = v.Bucket.Put("recent/"+loc, nil, "application/octet-stream", s3ACL, s3.Options{}) + err = v.bucket.Put("recent/"+loc, nil, "application/octet-stream", s3ACL, s3.Options{}) return v.translateError(err) } // Mtime returns the stored timestamp for the given locator. func (v *S3Volume) Mtime(loc string) (time.Time, error) { - _, err := v.Bucket.Head(loc, nil) + _, err := v.bucket.Head(loc, nil) if err != nil { return zeroTime, v.translateError(err) } - resp, err := v.Bucket.Head("recent/"+loc, nil) + resp, err := v.bucket.Head("recent/"+loc, nil) err = v.translateError(err) if os.IsNotExist(err) { // The data object X exists, but recent/X is missing. - err = v.Bucket.Put("recent/"+loc, nil, "application/octet-stream", s3ACL, s3.Options{}) + err = v.bucket.Put("recent/"+loc, nil, "application/octet-stream", s3ACL, s3.Options{}) if err != nil { log.Printf("error: creating %q: %s", "recent/"+loc, err) return zeroTime, v.translateError(err) } log.Printf("info: created %q to migrate existing block to new storage scheme", "recent/"+loc) - resp, err = v.Bucket.Head("recent/"+loc, nil) + resp, err = v.bucket.Head("recent/"+loc, nil) if err != nil { log.Printf("error: created %q but HEAD failed: %s", "recent/"+loc, err) return zeroTime, v.translateError(err) @@ -295,14 +338,14 @@ func (v *S3Volume) Mtime(loc string) (time.Time, error) { func (v *S3Volume) IndexTo(prefix string, writer io.Writer) error { // Use a merge sort to find matching sets of X and recent/X. dataL := s3Lister{ - Bucket: v.Bucket, + Bucket: v.bucket, Prefix: prefix, - PageSize: v.indexPageSize, + PageSize: v.IndexPageSize, } recentL := s3Lister{ - Bucket: v.Bucket, + Bucket: v.bucket, Prefix: "recent/" + prefix, - PageSize: v.indexPageSize, + PageSize: v.IndexPageSize, } for data, recent := dataL.First(), recentL.First(); data != nil; data = dataL.Next() { if data.Key >= "g" { @@ -349,19 +392,19 @@ func (v *S3Volume) IndexTo(prefix string, writer io.Writer) error { // Trash a Keep block. func (v *S3Volume) Trash(loc string) error { - if v.readonly { + if v.ReadOnly { return MethodDisabledError } if t, err := v.Mtime(loc); err != nil { return err - } else if time.Since(t) < blobSignatureTTL { + } else if time.Since(t) < theConfig.BlobSignatureTTL.Duration() { return nil } - if trashLifetime == 0 { + if theConfig.TrashLifetime == 0 { if !s3UnsafeDelete { return ErrS3TrashDisabled } - return v.Bucket.Del(loc) + return v.bucket.Del(loc) } err := v.checkRaceWindow(loc) if err != nil { @@ -371,13 +414,13 @@ func (v *S3Volume) Trash(loc string) error { if err != nil { return err } - return v.translateError(v.Bucket.Del(loc)) + return v.translateError(v.bucket.Del(loc)) } // checkRaceWindow returns a non-nil error if trash/loc is, or might // be, in the race window (i.e., it's not safe to trash loc). func (v *S3Volume) checkRaceWindow(loc string) error { - resp, err := v.Bucket.Head("trash/"+loc, nil) + resp, err := v.bucket.Head("trash/"+loc, nil) err = v.translateError(err) if os.IsNotExist(err) { // OK, trash/X doesn't exist so we're not in the race @@ -393,7 +436,7 @@ func (v *S3Volume) checkRaceWindow(loc string) error { // Can't parse timestamp return err } - safeWindow := t.Add(trashLifetime).Sub(time.Now().Add(v.raceWindow)) + safeWindow := t.Add(theConfig.TrashLifetime.Duration()).Sub(time.Now().Add(time.Duration(v.RaceWindow))) if safeWindow <= 0 { // We can't count on "touch trash/X" to prolong // trash/X's lifetime. The new timestamp might not @@ -411,10 +454,10 @@ func (v *S3Volume) checkRaceWindow(loc string) error { // (PutCopy returns 200 OK if the request was received, even if the // copy failed). func (v *S3Volume) safeCopy(dst, src string) error { - resp, err := v.Bucket.PutCopy(dst, s3ACL, s3.CopyOptions{ + resp, err := v.bucket.PutCopy(dst, s3ACL, s3.CopyOptions{ ContentType: "application/octet-stream", MetadataDirective: "REPLACE", - }, v.Bucket.Name+"/"+src) + }, v.bucket.Name+"/"+src) err = v.translateError(err) if err != nil { return err @@ -449,7 +492,7 @@ func (v *S3Volume) Untrash(loc string) error { if err != nil { return err } - err = v.Bucket.Put("recent/"+loc, nil, "application/octet-stream", s3ACL, s3.Options{}) + err = v.bucket.Put("recent/"+loc, nil, "application/octet-stream", s3ACL, s3.Options{}) return v.translateError(err) } @@ -466,19 +509,19 @@ func (v *S3Volume) Status() *VolumeStatus { // String implements fmt.Stringer. func (v *S3Volume) String() string { - return fmt.Sprintf("s3-bucket:%+q", v.Bucket.Name) + return fmt.Sprintf("s3-bucket:%+q", v.bucket.Name) } // Writable returns false if all future Put, Mtime, and Delete calls // are expected to fail. func (v *S3Volume) Writable() bool { - return !v.readonly + return !v.ReadOnly } // Replication returns the storage redundancy of the underlying // device. Configured via command line flag. func (v *S3Volume) Replication() int { - return v.replication + return v.S3Replication } var s3KeepBlockRegexp = regexp.MustCompile(`^[0-9a-f]{32}$`) @@ -492,7 +535,7 @@ func (v *S3Volume) isKeepBlock(s string) bool { // there was a race between Put and Trash, fixRace recovers from the // race by Untrashing the block. func (v *S3Volume) fixRace(loc string) bool { - trash, err := v.Bucket.Head("trash/"+loc, nil) + trash, err := v.bucket.Head("trash/"+loc, nil) if err != nil { if !os.IsNotExist(v.translateError(err)) { log.Printf("error: fixRace: HEAD %q: %s", "trash/"+loc, err) @@ -505,7 +548,7 @@ func (v *S3Volume) fixRace(loc string) bool { return false } - recent, err := v.Bucket.Head("recent/"+loc, nil) + recent, err := v.bucket.Head("recent/"+loc, nil) if err != nil { log.Printf("error: fixRace: HEAD %q: %s", "recent/"+loc, err) return false @@ -517,13 +560,13 @@ func (v *S3Volume) fixRace(loc string) bool { } ageWhenTrashed := trashTime.Sub(recentTime) - if ageWhenTrashed >= blobSignatureTTL { + if ageWhenTrashed >= theConfig.BlobSignatureTTL.Duration() { // No evidence of a race: block hasn't been written // since it became eligible for Trash. No fix needed. return false } - log.Printf("notice: fixRace: %q: trashed at %s but touched at %s (age when trashed = %s < %s)", loc, trashTime, recentTime, ageWhenTrashed, blobSignatureTTL) + log.Printf("notice: fixRace: %q: trashed at %s but touched at %s (age when trashed = %s < %s)", loc, trashTime, recentTime, ageWhenTrashed, theConfig.BlobSignatureTTL) log.Printf("notice: fixRace: copying %q to %q to recover from race between Put/Touch and Trash", "recent/"+loc, loc) err = v.safeCopy(loc, "trash/"+loc) if err != nil { @@ -548,52 +591,86 @@ func (v *S3Volume) translateError(err error) error { return err } -// EmptyTrash looks for trashed blocks that exceeded trashLifetime +// EmptyTrash looks for trashed blocks that exceeded TrashLifetime // and deletes them from the volume. func (v *S3Volume) EmptyTrash() { + var bytesInTrash, blocksInTrash, bytesDeleted, blocksDeleted int64 + // Use a merge sort to find matching sets of trash/X and recent/X. trashL := s3Lister{ - Bucket: v.Bucket, + Bucket: v.bucket, Prefix: "trash/", - PageSize: v.indexPageSize, + PageSize: v.IndexPageSize, } // Define "ready to delete" as "...when EmptyTrash started". - now := time.Now() + startT := time.Now() for trash := trashL.First(); trash != nil; trash = trashL.Next() { loc := trash.Key[6:] if !v.isKeepBlock(loc) { continue } - recent, err := v.Bucket.Head("recent/"+loc, nil) - if err != nil { - log.Printf("warning: %s: EmptyTrash: cannot delete trash %q with no corresponding recent/* marker", v, trash.Key) - continue - } + bytesInTrash += trash.Size + blocksInTrash++ + trashT, err := time.Parse(time.RFC3339, trash.LastModified) if err != nil { log.Printf("warning: %s: EmptyTrash: %q: parse %q: %s", v, trash.Key, trash.LastModified, err) continue } + recent, err := v.bucket.Head("recent/"+loc, nil) + if err != nil && os.IsNotExist(v.translateError(err)) { + log.Printf("warning: %s: EmptyTrash: found trash marker %q but no %q (%s); calling Untrash", v, trash.Key, "recent/"+loc, err) + err = v.Untrash(loc) + if err != nil { + log.Printf("error: %s: EmptyTrash: Untrash(%q): %s", v, loc, err) + } + continue + } else if err != nil { + log.Printf("warning: %s: EmptyTrash: HEAD %q: %s", v, "recent/"+loc, err) + continue + } recentT, err := v.lastModified(recent) if err != nil { log.Printf("warning: %s: EmptyTrash: %q: parse %q: %s", v, "recent/"+loc, recent.Header.Get("Last-Modified"), err) continue } - if trashT.Sub(recentT) < blobSignatureTTL { - v.fixRace(loc) - continue + if trashT.Sub(recentT) < theConfig.BlobSignatureTTL.Duration() { + if age := startT.Sub(recentT); age >= theConfig.BlobSignatureTTL.Duration()-time.Duration(v.RaceWindow) { + // recent/loc is too old to protect + // loc from being Trashed again during + // the raceWindow that starts if we + // delete trash/X now. + // + // Note this means (TrashCheckInterval + // < BlobSignatureTTL - raceWindow) is + // necessary to avoid starvation. + log.Printf("notice: %s: EmptyTrash: detected old race for %q, calling fixRace + Touch", v, loc) + v.fixRace(loc) + v.Touch(loc) + continue + } else if _, err := v.bucket.Head(loc, nil); os.IsNotExist(err) { + log.Printf("notice: %s: EmptyTrash: detected recent race for %q, calling fixRace", v, loc) + v.fixRace(loc) + continue + } else if err != nil { + log.Printf("warning: %s: EmptyTrash: HEAD %q: %s", v, loc, err) + continue + } } - if now.Sub(trashT) < trashLifetime { + if startT.Sub(trashT) < theConfig.TrashLifetime.Duration() { continue } - err = v.Bucket.Del(trash.Key) + err = v.bucket.Del(trash.Key) if err != nil { log.Printf("warning: %s: EmptyTrash: deleting %q: %s", v, trash.Key, err) continue } - _, err = v.Bucket.Head(loc, nil) + bytesDeleted += trash.Size + blocksDeleted++ + + _, err = v.bucket.Head(loc, nil) if os.IsNotExist(err) { - err = v.Bucket.Del("recent/" + loc) + err = v.bucket.Del("recent/" + loc) if err != nil { log.Printf("warning: %s: EmptyTrash: deleting %q: %s", v, "recent/"+loc, err) } @@ -604,6 +681,7 @@ func (v *S3Volume) EmptyTrash() { if err := trashL.Error(); err != nil { log.Printf("error: %s: EmptyTrash: lister: %s", v, err) } + log.Printf("EmptyTrash stats for %v: Deleted %v bytes in %v blocks. Remaining in trash: %v bytes in %v blocks.", v.String(), bytesDeleted, blocksDeleted, bytesInTrash-bytesDeleted, blocksInTrash-blocksDeleted) } type s3Lister struct {