11 "github.com/AdRoll/goamz/aws"
12 "github.com/AdRoll/goamz/s3"
13 "github.com/AdRoll/goamz/s3/s3test"
14 check "gopkg.in/check.v1"
17 type TestableS3Volume struct {
21 serverClock *fakeClock
25 TestBucketName = "testbucket"
28 type fakeClock struct {
32 func (c *fakeClock) Now() time.Time {
40 // Deleting isn't safe from races, but if it's turned on
41 // anyway we do expect it to pass the generic volume tests.
45 func NewTestableS3Volume(c *check.C, raceWindow time.Duration, readonly bool, replication int) *TestableS3Volume {
47 srv, err := s3test.NewServer(&s3test.Config{Clock: clock})
48 c.Assert(err, check.IsNil)
51 Name: "test-region-1",
52 S3Endpoint: srv.URL(),
53 S3LocationConstraint: true,
56 S3: s3.New(auth, region),
59 err = bucket.PutBucket(s3.ACL("private"))
60 c.Assert(err, check.IsNil)
62 return &TestableS3Volume{
63 S3Volume: NewS3Volume(auth, region, TestBucketName, raceWindow, readonly, replication),
69 var _ = check.Suite(&StubbedS3Suite{})
71 type StubbedS3Suite struct {
72 volumes []*TestableS3Volume
75 func (s *StubbedS3Suite) TestGeneric(c *check.C) {
76 DoGenericVolumeTests(c, func(t TB) TestableVolume {
77 // Use a negative raceWindow so s3test's 1-second
78 // timestamp precision doesn't confuse fixRace.
79 return NewTestableS3Volume(c, -2*time.Second, false, 2)
83 func (s *StubbedS3Suite) TestGenericReadOnly(c *check.C) {
84 DoGenericVolumeTests(c, func(t TB) TestableVolume {
85 return NewTestableS3Volume(c, -2*time.Second, true, 2)
89 func (s *StubbedS3Suite) TestIndex(c *check.C) {
90 v := NewTestableS3Volume(c, 0, false, 2)
92 for i := 0; i < 256; i++ {
93 v.PutRaw(fmt.Sprintf("%02x%030x", i, i), []byte{102, 111, 111})
95 for _, spec := range []struct {
104 buf := new(bytes.Buffer)
105 err := v.IndexTo(spec.prefix, buf)
106 c.Check(err, check.IsNil)
108 idx := bytes.SplitAfter(buf.Bytes(), []byte{10})
109 c.Check(len(idx), check.Equals, spec.expectMatch+1)
110 c.Check(len(idx[len(idx)-1]), check.Equals, 0)
114 func (s *StubbedS3Suite) TestBackendStates(c *check.C) {
115 defer func(tl, bs time.Duration) {
117 blobSignatureTTL = bs
118 }(trashLifetime, blobSignatureTTL)
119 trashLifetime = time.Hour
120 blobSignatureTTL = time.Hour
122 v := NewTestableS3Volume(c, 5*time.Minute, false, 2)
125 putS3Obj := func(t time.Time, key string, data []byte) {
129 v.serverClock.now = &t
130 v.Bucket.Put(key, data, "application/octet-stream", s3ACL, s3.Options{})
135 for _, scenario := range []struct {
142 canGetAfterTrash bool
144 haveTrashAfterEmpty bool
148 "No related objects",
150 false, false, false, false, false, false,
153 // Stored by older version, or there was a
154 // race between EmptyTrash and Put: Trash is a
155 // no-op even though the data object is very
158 t0.Add(-48 * time.Hour), none, none,
159 true, true, true, false, false, false,
162 "Not trash, but old enough to be eligible for trash",
163 t0.Add(-24 * time.Hour), t0.Add(-2 * time.Hour), none,
164 true, true, false, false, false, false,
167 "Not trash, and not old enough to be eligible for trash",
168 t0.Add(-24 * time.Hour), t0.Add(-30 * time.Minute), none,
169 true, true, true, false, false, false,
172 "Trashed + untrashed copies exist, due to recent race between Trash and Put",
173 t0.Add(-24 * time.Hour), t0.Add(-3 * time.Minute), t0.Add(-2 * time.Minute),
174 true, true, true, true, true, false,
177 "Trashed + untrashed copies exist, trash nearly eligible for deletion: prone to Trash race",
178 t0.Add(-24 * time.Hour), t0.Add(-12 * time.Hour), t0.Add(-59 * time.Minute),
179 true, false, true, true, true, false,
182 "Trashed + untrashed copies exist, trash is eligible for deletion: prone to Trash race",
183 t0.Add(-24 * time.Hour), t0.Add(-12 * time.Hour), t0.Add(-61 * time.Minute),
184 true, false, true, true, false, false,
187 "Trashed + untrashed copies exist, due to old race between Put and unfinished Trash: emptying trash is unsafe",
188 t0.Add(-24 * time.Hour), t0.Add(-12 * time.Hour), t0.Add(-12 * time.Hour),
189 true, false, true, true, true, true,
192 "Trashed + untrashed copies exist, used to be unsafe to empty, but since made safe by fixRace+Touch",
193 t0.Add(-time.Second), t0.Add(-time.Second), t0.Add(-12 * time.Hour),
194 true, true, true, true, false, false,
197 "Trashed + untrashed copies exist because Trash operation was interrupted (no race)",
198 t0.Add(-24 * time.Hour), t0.Add(-24 * time.Hour), t0.Add(-12 * time.Hour),
199 true, false, true, true, false, false,
202 "Trash, not yet eligible for deletion",
203 none, t0.Add(-12 * time.Hour), t0.Add(-time.Minute),
204 false, false, false, true, true, false,
207 "Trash, not yet eligible for deletion, prone to races",
208 none, t0.Add(-12 * time.Hour), t0.Add(-59 * time.Minute),
209 false, false, false, true, true, false,
212 "Trash, eligible for deletion",
213 none, t0.Add(-12 * time.Hour), t0.Add(-2 * time.Hour),
214 false, false, false, true, false, false,
217 "Erroneously trashed during a race, detected before trashLifetime",
218 none, t0.Add(-30 * time.Minute), t0.Add(-29 * time.Minute),
219 true, false, true, true, true, false,
222 "Erroneously trashed during a race, rescue during EmptyTrash despite reaching trashLifetime",
223 none, t0.Add(-90 * time.Minute), t0.Add(-89 * time.Minute),
224 true, false, true, true, true, false,
227 c.Log("Scenario: ", scenario.label)
229 // We have a few tests to run for each scenario, and
230 // the tests are expected to change state. By calling
231 // this setup func between tests, we (re)create the
232 // scenario as specified, using a new unique block
233 // locator to prevent interference from previous
236 setupScenario := func() (string, []byte) {
238 blk := []byte(fmt.Sprintf("%d", nextKey))
239 loc := fmt.Sprintf("%x", md5.Sum(blk))
241 putS3Obj(scenario.dataT, loc, blk)
242 putS3Obj(scenario.recentT, "recent/"+loc, nil)
243 putS3Obj(scenario.trashT, "trash/"+loc, blk)
244 v.serverClock.now = &t0
248 loc, blk := setupScenario()
249 buf := make([]byte, len(blk))
250 _, err := v.Get(loc, buf)
251 c.Check(err == nil, check.Equals, scenario.canGet)
253 c.Check(os.IsNotExist(err), check.Equals, true)
256 loc, blk = setupScenario()
258 c.Check(err == nil, check.Equals, scenario.canTrash)
259 _, err = v.Get(loc, buf)
260 c.Check(err == nil, check.Equals, scenario.canGetAfterTrash)
262 c.Check(os.IsNotExist(err), check.Equals, true)
265 loc, blk = setupScenario()
267 c.Check(err == nil, check.Equals, scenario.canUntrash)
269 loc, blk = setupScenario()
271 _, err = v.Bucket.Head("trash/"+loc, nil)
272 c.Check(err == nil, check.Equals, scenario.haveTrashAfterEmpty)
273 if scenario.freshAfterEmpty {
274 t, err := v.Mtime(loc)
275 c.Check(err, check.IsNil)
276 // new mtime must be current (with an
277 // allowance for 1s timestamp precision)
278 c.Check(t.After(t0.Add(-time.Second)), check.Equals, true)
283 // PutRaw skips the ContentMD5 test
284 func (v *TestableS3Volume) PutRaw(loc string, block []byte) {
285 err := v.Bucket.Put(loc, block, "application/octet-stream", s3ACL, s3.Options{})
287 log.Printf("PutRaw: %+v", err)
291 // TouchWithDate turns back the clock while doing a Touch(). We assume
292 // there are no other operations happening on the same s3test server
294 func (v *TestableS3Volume) TouchWithDate(locator string, lastPut time.Time) {
295 v.serverClock.now = &lastPut
296 err := v.Bucket.Put("recent/"+locator, nil, "application/octet-stream", s3ACL, s3.Options{})
300 v.serverClock.now = nil
303 func (v *TestableS3Volume) Teardown() {