13 "git.curoverse.com/arvados.git/sdk/go/arvadostest"
17 Error(args ...interface{})
18 Errorf(format string, args ...interface{})
22 Fatal(args ...interface{})
23 Fatalf(format string, args ...interface{})
24 Log(args ...interface{})
25 Logf(format string, args ...interface{})
28 // A TestableVolumeFactory returns a new TestableVolume. The factory
29 // function, and the TestableVolume it returns, can use "t" to write
30 // logs, fail the current test, etc.
31 type TestableVolumeFactory func(t TB) TestableVolume
33 // DoGenericVolumeTests runs a set of tests that every TestableVolume
34 // is expected to pass. It calls factory to create a new TestableVolume
35 // for each test case, to avoid leaking state between tests.
36 func DoGenericVolumeTests(t TB, factory TestableVolumeFactory) {
38 testGetNoSuchBlock(t, factory)
40 testCompareNonexistent(t, factory)
41 testCompareSameContent(t, factory, TestHash, TestBlock)
42 testCompareSameContent(t, factory, EmptyHash, EmptyBlock)
43 testCompareWithCollision(t, factory, TestHash, TestBlock, []byte("baddata"))
44 testCompareWithCollision(t, factory, TestHash, TestBlock, EmptyBlock)
45 testCompareWithCollision(t, factory, EmptyHash, EmptyBlock, TestBlock)
46 testCompareWithCorruptStoredData(t, factory, TestHash, TestBlock, []byte("baddata"))
47 testCompareWithCorruptStoredData(t, factory, TestHash, TestBlock, EmptyBlock)
48 testCompareWithCorruptStoredData(t, factory, EmptyHash, EmptyBlock, []byte("baddata"))
50 testPutBlockWithSameContent(t, factory, TestHash, TestBlock)
51 testPutBlockWithSameContent(t, factory, EmptyHash, EmptyBlock)
52 testPutBlockWithDifferentContent(t, factory, arvadostest.MD5CollisionMD5, arvadostest.MD5CollisionData[0], arvadostest.MD5CollisionData[1])
53 testPutBlockWithDifferentContent(t, factory, arvadostest.MD5CollisionMD5, EmptyBlock, arvadostest.MD5CollisionData[0])
54 testPutBlockWithDifferentContent(t, factory, arvadostest.MD5CollisionMD5, arvadostest.MD5CollisionData[0], EmptyBlock)
55 testPutBlockWithDifferentContent(t, factory, EmptyHash, EmptyBlock, arvadostest.MD5CollisionData[0])
56 testPutMultipleBlocks(t, factory)
58 testPutAndTouch(t, factory)
59 testTouchNoSuchBlock(t, factory)
61 testMtimeNoSuchBlock(t, factory)
63 testIndexTo(t, factory)
65 testDeleteNewBlock(t, factory)
66 testDeleteOldBlock(t, factory)
67 testDeleteNoSuchBlock(t, factory)
69 testStatus(t, factory)
71 testString(t, factory)
73 testUpdateReadOnly(t, factory)
75 testGetConcurrent(t, factory)
76 testPutConcurrent(t, factory)
78 testPutFullBlock(t, factory)
80 testTrashUntrash(t, factory)
81 testTrashEmptyTrashUntrash(t, factory)
84 // Put a test block, get it and verify content
85 // Test should pass for both writable and read-only volumes
86 func testGet(t TB, factory TestableVolumeFactory) {
90 v.PutRaw(TestHash, TestBlock)
92 buf := make([]byte, BlockSize)
93 n, err := v.Get(TestHash, buf)
98 if bytes.Compare(buf[:n], TestBlock) != 0 {
99 t.Errorf("expected %s, got %s", string(TestBlock), string(buf))
103 // Invoke get on a block that does not exist in volume; should result in error
104 // Test should pass for both writable and read-only volumes
105 func testGetNoSuchBlock(t TB, factory TestableVolumeFactory) {
109 buf := make([]byte, BlockSize)
110 if _, err := v.Get(TestHash2, buf); err == nil {
111 t.Errorf("Expected error while getting non-existing block %v", TestHash2)
115 // Compare() should return os.ErrNotExist if the block does not exist.
116 // Otherwise, writing new data causes CompareAndTouch() to generate
117 // error logs even though everything is working fine.
118 func testCompareNonexistent(t TB, factory TestableVolumeFactory) {
122 err := v.Compare(TestHash, TestBlock)
123 if err != os.ErrNotExist {
124 t.Errorf("Got err %T %q, expected os.ErrNotExist", err, err)
128 // Put a test block and compare the locator with same content
129 // Test should pass for both writable and read-only volumes
130 func testCompareSameContent(t TB, factory TestableVolumeFactory, testHash string, testData []byte) {
134 v.PutRaw(testHash, testData)
136 // Compare the block locator with same content
137 err := v.Compare(testHash, testData)
139 t.Errorf("Got err %q, expected nil", err)
143 // Test behavior of Compare() when stored data matches expected
144 // checksum but differs from new data we need to store. Requires
145 // testHash = md5(testDataA).
147 // Test should pass for both writable and read-only volumes
148 func testCompareWithCollision(t TB, factory TestableVolumeFactory, testHash string, testDataA, testDataB []byte) {
152 v.PutRaw(testHash, testDataA)
154 // Compare the block locator with different content; collision
155 err := v.Compare(TestHash, testDataB)
157 t.Errorf("Got err nil, expected error due to collision")
161 // Test behavior of Compare() when stored data has become
162 // corrupted. Requires testHash = md5(testDataA) != md5(testDataB).
164 // Test should pass for both writable and read-only volumes
165 func testCompareWithCorruptStoredData(t TB, factory TestableVolumeFactory, testHash string, testDataA, testDataB []byte) {
169 v.PutRaw(TestHash, testDataB)
171 err := v.Compare(testHash, testDataA)
172 if err == nil || err == CollisionError {
173 t.Errorf("Got err %+v, expected non-collision error", err)
177 // Put a block and put again with same content
178 // Test is intended for only writable volumes
179 func testPutBlockWithSameContent(t TB, factory TestableVolumeFactory, testHash string, testData []byte) {
183 if v.Writable() == false {
187 err := v.Put(testHash, testData)
189 t.Errorf("Got err putting block %q: %q, expected nil", TestBlock, err)
192 err = v.Put(testHash, testData)
194 t.Errorf("Got err putting block second time %q: %q, expected nil", TestBlock, err)
198 // Put a block and put again with different content
199 // Test is intended for only writable volumes
200 func testPutBlockWithDifferentContent(t TB, factory TestableVolumeFactory, testHash string, testDataA, testDataB []byte) {
204 if v.Writable() == false {
208 v.PutRaw(testHash, testDataA)
210 putErr := v.Put(testHash, testDataB)
211 buf := make([]byte, BlockSize)
212 n, getErr := v.Get(testHash, buf)
214 // Put must not return a nil error unless it has
215 // overwritten the existing data.
216 if bytes.Compare(buf[:n], testDataB) != 0 {
217 t.Errorf("Put succeeded but Get returned %+q, expected %+q", buf[:n], testDataB)
220 // It is permissible for Put to fail, but it must
221 // leave us with either the original data, the new
222 // data, or nothing at all.
223 if getErr == nil && bytes.Compare(buf[:n], testDataA) != 0 && bytes.Compare(buf[:n], testDataB) != 0 {
224 t.Errorf("Put failed but Get returned %+q, which is neither %+q nor %+q", buf[:n], testDataA, testDataB)
229 // Put and get multiple blocks
230 // Test is intended for only writable volumes
231 func testPutMultipleBlocks(t TB, factory TestableVolumeFactory) {
235 if v.Writable() == false {
239 err := v.Put(TestHash, TestBlock)
241 t.Errorf("Got err putting block %q: %q, expected nil", TestBlock, err)
244 err = v.Put(TestHash2, TestBlock2)
246 t.Errorf("Got err putting block %q: %q, expected nil", TestBlock2, err)
249 err = v.Put(TestHash3, TestBlock3)
251 t.Errorf("Got err putting block %q: %q, expected nil", TestBlock3, err)
254 data := make([]byte, BlockSize)
255 n, err := v.Get(TestHash, data)
259 if bytes.Compare(data[:n], TestBlock) != 0 {
260 t.Errorf("Block present, but got %+q, expected %+q", data[:n], TestBlock)
264 n, err = v.Get(TestHash2, data)
268 if bytes.Compare(data[:n], TestBlock2) != 0 {
269 t.Errorf("Block present, but got %+q, expected %+q", data[:n], TestBlock2)
273 n, err = v.Get(TestHash3, data)
277 if bytes.Compare(data[:n], TestBlock3) != 0 {
278 t.Errorf("Block present, but to %+q, expected %+q", data[:n], TestBlock3)
284 // Test that when applying PUT to a block that already exists,
285 // the block's modification time is updated.
286 // Test is intended for only writable volumes
287 func testPutAndTouch(t TB, factory TestableVolumeFactory) {
291 if v.Writable() == false {
295 if err := v.Put(TestHash, TestBlock); err != nil {
299 // We'll verify { t0 < threshold < t1 }, where t0 is the
300 // existing block's timestamp on disk before Put() and t1 is
301 // its timestamp after Put().
302 threshold := time.Now().Add(-time.Second)
304 // Set the stored block's mtime far enough in the past that we
305 // can see the difference between "timestamp didn't change"
306 // and "timestamp granularity is too low".
307 v.TouchWithDate(TestHash, time.Now().Add(-20*time.Second))
309 // Make sure v.Mtime() agrees the above Utime really worked.
310 if t0, err := v.Mtime(TestHash); err != nil || t0.IsZero() || !t0.Before(threshold) {
311 t.Errorf("Setting mtime failed: %v, %v", t0, err)
314 // Write the same block again.
315 if err := v.Put(TestHash, TestBlock); err != nil {
319 // Verify threshold < t1
320 if t1, err := v.Mtime(TestHash); err != nil {
322 } else if t1.Before(threshold) {
323 t.Errorf("t1 %v should be >= threshold %v after v.Put ", t1, threshold)
327 // Touching a non-existing block should result in error.
328 // Test should pass for both writable and read-only volumes
329 func testTouchNoSuchBlock(t TB, factory TestableVolumeFactory) {
333 if err := v.Touch(TestHash); err == nil {
334 t.Error("Expected error when attempted to touch a non-existing block")
338 // Invoking Mtime on a non-existing block should result in error.
339 // Test should pass for both writable and read-only volumes
340 func testMtimeNoSuchBlock(t TB, factory TestableVolumeFactory) {
344 if _, err := v.Mtime("12345678901234567890123456789012"); err == nil {
345 t.Error("Expected error when updating Mtime on a non-existing block")
349 // Put a few blocks and invoke IndexTo with:
352 // * with no such prefix
353 // Test should pass for both writable and read-only volumes
354 func testIndexTo(t TB, factory TestableVolumeFactory) {
358 v.PutRaw(TestHash, TestBlock)
359 v.PutRaw(TestHash2, TestBlock2)
360 v.PutRaw(TestHash3, TestBlock3)
362 // Blocks whose names aren't Keep hashes should be omitted from
364 v.PutRaw("fffffffffnotreallyahashfffffffff", nil)
365 v.PutRaw("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF", nil)
366 v.PutRaw("f0000000000000000000000000000000f", nil)
369 buf := new(bytes.Buffer)
371 indexRows := strings.Split(string(buf.Bytes()), "\n")
372 sort.Strings(indexRows)
373 sortedIndex := strings.Join(indexRows, "\n")
374 m, err := regexp.MatchString(
375 `^\n`+TestHash+`\+\d+ \d+\n`+
376 TestHash3+`\+\d+ \d+\n`+
377 TestHash2+`\+\d+ \d+$`,
382 t.Errorf("Got index %q for empty prefix", sortedIndex)
385 for _, prefix := range []string{"f", "f15", "f15ac"} {
386 buf = new(bytes.Buffer)
387 v.IndexTo(prefix, buf)
389 m, err := regexp.MatchString(`^`+TestHash2+`\+\d+ \d+\n$`, string(buf.Bytes()))
393 t.Errorf("Got index %q for prefix %s", string(buf.Bytes()), prefix)
397 for _, prefix := range []string{"zero", "zip", "zilch"} {
398 buf = new(bytes.Buffer)
399 v.IndexTo(prefix, buf)
401 t.Errorf("Got error on IndexTo with no such prefix %v", err.Error())
402 } else if buf.Len() != 0 {
403 t.Errorf("Expected empty list for IndexTo with no such prefix %s", prefix)
408 // Calling Delete() for a block immediately after writing it (not old enough)
409 // should neither delete the data nor return an error.
410 // Test is intended for only writable volumes
411 func testDeleteNewBlock(t TB, factory TestableVolumeFactory) {
414 blobSignatureTTL = 300 * time.Second
416 if v.Writable() == false {
420 v.Put(TestHash, TestBlock)
422 if err := v.Trash(TestHash); err != nil {
425 data := make([]byte, BlockSize)
426 n, err := v.Get(TestHash, data)
429 } else if bytes.Compare(data[:n], TestBlock) != 0 {
430 t.Errorf("Got data %+q, expected %+q", data[:n], TestBlock)
434 // Calling Delete() for a block with a timestamp older than
435 // blobSignatureTTL seconds in the past should delete the data.
436 // Test is intended for only writable volumes
437 func testDeleteOldBlock(t TB, factory TestableVolumeFactory) {
440 blobSignatureTTL = 300 * time.Second
442 if v.Writable() == false {
446 v.Put(TestHash, TestBlock)
447 v.TouchWithDate(TestHash, time.Now().Add(-2*blobSignatureTTL))
449 if err := v.Trash(TestHash); err != nil {
452 data := make([]byte, BlockSize)
453 if _, err := v.Get(TestHash, data); err == nil || !os.IsNotExist(err) {
454 t.Errorf("os.IsNotExist(%v) should have been true", err)
458 // Calling Delete() for a block that does not exist should result in error.
459 // Test should pass for both writable and read-only volumes
460 func testDeleteNoSuchBlock(t TB, factory TestableVolumeFactory) {
464 if err := v.Trash(TestHash2); err == nil {
465 t.Errorf("Expected error when attempting to delete a non-existing block")
469 // Invoke Status and verify that VolumeStatus is returned
470 // Test should pass for both writable and read-only volumes
471 func testStatus(t TB, factory TestableVolumeFactory) {
475 // Get node status and make a basic sanity check.
477 if status.DeviceNum == 0 {
478 t.Errorf("uninitialized device_num in %v", status)
481 if status.BytesFree == 0 {
482 t.Errorf("uninitialized bytes_free in %v", status)
485 if status.BytesUsed == 0 {
486 t.Errorf("uninitialized bytes_used in %v", status)
490 // Invoke String for the volume; expect non-empty result
491 // Test should pass for both writable and read-only volumes
492 func testString(t TB, factory TestableVolumeFactory) {
496 if id := v.String(); len(id) == 0 {
497 t.Error("Got empty string for v.String()")
501 // Putting, updating, touching, and deleting blocks from a read-only volume result in error.
502 // Test is intended for only read-only volumes
503 func testUpdateReadOnly(t TB, factory TestableVolumeFactory) {
507 if v.Writable() == true {
511 v.PutRaw(TestHash, TestBlock)
512 buf := make([]byte, BlockSize)
514 // Get from read-only volume should succeed
515 _, err := v.Get(TestHash, buf)
517 t.Errorf("got err %v, expected nil", err)
520 // Put a new block to read-only volume should result in error
521 err = v.Put(TestHash2, TestBlock2)
523 t.Errorf("Expected error when putting block in a read-only volume")
525 _, err = v.Get(TestHash2, buf)
527 t.Errorf("Expected error when getting block whose put in read-only volume failed")
530 // Touch a block in read-only volume should result in error
531 err = v.Touch(TestHash)
533 t.Errorf("Expected error when touching block in a read-only volume")
536 // Delete a block from a read-only volume should result in error
537 err = v.Trash(TestHash)
539 t.Errorf("Expected error when deleting block from a read-only volume")
542 // Overwriting an existing block in read-only volume should result in error
543 err = v.Put(TestHash, TestBlock)
545 t.Errorf("Expected error when putting block in a read-only volume")
549 // Launch concurrent Gets
550 // Test should pass for both writable and read-only volumes
551 func testGetConcurrent(t TB, factory TestableVolumeFactory) {
555 v.PutRaw(TestHash, TestBlock)
556 v.PutRaw(TestHash2, TestBlock2)
557 v.PutRaw(TestHash3, TestBlock3)
559 sem := make(chan int)
561 buf := make([]byte, BlockSize)
562 n, err := v.Get(TestHash, buf)
564 t.Errorf("err1: %v", err)
566 if bytes.Compare(buf[:n], TestBlock) != 0 {
567 t.Errorf("buf should be %s, is %s", string(TestBlock), string(buf[:n]))
573 buf := make([]byte, BlockSize)
574 n, err := v.Get(TestHash2, buf)
576 t.Errorf("err2: %v", err)
578 if bytes.Compare(buf[:n], TestBlock2) != 0 {
579 t.Errorf("buf should be %s, is %s", string(TestBlock2), string(buf[:n]))
585 buf := make([]byte, BlockSize)
586 n, err := v.Get(TestHash3, buf)
588 t.Errorf("err3: %v", err)
590 if bytes.Compare(buf[:n], TestBlock3) != 0 {
591 t.Errorf("buf should be %s, is %s", string(TestBlock3), string(buf[:n]))
596 // Wait for all goroutines to finish
597 for done := 0; done < 3; done++ {
602 // Launch concurrent Puts
603 // Test is intended for only writable volumes
604 func testPutConcurrent(t TB, factory TestableVolumeFactory) {
608 if v.Writable() == false {
612 sem := make(chan int)
613 go func(sem chan int) {
614 err := v.Put(TestHash, TestBlock)
616 t.Errorf("err1: %v", err)
621 go func(sem chan int) {
622 err := v.Put(TestHash2, TestBlock2)
624 t.Errorf("err2: %v", err)
629 go func(sem chan int) {
630 err := v.Put(TestHash3, TestBlock3)
632 t.Errorf("err3: %v", err)
637 // Wait for all goroutines to finish
638 for done := 0; done < 3; done++ {
642 // Double check that we actually wrote the blocks we expected to write.
643 buf := make([]byte, BlockSize)
644 n, err := v.Get(TestHash, buf)
646 t.Errorf("Get #1: %v", err)
648 if bytes.Compare(buf[:n], TestBlock) != 0 {
649 t.Errorf("Get #1: expected %s, got %s", string(TestBlock), string(buf[:n]))
652 n, err = v.Get(TestHash2, buf)
654 t.Errorf("Get #2: %v", err)
656 if bytes.Compare(buf[:n], TestBlock2) != 0 {
657 t.Errorf("Get #2: expected %s, got %s", string(TestBlock2), string(buf[:n]))
660 n, err = v.Get(TestHash3, buf)
662 t.Errorf("Get #3: %v", err)
664 if bytes.Compare(buf[:n], TestBlock3) != 0 {
665 t.Errorf("Get #3: expected %s, got %s", string(TestBlock3), string(buf[:n]))
669 // Write and read back a full size block
670 func testPutFullBlock(t TB, factory TestableVolumeFactory) {
678 wdata := make([]byte, BlockSize)
680 wdata[BlockSize-1] = 'z'
681 hash := fmt.Sprintf("%x", md5.Sum(wdata))
682 err := v.Put(hash, wdata)
686 buf := make([]byte, BlockSize)
687 n, err := v.Get(hash, buf)
691 if bytes.Compare(buf[:n], wdata) != 0 {
692 t.Error("buf %+q != wdata %+q", buf[:n], wdata)
696 // With trashLifetime != 0, perform:
697 // Trash an old block - which either raises ErrNotImplemented or succeeds
698 // Untrash - which either raises ErrNotImplemented or succeeds
699 // Get - which must succeed
700 func testTrashUntrash(t TB, factory TestableVolumeFactory) {
704 trashLifetime = 0 * time.Second
707 trashLifetime = 3600 * time.Second
709 // put block and backdate it
710 v.PutRaw(TestHash, TestBlock)
711 v.TouchWithDate(TestHash, time.Now().Add(-2*blobSignatureTTL))
713 buf := make([]byte, BlockSize)
714 n, err := v.Get(TestHash, buf)
718 if bytes.Compare(buf[:n], TestBlock) != 0 {
719 t.Errorf("Got data %+q, expected %+q", buf[:n], TestBlock)
723 err = v.Trash(TestHash)
724 if v.Writable() == false {
725 if err != MethodDisabledError {
728 } else if err != nil {
729 if err != ErrNotImplemented {
733 _, err = v.Get(TestHash, buf)
734 if err == nil || !os.IsNotExist(err) {
735 t.Errorf("os.IsNotExist(%v) should have been true", err)
739 err = v.Untrash(TestHash)
745 // Get the block - after trash and untrash sequence
746 n, err = v.Get(TestHash, buf)
750 if bytes.Compare(buf[:n], TestBlock) != 0 {
751 t.Errorf("Got data %+q, expected %+q", buf[:n], TestBlock)
755 func testTrashEmptyTrashUntrash(t TB, factory TestableVolumeFactory) {
758 defer func(orig time.Duration) {
762 checkGet := func() error {
763 buf := make([]byte, BlockSize)
764 n, err := v.Get(TestHash, buf)
768 if bytes.Compare(buf[:n], TestBlock) != 0 {
769 t.Fatalf("Got data %+q, expected %+q", buf[:n], TestBlock)
774 // First set: EmptyTrash before reaching the trash deadline.
776 trashLifetime = 1 * time.Hour
778 v.PutRaw(TestHash, TestBlock)
779 v.TouchWithDate(TestHash, time.Now().Add(-2*blobSignatureTTL))
786 err = v.Trash(TestHash)
787 if err == MethodDisabledError || err == ErrNotImplemented {
788 // Skip the trash tests for read-only volumes, and
789 // volume types that don't support trashLifetime>0.
794 if err == nil || !os.IsNotExist(err) {
795 t.Fatalf("os.IsNotExist(%v) should have been true", err)
800 // Even after emptying the trash, we can untrash our block
801 // because the deadline hasn't been reached.
802 err = v.Untrash(TestHash)
811 // Untrash should fail if the only block in the trash has
812 // already been untrashed.
813 err = v.Untrash(TestHash)
814 if err == nil || !os.IsNotExist(err) {
815 t.Fatalf("os.IsNotExist(%v) should have been true", err)
818 // The failed Untrash should not interfere with our
819 // already-untrashed copy.
825 // Second set: EmptyTrash after the trash deadline has passed.
827 trashLifetime = 1 * time.Nanosecond
829 err = v.Trash(TestHash)
834 if err == nil || !os.IsNotExist(err) {
835 t.Fatalf("os.IsNotExist(%v) should have been true", err)
838 // Even though 1ns has passed, we can untrash because we
839 // haven't called EmptyTrash yet.
840 err = v.Untrash(TestHash)
849 // Trash it again, and this time call EmptyTrash so it really
851 err = v.Trash(TestHash)
853 if err == nil || !os.IsNotExist(err) {
854 t.Errorf("os.IsNotExist(%v) should have been true", err)
858 // Untrash won't find it
859 err = v.Untrash(TestHash)
860 if err == nil || !os.IsNotExist(err) {
861 t.Fatalf("os.IsNotExist(%v) should have been true", err)
864 // Get block won't find it
866 if err == nil || !os.IsNotExist(err) {
867 t.Fatalf("os.IsNotExist(%v) should have been true", err)
870 // Third set: If the same data block gets written again after
871 // being trashed, and then the trash gets emptied, the newer
872 // un-trashed copy doesn't get deleted along with it.
874 v.PutRaw(TestHash, TestBlock)
875 v.TouchWithDate(TestHash, time.Now().Add(-2*blobSignatureTTL))
877 trashLifetime = time.Nanosecond
878 err = v.Trash(TestHash)
883 if err == nil || !os.IsNotExist(err) {
884 t.Fatalf("os.IsNotExist(%v) should have been true", err)
887 v.PutRaw(TestHash, TestBlock)
888 v.TouchWithDate(TestHash, time.Now().Add(-2*blobSignatureTTL))
890 // EmptyTrash should not delete the untrashed copy.
897 // Fourth set: If the same data block gets trashed twice with
898 // different deadlines A and C, and then the trash is emptied
899 // at intermediate time B (A < B < C), it is still possible to
900 // untrash the block whose deadline is "C".
902 v.PutRaw(TestHash, TestBlock)
903 v.TouchWithDate(TestHash, time.Now().Add(-2*blobSignatureTTL))
905 trashLifetime = time.Nanosecond
906 err = v.Trash(TestHash)
911 v.PutRaw(TestHash, TestBlock)
912 v.TouchWithDate(TestHash, time.Now().Add(-2*blobSignatureTTL))
914 trashLifetime = time.Hour
915 err = v.Trash(TestHash)
920 // EmptyTrash should not prevent us from recovering the
921 // time.Hour ("C") trash
923 err = v.Untrash(TestHash)