X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/00a96a5bd0203a1330963aaa3b7c3f5a2f970438..c405873d87e0764acf3855c559c85fa6d7a63cfb:/services/keepstore/volume_generic_test.go diff --git a/services/keepstore/volume_generic_test.go b/services/keepstore/volume_generic_test.go index be83151461..f8fe0d0ebc 100644 --- a/services/keepstore/volume_generic_test.go +++ b/services/keepstore/volume_generic_test.go @@ -79,7 +79,6 @@ func DoGenericVolumeTests(t TB, factory TestableVolumeFactory) { testTrashUntrash(t, factory) testTrashEmptyTrashUntrash(t, factory) - testTrashUntrashWithEmptyTrashGoroutine(t, factory) } // Put a test block, get it and verify content @@ -90,14 +89,13 @@ func testGet(t TB, factory TestableVolumeFactory) { v.PutRaw(TestHash, TestBlock) - buf, err := v.Get(TestHash) + buf := make([]byte, BlockSize) + n, err := v.Get(TestHash, buf) if err != nil { t.Fatal(err) } - bufs.Put(buf) - - if bytes.Compare(buf, TestBlock) != 0 { + if bytes.Compare(buf[:n], TestBlock) != 0 { t.Errorf("expected %s, got %s", string(TestBlock), string(buf)) } } @@ -108,7 +106,8 @@ func testGetNoSuchBlock(t TB, factory TestableVolumeFactory) { v := factory(t) defer v.Teardown() - if _, err := v.Get(TestHash2); err == nil { + buf := make([]byte, BlockSize) + if _, err := v.Get(TestHash2, buf); err == nil { t.Errorf("Expected error while getting non-existing block %v", TestHash2) } } @@ -209,24 +208,22 @@ func testPutBlockWithDifferentContent(t TB, factory TestableVolumeFactory, testH v.PutRaw(testHash, testDataA) putErr := v.Put(testHash, testDataB) - buf, getErr := v.Get(testHash) + buf := make([]byte, BlockSize) + n, getErr := v.Get(testHash, buf) if putErr == nil { // Put must not return a nil error unless it has // overwritten the existing data. - if bytes.Compare(buf, testDataB) != 0 { - t.Errorf("Put succeeded but Get returned %+q, expected %+q", buf, testDataB) + if bytes.Compare(buf[:n], testDataB) != 0 { + t.Errorf("Put succeeded but Get returned %+q, expected %+q", buf[:n], testDataB) } } else { // It is permissible for Put to fail, but it must // leave us with either the original data, the new // data, or nothing at all. - if getErr == nil && bytes.Compare(buf, testDataA) != 0 && bytes.Compare(buf, testDataB) != 0 { - t.Errorf("Put failed but Get returned %+q, which is neither %+q nor %+q", buf, testDataA, testDataB) + if getErr == nil && bytes.Compare(buf[:n], testDataA) != 0 && bytes.Compare(buf[:n], testDataB) != 0 { + t.Errorf("Put failed but Get returned %+q, which is neither %+q nor %+q", buf[:n], testDataA, testDataB) } } - if getErr == nil { - bufs.Put(buf) - } } // Put and get multiple blocks @@ -254,34 +251,32 @@ func testPutMultipleBlocks(t TB, factory TestableVolumeFactory) { t.Errorf("Got err putting block %q: %q, expected nil", TestBlock3, err) } - data, err := v.Get(TestHash) + data := make([]byte, BlockSize) + n, err := v.Get(TestHash, data) if err != nil { t.Error(err) } else { - if bytes.Compare(data, TestBlock) != 0 { - t.Errorf("Block present, but got %+q, expected %+q", data, TestBlock) + if bytes.Compare(data[:n], TestBlock) != 0 { + t.Errorf("Block present, but got %+q, expected %+q", data[:n], TestBlock) } - bufs.Put(data) } - data, err = v.Get(TestHash2) + n, err = v.Get(TestHash2, data) if err != nil { t.Error(err) } else { - if bytes.Compare(data, TestBlock2) != 0 { - t.Errorf("Block present, but got %+q, expected %+q", data, TestBlock2) + if bytes.Compare(data[:n], TestBlock2) != 0 { + t.Errorf("Block present, but got %+q, expected %+q", data[:n], TestBlock2) } - bufs.Put(data) } - data, err = v.Get(TestHash3) + n, err = v.Get(TestHash3, data) if err != nil { t.Error(err) } else { - if bytes.Compare(data, TestBlock3) != 0 { - t.Errorf("Block present, but to %+q, expected %+q", data, TestBlock3) + if bytes.Compare(data[:n], TestBlock3) != 0 { + t.Errorf("Block present, but to %+q, expected %+q", data[:n], TestBlock3) } - bufs.Put(data) } } @@ -427,14 +422,12 @@ func testDeleteNewBlock(t TB, factory TestableVolumeFactory) { if err := v.Trash(TestHash); err != nil { t.Error(err) } - data, err := v.Get(TestHash) + data := make([]byte, BlockSize) + n, err := v.Get(TestHash, data) if err != nil { t.Error(err) - } else { - if bytes.Compare(data, TestBlock) != 0 { - t.Errorf("Got data %+q, expected %+q", data, TestBlock) - } - bufs.Put(data) + } else if bytes.Compare(data[:n], TestBlock) != 0 { + t.Errorf("Got data %+q, expected %+q", data[:n], TestBlock) } } @@ -456,9 +449,31 @@ func testDeleteOldBlock(t TB, factory TestableVolumeFactory) { if err := v.Trash(TestHash); err != nil { t.Error(err) } - if _, err := v.Get(TestHash); err == nil || !os.IsNotExist(err) { + data := make([]byte, BlockSize) + if _, err := v.Get(TestHash, data); err == nil || !os.IsNotExist(err) { t.Errorf("os.IsNotExist(%v) should have been true", err) } + + _, err := v.Mtime(TestHash) + if err == nil || !os.IsNotExist(err) { + t.Fatalf("os.IsNotExist(%v) should have been true", err) + } + + err = v.Compare(TestHash, TestBlock) + if err == nil || !os.IsNotExist(err) { + t.Fatalf("os.IsNotExist(%v) should have been true", err) + } + + indexBuf := new(bytes.Buffer) + v.IndexTo("", indexBuf) + if strings.Contains(string(indexBuf.Bytes()), TestHash) { + t.Fatalf("Found trashed block in IndexTo") + } + + err = v.Touch(TestHash) + if err == nil || !os.IsNotExist(err) { + t.Fatalf("os.IsNotExist(%v) should have been true", err) + } } // Calling Delete() for a block that does not exist should result in error. @@ -515,9 +530,10 @@ func testUpdateReadOnly(t TB, factory TestableVolumeFactory) { } v.PutRaw(TestHash, TestBlock) + buf := make([]byte, BlockSize) // Get from read-only volume should succeed - _, err := v.Get(TestHash) + _, err := v.Get(TestHash, buf) if err != nil { t.Errorf("got err %v, expected nil", err) } @@ -527,7 +543,7 @@ func testUpdateReadOnly(t TB, factory TestableVolumeFactory) { if err == nil { t.Errorf("Expected error when putting block in a read-only volume") } - _, err = v.Get(TestHash2) + _, err = v.Get(TestHash2, buf) if err == nil { t.Errorf("Expected error when getting block whose put in read-only volume failed") } @@ -562,45 +578,45 @@ func testGetConcurrent(t TB, factory TestableVolumeFactory) { v.PutRaw(TestHash3, TestBlock3) sem := make(chan int) - go func(sem chan int) { - buf, err := v.Get(TestHash) + go func() { + buf := make([]byte, BlockSize) + n, err := v.Get(TestHash, buf) if err != nil { t.Errorf("err1: %v", err) } - bufs.Put(buf) - if bytes.Compare(buf, TestBlock) != 0 { - t.Errorf("buf should be %s, is %s", string(TestBlock), string(buf)) + if bytes.Compare(buf[:n], TestBlock) != 0 { + t.Errorf("buf should be %s, is %s", string(TestBlock), string(buf[:n])) } sem <- 1 - }(sem) + }() - go func(sem chan int) { - buf, err := v.Get(TestHash2) + go func() { + buf := make([]byte, BlockSize) + n, err := v.Get(TestHash2, buf) if err != nil { t.Errorf("err2: %v", err) } - bufs.Put(buf) - if bytes.Compare(buf, TestBlock2) != 0 { - t.Errorf("buf should be %s, is %s", string(TestBlock2), string(buf)) + if bytes.Compare(buf[:n], TestBlock2) != 0 { + t.Errorf("buf should be %s, is %s", string(TestBlock2), string(buf[:n])) } sem <- 1 - }(sem) + }() - go func(sem chan int) { - buf, err := v.Get(TestHash3) + go func() { + buf := make([]byte, BlockSize) + n, err := v.Get(TestHash3, buf) if err != nil { t.Errorf("err3: %v", err) } - bufs.Put(buf) - if bytes.Compare(buf, TestBlock3) != 0 { - t.Errorf("buf should be %s, is %s", string(TestBlock3), string(buf)) + if bytes.Compare(buf[:n], TestBlock3) != 0 { + t.Errorf("buf should be %s, is %s", string(TestBlock3), string(buf[:n])) } sem <- 1 - }(sem) + }() // Wait for all goroutines to finish - for done := 0; done < 3; { - done += <-sem + for done := 0; done < 3; done++ { + <-sem } } @@ -640,36 +656,34 @@ func testPutConcurrent(t TB, factory TestableVolumeFactory) { }(sem) // Wait for all goroutines to finish - for done := 0; done < 3; { - done += <-sem + for done := 0; done < 3; done++ { + <-sem } // Double check that we actually wrote the blocks we expected to write. - buf, err := v.Get(TestHash) + buf := make([]byte, BlockSize) + n, err := v.Get(TestHash, buf) if err != nil { t.Errorf("Get #1: %v", err) } - bufs.Put(buf) - if bytes.Compare(buf, TestBlock) != 0 { - t.Errorf("Get #1: expected %s, got %s", string(TestBlock), string(buf)) + if bytes.Compare(buf[:n], TestBlock) != 0 { + t.Errorf("Get #1: expected %s, got %s", string(TestBlock), string(buf[:n])) } - buf, err = v.Get(TestHash2) + n, err = v.Get(TestHash2, buf) if err != nil { t.Errorf("Get #2: %v", err) } - bufs.Put(buf) - if bytes.Compare(buf, TestBlock2) != 0 { - t.Errorf("Get #2: expected %s, got %s", string(TestBlock2), string(buf)) + if bytes.Compare(buf[:n], TestBlock2) != 0 { + t.Errorf("Get #2: expected %s, got %s", string(TestBlock2), string(buf[:n])) } - buf, err = v.Get(TestHash3) + n, err = v.Get(TestHash3, buf) if err != nil { t.Errorf("Get #3: %v", err) } - bufs.Put(buf) - if bytes.Compare(buf, TestBlock3) != 0 { - t.Errorf("Get #3: expected %s, got %s", string(TestBlock3), string(buf)) + if bytes.Compare(buf[:n], TestBlock3) != 0 { + t.Errorf("Get #3: expected %s, got %s", string(TestBlock3), string(buf[:n])) } } @@ -690,14 +704,13 @@ func testPutFullBlock(t TB, factory TestableVolumeFactory) { if err != nil { t.Fatal(err) } - rdata, err := v.Get(hash) + buf := make([]byte, BlockSize) + n, err := v.Get(hash, buf) if err != nil { t.Error(err) - } else { - defer bufs.Put(rdata) } - if bytes.Compare(rdata, wdata) != 0 { - t.Error("rdata != wdata") + if bytes.Compare(buf[:n], wdata) != 0 { + t.Error("buf %+q != wdata %+q", buf[:n], wdata) } } @@ -718,27 +731,27 @@ func testTrashUntrash(t TB, factory TestableVolumeFactory) { v.PutRaw(TestHash, TestBlock) v.TouchWithDate(TestHash, time.Now().Add(-2*blobSignatureTTL)) - buf, err := v.Get(TestHash) + buf := make([]byte, BlockSize) + n, err := v.Get(TestHash, buf) if err != nil { t.Fatal(err) } - if bytes.Compare(buf, TestBlock) != 0 { - t.Errorf("Got data %+q, expected %+q", buf, TestBlock) + if bytes.Compare(buf[:n], TestBlock) != 0 { + t.Errorf("Got data %+q, expected %+q", buf[:n], TestBlock) } - bufs.Put(buf) // Trash err = v.Trash(TestHash) if v.Writable() == false { if err != MethodDisabledError { - t.Error(err) + t.Fatal(err) } } else if err != nil { if err != ErrNotImplemented { - t.Error(err) + t.Fatal(err) } } else { - _, err = v.Get(TestHash) + _, err = v.Get(TestHash, buf) if err == nil || !os.IsNotExist(err) { t.Errorf("os.IsNotExist(%v) should have been true", err) } @@ -751,128 +764,151 @@ func testTrashUntrash(t TB, factory TestableVolumeFactory) { } // Get the block - after trash and untrash sequence - buf, err = v.Get(TestHash) + n, err = v.Get(TestHash, buf) if err != nil { t.Fatal(err) } - if bytes.Compare(buf, TestBlock) != 0 { - t.Errorf("Got data %+q, expected %+q", buf, TestBlock) + if bytes.Compare(buf[:n], TestBlock) != 0 { + t.Errorf("Got data %+q, expected %+q", buf[:n], TestBlock) } - bufs.Put(buf) } func testTrashEmptyTrashUntrash(t TB, factory TestableVolumeFactory) { v := factory(t) defer v.Teardown() - defer func() { - trashLifetime = 0 * time.Second - }() + defer func(orig time.Duration) { + trashLifetime = orig + }(trashLifetime) - // First set of tests + checkGet := func() error { + buf := make([]byte, BlockSize) + n, err := v.Get(TestHash, buf) + if err != nil { + return err + } + if bytes.Compare(buf[:n], TestBlock) != 0 { + t.Fatalf("Got data %+q, expected %+q", buf[:n], TestBlock) + } + + _, err = v.Mtime(TestHash) + if err != nil { + return err + } + + err = v.Compare(TestHash, TestBlock) + if err != nil { + return err + } + + indexBuf := new(bytes.Buffer) + v.IndexTo("", indexBuf) + if !strings.Contains(string(indexBuf.Bytes()), TestHash) { + return os.ErrNotExist + } + + return nil + } + + // First set: EmptyTrash before reaching the trash deadline. - // With trashLifetime = 1h, test trash/untrash cycle. trashLifetime = 1 * time.Hour - // put block and backdate it v.PutRaw(TestHash, TestBlock) v.TouchWithDate(TestHash, time.Now().Add(-2*blobSignatureTTL)) - buf, err := v.Get(TestHash) + err := checkGet() if err != nil { t.Fatal(err) } - if bytes.Compare(buf, TestBlock) != 0 { - t.Fatalf("Got data %+q, expected %+q", buf, TestBlock) - } - bufs.Put(buf) - // Trash it + // Trash the block err = v.Trash(TestHash) if err == MethodDisabledError || err == ErrNotImplemented { + // Skip the trash tests for read-only volumes, and + // volume types that don't support trashLifetime>0. return } - buf, err = v.Get(TestHash) + err = checkGet() + if err == nil || !os.IsNotExist(err) { + t.Fatalf("os.IsNotExist(%v) should have been true", err) + } + + err = v.Touch(TestHash) if err == nil || !os.IsNotExist(err) { t.Fatalf("os.IsNotExist(%v) should have been true", err) } - // Empty trash; the block is still within trashLifetime and hence is not emptied v.EmptyTrash() - // Untrash will hence rescue it + // Even after emptying the trash, we can untrash our block + // because the deadline hasn't been reached. err = v.Untrash(TestHash) if err != nil { t.Fatal(err) } - // Get block will find it - buf, err = v.Get(TestHash) + err = checkGet() if err != nil { t.Fatal(err) } - if bytes.Compare(buf, TestBlock) != 0 { - t.Fatalf("Got data %+q, expected %+q", buf, TestBlock) + + err = v.Touch(TestHash) + if err != nil { + t.Fatal(err) } - bufs.Put(buf) - // Untrash again; should fail + // Because we Touch'ed, need to backdate again for next set of tests + v.TouchWithDate(TestHash, time.Now().Add(-2*blobSignatureTTL)) + + // Untrash should fail if the only block in the trash has + // already been untrashed. err = v.Untrash(TestHash) if err == nil || !os.IsNotExist(err) { t.Fatalf("os.IsNotExist(%v) should have been true", err) } - buf, err = v.Get(TestHash) + // The failed Untrash should not interfere with our + // already-untrashed copy. + err = checkGet() if err != nil { t.Fatal(err) } - if bytes.Compare(buf, TestBlock) != 0 { - t.Fatalf("Got data %+q, expected %+q", buf, TestBlock) - } - bufs.Put(buf) - // Second set of tests + // Second set: EmptyTrash after the trash deadline has passed. - // With trashLifetime = 1ns, test trash/untrash cycle. trashLifetime = 1 * time.Nanosecond - // Trash it err = v.Trash(TestHash) if err != nil { t.Fatal(err) } - buf, err = v.Get(TestHash) + err = checkGet() if err == nil || !os.IsNotExist(err) { t.Fatalf("os.IsNotExist(%v) should have been true", err) } - // Untrash + // Even though 1ns has passed, we can untrash because we + // haven't called EmptyTrash yet. err = v.Untrash(TestHash) if err != nil { t.Fatal(err) } - - // Get block will find it - buf, err = v.Get(TestHash) + err = checkGet() if err != nil { t.Fatal(err) } - if bytes.Compare(buf, TestBlock) != 0 { - t.Fatalf("Got data %+q, expected %+q", buf, TestBlock) - } - bufs.Put(buf) - // Trash it again + // Trash it again, and this time call EmptyTrash so it really + // goes away. + // (In Azure volumes, un/trash changes Mtime, so first backdate again) + v.TouchWithDate(TestHash, time.Now().Add(-2*blobSignatureTTL)) err = v.Trash(TestHash) - if err == MethodDisabledError || err == ErrNotImplemented { - return - } - buf, err = v.Get(TestHash) + err = checkGet() if err == nil || !os.IsNotExist(err) { - t.Errorf("os.IsNotExist(%v) should have been true", err) + t.Fatalf("os.IsNotExist(%v) should have been true", err) } - - // Empty trash will empty it + // EmptryTrash v.EmptyTrash() // Untrash won't find it @@ -882,142 +918,70 @@ func testTrashEmptyTrashUntrash(t TB, factory TestableVolumeFactory) { } // Get block won't find it - buf, err = v.Get(TestHash) + err = checkGet() if err == nil || !os.IsNotExist(err) { t.Fatalf("os.IsNotExist(%v) should have been true", err) } - // Third set of tests + // Third set: If the same data block gets written again after + // being trashed, and then the trash gets emptied, the newer + // un-trashed copy doesn't get deleted along with it. - // Still with trashLifetime = 1ns: put, trash, put one more, trash etc - // put block and backdate it v.PutRaw(TestHash, TestBlock) v.TouchWithDate(TestHash, time.Now().Add(-2*blobSignatureTTL)) - // Trash + trashLifetime = time.Nanosecond err = v.Trash(TestHash) - if err == MethodDisabledError || err == ErrNotImplemented { - return + if err != nil { + t.Fatal(err) } - buf, err = v.Get(TestHash) + err = checkGet() if err == nil || !os.IsNotExist(err) { t.Fatalf("os.IsNotExist(%v) should have been true", err) } - // put again - err = v.Put(TestHash, TestBlock) - if err != nil { - t.Fatal(err) - } + v.PutRaw(TestHash, TestBlock) v.TouchWithDate(TestHash, time.Now().Add(-2*blobSignatureTTL)) - // Empty trash will empty the trashed block but the second one is untouched + // EmptyTrash should not delete the untrashed copy. v.EmptyTrash() - - // Get block should work because of the second block - buf, err = v.Get(TestHash) + err = checkGet() if err != nil { t.Fatal(err) } - if bytes.Compare(buf, TestBlock) != 0 { - t.Fatalf("Got data %+q, expected %+q", buf, TestBlock) - } - bufs.Put(buf) - // set trashLifetime to one hour - trashLifetime = 1 * time.Hour + // Fourth set: If the same data block gets trashed twice with + // different deadlines A and C, and then the trash is emptied + // at intermediate time B (A < B < C), it is still possible to + // untrash the block whose deadline is "C". - // trash block - err = v.Trash(TestHash) - if err != nil { - t.Fatal(err) - } - - // Empty trash won't empty this second block which is still within trashLifetime - v.EmptyTrash() + v.PutRaw(TestHash, TestBlock) + v.TouchWithDate(TestHash, time.Now().Add(-2*blobSignatureTTL)) - // Untrash; the second block which is still within trashLifetime will be rescued - err = v.Untrash(TestHash) + trashLifetime = time.Nanosecond + err = v.Trash(TestHash) if err != nil { t.Fatal(err) } - // Get block should work because of the second block - buf, err = v.Get(TestHash) - if err != nil { - t.Fatal(err) - } - if bytes.Compare(buf, TestBlock) != 0 { - t.Fatalf("Got data %+q, expected %+q", buf, TestBlock) - } - bufs.Put(buf) -} - -// With trashLifetime = 1ns, perform: -// Run emptyTrash goroutine -// Trash an old block - which either raises ErrNotImplemented or succeeds -// Untrash - after emptyTrash goroutine ticks, and hence does not actually untrash -// Get - which must fail to find the block -func testTrashUntrashWithEmptyTrashGoroutine(t TB, factory TestableVolumeFactory) { - v := factory(t) - defer v.Teardown() - - doneEmptyingTrash := make(chan bool) - defer func() { - trashLifetime = 0 * time.Second - doneEmptyingTrash <- true - }() - - volumes = append(volumes, v) - - trashLifetime = 1 * time.Nanosecond - trashCheckInterval = 1 * time.Nanosecond - - go emptyTrash(doneEmptyingTrash, trashCheckInterval) - - // Trash old block and untrash a little after first trashCheckInterval v.PutRaw(TestHash, TestBlock) v.TouchWithDate(TestHash, time.Now().Add(-2*blobSignatureTTL)) - buf, err := v.Get(TestHash) + trashLifetime = time.Hour + err = v.Trash(TestHash) if err != nil { t.Fatal(err) } - if bytes.Compare(buf, TestBlock) != 0 { - t.Fatalf("Got data %+q, expected %+q", buf, TestBlock) - } - bufs.Put(buf) - // Trash - err = v.Trash(TestHash) - if err == MethodDisabledError || err == ErrNotImplemented { - return - } - - _, err = v.Get(TestHash) - if err == nil || !os.IsNotExist(err) { - t.Fatalf("os.IsNotExist(%v) should have been true", err) - } - - time.Sleep(2 * time.Nanosecond) - - // Untrash + // EmptyTrash should not prevent us from recovering the + // time.Hour ("C") trash + v.EmptyTrash() err = v.Untrash(TestHash) if err != nil { t.Fatal(err) } - - // Get is expected to fail due to EmptyTrash before Untrash - // It is still found on readonly volumes - buf, err = v.Get(TestHash) + err = checkGet() if err != nil { - if !os.IsNotExist(err) { - t.Errorf("os.IsNotExist(%v) should have been true", err) - } - } else { - if bytes.Compare(buf, TestBlock) != 0 { - t.Errorf("Got data %+q, expected %+q", buf, TestBlock) - } - bufs.Put(buf) + t.Fatal(err) } }