X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/a95f899d7ac84f29b3d019aa410d265bb40833e5..dc7d01f4d4031962ffd5734ca0c64146a7217e4a:/services/keepstore/volume_generic_test.go diff --git a/services/keepstore/volume_generic_test.go b/services/keepstore/volume_generic_test.go index f8fe0d0ebc..7e72a8f246 100644 --- a/services/keepstore/volume_generic_test.go +++ b/services/keepstore/volume_generic_test.go @@ -2,14 +2,17 @@ package main import ( "bytes" + "context" "crypto/md5" "fmt" "os" "regexp" "sort" + "strconv" "strings" "time" + "git.curoverse.com/arvados.git/sdk/go/arvados" "git.curoverse.com/arvados.git/sdk/go/arvadostest" ) @@ -90,7 +93,7 @@ func testGet(t TB, factory TestableVolumeFactory) { v.PutRaw(TestHash, TestBlock) buf := make([]byte, BlockSize) - n, err := v.Get(TestHash, buf) + n, err := v.Get(context.Background(), TestHash, buf) if err != nil { t.Fatal(err) } @@ -107,7 +110,7 @@ func testGetNoSuchBlock(t TB, factory TestableVolumeFactory) { defer v.Teardown() buf := make([]byte, BlockSize) - if _, err := v.Get(TestHash2, buf); err == nil { + if _, err := v.Get(context.Background(), TestHash2, buf); err == nil { t.Errorf("Expected error while getting non-existing block %v", TestHash2) } } @@ -119,7 +122,7 @@ func testCompareNonexistent(t TB, factory TestableVolumeFactory) { v := factory(t) defer v.Teardown() - err := v.Compare(TestHash, TestBlock) + err := v.Compare(context.Background(), TestHash, TestBlock) if err != os.ErrNotExist { t.Errorf("Got err %T %q, expected os.ErrNotExist", err, err) } @@ -134,7 +137,7 @@ func testCompareSameContent(t TB, factory TestableVolumeFactory, testHash string v.PutRaw(testHash, testData) // Compare the block locator with same content - err := v.Compare(testHash, testData) + err := v.Compare(context.Background(), testHash, testData) if err != nil { t.Errorf("Got err %q, expected nil", err) } @@ -152,7 +155,7 @@ func testCompareWithCollision(t TB, factory TestableVolumeFactory, testHash stri v.PutRaw(testHash, testDataA) // Compare the block locator with different content; collision - err := v.Compare(TestHash, testDataB) + err := v.Compare(context.Background(), TestHash, testDataB) if err == nil { t.Errorf("Got err nil, expected error due to collision") } @@ -168,7 +171,7 @@ func testCompareWithCorruptStoredData(t TB, factory TestableVolumeFactory, testH v.PutRaw(TestHash, testDataB) - err := v.Compare(testHash, testDataA) + err := v.Compare(context.Background(), testHash, testDataA) if err == nil || err == CollisionError { t.Errorf("Got err %+v, expected non-collision error", err) } @@ -184,12 +187,12 @@ func testPutBlockWithSameContent(t TB, factory TestableVolumeFactory, testHash s return } - err := v.Put(testHash, testData) + err := v.Put(context.Background(), testHash, testData) if err != nil { t.Errorf("Got err putting block %q: %q, expected nil", TestBlock, err) } - err = v.Put(testHash, testData) + err = v.Put(context.Background(), testHash, testData) if err != nil { t.Errorf("Got err putting block second time %q: %q, expected nil", TestBlock, err) } @@ -207,9 +210,9 @@ func testPutBlockWithDifferentContent(t TB, factory TestableVolumeFactory, testH v.PutRaw(testHash, testDataA) - putErr := v.Put(testHash, testDataB) + putErr := v.Put(context.Background(), testHash, testDataB) buf := make([]byte, BlockSize) - n, getErr := v.Get(testHash, buf) + n, getErr := v.Get(context.Background(), testHash, buf) if putErr == nil { // Put must not return a nil error unless it has // overwritten the existing data. @@ -236,23 +239,23 @@ func testPutMultipleBlocks(t TB, factory TestableVolumeFactory) { return } - err := v.Put(TestHash, TestBlock) + err := v.Put(context.Background(), TestHash, TestBlock) if err != nil { t.Errorf("Got err putting block %q: %q, expected nil", TestBlock, err) } - err = v.Put(TestHash2, TestBlock2) + err = v.Put(context.Background(), TestHash2, TestBlock2) if err != nil { t.Errorf("Got err putting block %q: %q, expected nil", TestBlock2, err) } - err = v.Put(TestHash3, TestBlock3) + err = v.Put(context.Background(), TestHash3, TestBlock3) if err != nil { t.Errorf("Got err putting block %q: %q, expected nil", TestBlock3, err) } data := make([]byte, BlockSize) - n, err := v.Get(TestHash, data) + n, err := v.Get(context.Background(), TestHash, data) if err != nil { t.Error(err) } else { @@ -261,7 +264,7 @@ func testPutMultipleBlocks(t TB, factory TestableVolumeFactory) { } } - n, err = v.Get(TestHash2, data) + n, err = v.Get(context.Background(), TestHash2, data) if err != nil { t.Error(err) } else { @@ -270,7 +273,7 @@ func testPutMultipleBlocks(t TB, factory TestableVolumeFactory) { } } - n, err = v.Get(TestHash3, data) + n, err = v.Get(context.Background(), TestHash3, data) if err != nil { t.Error(err) } else { @@ -292,7 +295,7 @@ func testPutAndTouch(t TB, factory TestableVolumeFactory) { return } - if err := v.Put(TestHash, TestBlock); err != nil { + if err := v.Put(context.Background(), TestHash, TestBlock); err != nil { t.Error(err) } @@ -312,7 +315,7 @@ func testPutAndTouch(t TB, factory TestableVolumeFactory) { } // Write the same block again. - if err := v.Put(TestHash, TestBlock); err != nil { + if err := v.Put(context.Background(), TestHash, TestBlock); err != nil { t.Error(err) } @@ -355,10 +358,22 @@ func testIndexTo(t TB, factory TestableVolumeFactory) { v := factory(t) defer v.Teardown() + // minMtime and maxMtime are the minimum and maximum + // acceptable values the index can report for our test + // blocks. 1-second precision is acceptable. + minMtime := time.Now().UTC().UnixNano() + minMtime -= minMtime % 1e9 + v.PutRaw(TestHash, TestBlock) v.PutRaw(TestHash2, TestBlock2) v.PutRaw(TestHash3, TestBlock3) + maxMtime := time.Now().UTC().UnixNano() + if maxMtime%1e9 > 0 { + maxMtime -= maxMtime % 1e9 + maxMtime += 1e9 + } + // Blocks whose names aren't Keep hashes should be omitted from // index v.PutRaw("fffffffffnotreallyahashfffffffff", nil) @@ -371,15 +386,21 @@ func testIndexTo(t TB, factory TestableVolumeFactory) { indexRows := strings.Split(string(buf.Bytes()), "\n") sort.Strings(indexRows) sortedIndex := strings.Join(indexRows, "\n") - m, err := regexp.MatchString( - `^\n`+TestHash+`\+\d+ \d+\n`+ - TestHash3+`\+\d+ \d+\n`+ - TestHash2+`\+\d+ \d+$`, - sortedIndex) - if err != nil { - t.Error(err) - } else if !m { + m := regexp.MustCompile( + `^\n` + TestHash + `\+\d+ (\d+)\n` + + TestHash3 + `\+\d+ \d+\n` + + TestHash2 + `\+\d+ \d+$`, + ).FindStringSubmatch(sortedIndex) + if m == nil { t.Errorf("Got index %q for empty prefix", sortedIndex) + } else { + mtime, err := strconv.ParseInt(m[1], 10, 64) + if err != nil { + t.Error(err) + } else if mtime < minMtime || mtime > maxMtime { + t.Errorf("got %d for TestHash timestamp, expected %d <= t <= %d", + mtime, minMtime, maxMtime) + } } for _, prefix := range []string{"f", "f15", "f15ac"} { @@ -396,7 +417,7 @@ func testIndexTo(t TB, factory TestableVolumeFactory) { for _, prefix := range []string{"zero", "zip", "zilch"} { buf = new(bytes.Buffer) - v.IndexTo(prefix, buf) + err := v.IndexTo(prefix, buf) if err != nil { t.Errorf("Got error on IndexTo with no such prefix %v", err.Error()) } else if buf.Len() != 0 { @@ -411,19 +432,19 @@ func testIndexTo(t TB, factory TestableVolumeFactory) { func testDeleteNewBlock(t TB, factory TestableVolumeFactory) { v := factory(t) defer v.Teardown() - blobSignatureTTL = 300 * time.Second + theConfig.BlobSignatureTTL.Set("5m") if v.Writable() == false { return } - v.Put(TestHash, TestBlock) + v.Put(context.Background(), TestHash, TestBlock) if err := v.Trash(TestHash); err != nil { t.Error(err) } data := make([]byte, BlockSize) - n, err := v.Get(TestHash, data) + n, err := v.Get(context.Background(), TestHash, data) if err != nil { t.Error(err) } else if bytes.Compare(data[:n], TestBlock) != 0 { @@ -432,25 +453,25 @@ func testDeleteNewBlock(t TB, factory TestableVolumeFactory) { } // Calling Delete() for a block with a timestamp older than -// blobSignatureTTL seconds in the past should delete the data. +// BlobSignatureTTL seconds in the past should delete the data. // Test is intended for only writable volumes func testDeleteOldBlock(t TB, factory TestableVolumeFactory) { v := factory(t) defer v.Teardown() - blobSignatureTTL = 300 * time.Second + theConfig.BlobSignatureTTL.Set("5m") if v.Writable() == false { return } - v.Put(TestHash, TestBlock) - v.TouchWithDate(TestHash, time.Now().Add(-2*blobSignatureTTL)) + v.Put(context.Background(), TestHash, TestBlock) + v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration())) if err := v.Trash(TestHash); err != nil { t.Error(err) } data := make([]byte, BlockSize) - if _, err := v.Get(TestHash, data); err == nil || !os.IsNotExist(err) { + if _, err := v.Get(context.Background(), TestHash, data); err == nil || !os.IsNotExist(err) { t.Errorf("os.IsNotExist(%v) should have been true", err) } @@ -459,7 +480,7 @@ func testDeleteOldBlock(t TB, factory TestableVolumeFactory) { t.Fatalf("os.IsNotExist(%v) should have been true", err) } - err = v.Compare(TestHash, TestBlock) + err = v.Compare(context.Background(), TestHash, TestBlock) if err == nil || !os.IsNotExist(err) { t.Fatalf("os.IsNotExist(%v) should have been true", err) } @@ -533,17 +554,17 @@ func testUpdateReadOnly(t TB, factory TestableVolumeFactory) { buf := make([]byte, BlockSize) // Get from read-only volume should succeed - _, err := v.Get(TestHash, buf) + _, err := v.Get(context.Background(), TestHash, buf) if err != nil { t.Errorf("got err %v, expected nil", err) } // Put a new block to read-only volume should result in error - err = v.Put(TestHash2, TestBlock2) + err = v.Put(context.Background(), TestHash2, TestBlock2) if err == nil { t.Errorf("Expected error when putting block in a read-only volume") } - _, err = v.Get(TestHash2, buf) + _, err = v.Get(context.Background(), TestHash2, buf) if err == nil { t.Errorf("Expected error when getting block whose put in read-only volume failed") } @@ -561,7 +582,7 @@ func testUpdateReadOnly(t TB, factory TestableVolumeFactory) { } // Overwriting an existing block in read-only volume should result in error - err = v.Put(TestHash, TestBlock) + err = v.Put(context.Background(), TestHash, TestBlock) if err == nil { t.Errorf("Expected error when putting block in a read-only volume") } @@ -580,7 +601,7 @@ func testGetConcurrent(t TB, factory TestableVolumeFactory) { sem := make(chan int) go func() { buf := make([]byte, BlockSize) - n, err := v.Get(TestHash, buf) + n, err := v.Get(context.Background(), TestHash, buf) if err != nil { t.Errorf("err1: %v", err) } @@ -592,7 +613,7 @@ func testGetConcurrent(t TB, factory TestableVolumeFactory) { go func() { buf := make([]byte, BlockSize) - n, err := v.Get(TestHash2, buf) + n, err := v.Get(context.Background(), TestHash2, buf) if err != nil { t.Errorf("err2: %v", err) } @@ -604,7 +625,7 @@ func testGetConcurrent(t TB, factory TestableVolumeFactory) { go func() { buf := make([]byte, BlockSize) - n, err := v.Get(TestHash3, buf) + n, err := v.Get(context.Background(), TestHash3, buf) if err != nil { t.Errorf("err3: %v", err) } @@ -632,7 +653,7 @@ func testPutConcurrent(t TB, factory TestableVolumeFactory) { sem := make(chan int) go func(sem chan int) { - err := v.Put(TestHash, TestBlock) + err := v.Put(context.Background(), TestHash, TestBlock) if err != nil { t.Errorf("err1: %v", err) } @@ -640,7 +661,7 @@ func testPutConcurrent(t TB, factory TestableVolumeFactory) { }(sem) go func(sem chan int) { - err := v.Put(TestHash2, TestBlock2) + err := v.Put(context.Background(), TestHash2, TestBlock2) if err != nil { t.Errorf("err2: %v", err) } @@ -648,7 +669,7 @@ func testPutConcurrent(t TB, factory TestableVolumeFactory) { }(sem) go func(sem chan int) { - err := v.Put(TestHash3, TestBlock3) + err := v.Put(context.Background(), TestHash3, TestBlock3) if err != nil { t.Errorf("err3: %v", err) } @@ -662,7 +683,7 @@ func testPutConcurrent(t TB, factory TestableVolumeFactory) { // Double check that we actually wrote the blocks we expected to write. buf := make([]byte, BlockSize) - n, err := v.Get(TestHash, buf) + n, err := v.Get(context.Background(), TestHash, buf) if err != nil { t.Errorf("Get #1: %v", err) } @@ -670,7 +691,7 @@ func testPutConcurrent(t TB, factory TestableVolumeFactory) { t.Errorf("Get #1: expected %s, got %s", string(TestBlock), string(buf[:n])) } - n, err = v.Get(TestHash2, buf) + n, err = v.Get(context.Background(), TestHash2, buf) if err != nil { t.Errorf("Get #2: %v", err) } @@ -678,7 +699,7 @@ func testPutConcurrent(t TB, factory TestableVolumeFactory) { t.Errorf("Get #2: expected %s, got %s", string(TestBlock2), string(buf[:n])) } - n, err = v.Get(TestHash3, buf) + n, err = v.Get(context.Background(), TestHash3, buf) if err != nil { t.Errorf("Get #3: %v", err) } @@ -700,12 +721,12 @@ func testPutFullBlock(t TB, factory TestableVolumeFactory) { wdata[0] = 'a' wdata[BlockSize-1] = 'z' hash := fmt.Sprintf("%x", md5.Sum(wdata)) - err := v.Put(hash, wdata) + err := v.Put(context.Background(), hash, wdata) if err != nil { t.Fatal(err) } buf := make([]byte, BlockSize) - n, err := v.Get(hash, buf) + n, err := v.Get(context.Background(), hash, buf) if err != nil { t.Error(err) } @@ -714,7 +735,7 @@ func testPutFullBlock(t TB, factory TestableVolumeFactory) { } } -// With trashLifetime != 0, perform: +// With TrashLifetime != 0, perform: // Trash an old block - which either raises ErrNotImplemented or succeeds // Untrash - which either raises ErrNotImplemented or succeeds // Get - which must succeed @@ -722,17 +743,17 @@ func testTrashUntrash(t TB, factory TestableVolumeFactory) { v := factory(t) defer v.Teardown() defer func() { - trashLifetime = 0 * time.Second + theConfig.TrashLifetime = 0 }() - trashLifetime = 3600 * time.Second + theConfig.TrashLifetime.Set("1h") // put block and backdate it v.PutRaw(TestHash, TestBlock) - v.TouchWithDate(TestHash, time.Now().Add(-2*blobSignatureTTL)) + v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration())) buf := make([]byte, BlockSize) - n, err := v.Get(TestHash, buf) + n, err := v.Get(context.Background(), TestHash, buf) if err != nil { t.Fatal(err) } @@ -751,7 +772,7 @@ func testTrashUntrash(t TB, factory TestableVolumeFactory) { t.Fatal(err) } } else { - _, err = v.Get(TestHash, buf) + _, err = v.Get(context.Background(), TestHash, buf) if err == nil || !os.IsNotExist(err) { t.Errorf("os.IsNotExist(%v) should have been true", err) } @@ -764,7 +785,7 @@ func testTrashUntrash(t TB, factory TestableVolumeFactory) { } // Get the block - after trash and untrash sequence - n, err = v.Get(TestHash, buf) + n, err = v.Get(context.Background(), TestHash, buf) if err != nil { t.Fatal(err) } @@ -776,13 +797,13 @@ func testTrashUntrash(t TB, factory TestableVolumeFactory) { func testTrashEmptyTrashUntrash(t TB, factory TestableVolumeFactory) { v := factory(t) defer v.Teardown() - defer func(orig time.Duration) { - trashLifetime = orig - }(trashLifetime) + defer func(orig arvados.Duration) { + theConfig.TrashLifetime = orig + }(theConfig.TrashLifetime) checkGet := func() error { buf := make([]byte, BlockSize) - n, err := v.Get(TestHash, buf) + n, err := v.Get(context.Background(), TestHash, buf) if err != nil { return err } @@ -795,7 +816,7 @@ func testTrashEmptyTrashUntrash(t TB, factory TestableVolumeFactory) { return err } - err = v.Compare(TestHash, TestBlock) + err = v.Compare(context.Background(), TestHash, TestBlock) if err != nil { return err } @@ -811,10 +832,10 @@ func testTrashEmptyTrashUntrash(t TB, factory TestableVolumeFactory) { // First set: EmptyTrash before reaching the trash deadline. - trashLifetime = 1 * time.Hour + theConfig.TrashLifetime.Set("1h") v.PutRaw(TestHash, TestBlock) - v.TouchWithDate(TestHash, time.Now().Add(-2*blobSignatureTTL)) + v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration())) err := checkGet() if err != nil { @@ -825,7 +846,7 @@ func testTrashEmptyTrashUntrash(t TB, factory TestableVolumeFactory) { err = v.Trash(TestHash) if err == MethodDisabledError || err == ErrNotImplemented { // Skip the trash tests for read-only volumes, and - // volume types that don't support trashLifetime>0. + // volume types that don't support TrashLifetime>0. return } @@ -859,25 +880,29 @@ func testTrashEmptyTrashUntrash(t TB, factory TestableVolumeFactory) { } // Because we Touch'ed, need to backdate again for next set of tests - v.TouchWithDate(TestHash, time.Now().Add(-2*blobSignatureTTL)) + v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration())) - // Untrash should fail if the only block in the trash has - // already been untrashed. + // If the only block in the trash has already been untrashed, + // most volumes will fail a subsequent Untrash with a 404, but + // it's also acceptable for Untrash to succeed. err = v.Untrash(TestHash) - if err == nil || !os.IsNotExist(err) { - t.Fatalf("os.IsNotExist(%v) should have been true", err) + if err != nil && !os.IsNotExist(err) { + t.Fatalf("Expected success or os.IsNotExist(), but got: %v", err) } - // The failed Untrash should not interfere with our + // The additional Untrash should not interfere with our // already-untrashed copy. err = checkGet() if err != nil { t.Fatal(err) } + // Untrash might have updated the timestamp, so backdate again + v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration())) + // Second set: EmptyTrash after the trash deadline has passed. - trashLifetime = 1 * time.Nanosecond + theConfig.TrashLifetime.Set("1ns") err = v.Trash(TestHash) if err != nil { @@ -902,13 +927,12 @@ func testTrashEmptyTrashUntrash(t TB, factory TestableVolumeFactory) { // Trash it again, and this time call EmptyTrash so it really // goes away. // (In Azure volumes, un/trash changes Mtime, so first backdate again) - v.TouchWithDate(TestHash, time.Now().Add(-2*blobSignatureTTL)) + v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration())) err = v.Trash(TestHash) err = checkGet() if err == nil || !os.IsNotExist(err) { t.Fatalf("os.IsNotExist(%v) should have been true", err) } - // EmptryTrash v.EmptyTrash() // Untrash won't find it @@ -928,9 +952,9 @@ func testTrashEmptyTrashUntrash(t TB, factory TestableVolumeFactory) { // un-trashed copy doesn't get deleted along with it. v.PutRaw(TestHash, TestBlock) - v.TouchWithDate(TestHash, time.Now().Add(-2*blobSignatureTTL)) + v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration())) - trashLifetime = time.Nanosecond + theConfig.TrashLifetime.Set("1ns") err = v.Trash(TestHash) if err != nil { t.Fatal(err) @@ -941,7 +965,7 @@ func testTrashEmptyTrashUntrash(t TB, factory TestableVolumeFactory) { } v.PutRaw(TestHash, TestBlock) - v.TouchWithDate(TestHash, time.Now().Add(-2*blobSignatureTTL)) + v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration())) // EmptyTrash should not delete the untrashed copy. v.EmptyTrash() @@ -956,18 +980,18 @@ func testTrashEmptyTrashUntrash(t TB, factory TestableVolumeFactory) { // untrash the block whose deadline is "C". v.PutRaw(TestHash, TestBlock) - v.TouchWithDate(TestHash, time.Now().Add(-2*blobSignatureTTL)) + v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration())) - trashLifetime = time.Nanosecond + theConfig.TrashLifetime.Set("1ns") err = v.Trash(TestHash) if err != nil { t.Fatal(err) } v.PutRaw(TestHash, TestBlock) - v.TouchWithDate(TestHash, time.Now().Add(-2*blobSignatureTTL)) + v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration())) - trashLifetime = time.Hour + theConfig.TrashLifetime.Set("1h") err = v.Trash(TestHash) if err != nil { t.Fatal(err)