X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/b10dd532008ee5ea7b994b07bb6f6e458786c0eb..cffa542e4cb069b40764c889613c249e88af62d0:/services/datamanager/datamanager_test.go diff --git a/services/datamanager/datamanager_test.go b/services/datamanager/datamanager_test.go index 983e25cf01..a99ec6b052 100644 --- a/services/datamanager/datamanager_test.go +++ b/services/datamanager/datamanager_test.go @@ -6,10 +6,13 @@ import ( "git.curoverse.com/arvados.git/sdk/go/arvadosclient" "git.curoverse.com/arvados.git/sdk/go/arvadostest" "git.curoverse.com/arvados.git/sdk/go/keepclient" + "git.curoverse.com/arvados.git/services/datamanager/collection" + "git.curoverse.com/arvados.git/services/datamanager/summary" "io/ioutil" "net/http" "os" "os/exec" + "path" "regexp" "strings" "testing" @@ -28,19 +31,22 @@ func SetupDataManagerTest(t *testing.T) { arvadostest.StartAPI() arvadostest.StartKeep(2, false) - arv = makeArvadosClient() + var err error + arv, err = arvadosclient.MakeArvadosClient() + if err != nil { + t.Fatalf("Error making arvados client: %s", err) + } arv.ApiToken = arvadostest.DataManagerToken // keep client keepClient = &keepclient.KeepClient{ Arvados: &arv, Want_replicas: 2, - Using_proxy: true, Client: &http.Client{}, } // discover keep services - if err := keepClient.DiscoverKeepServers(); err != nil { + if err = keepClient.DiscoverKeepServers(); err != nil { t.Fatalf("Error discovering keep services: %s", err) } keepServers = []string{} @@ -52,6 +58,8 @@ func SetupDataManagerTest(t *testing.T) { func TearDownDataManagerTest(t *testing.T) { arvadostest.StopKeep(2) arvadostest.StopAPI() + summary.WriteDataTo = "" + collection.HeapProfileFilename = "" } func putBlock(t *testing.T, data string) string { @@ -250,13 +258,10 @@ func valueInArray(value string, list []string) bool { return false } -/* -Test env uses two keep volumes. The volume names can be found by reading the files - ARVADOS_HOME/tmp/keep0.volume and ARVADOS_HOME/tmp/keep1.volume - -The keep volumes are of the dir structure: - volumeN/subdir/locator -*/ +// Test env uses two keep volumes. The volume names can be found by reading the files +// ARVADOS_HOME/tmp/keep0.volume and ARVADOS_HOME/tmp/keep1.volume +// +// The keep volumes are of the dir structure: volumeN/subdir/locator func backdateBlocks(t *testing.T, oldUnusedBlockLocators []string) { // First get rid of any size hints in the locators var trimmedBlockLocators []string @@ -338,11 +343,9 @@ func waitUntilQueuesFinishWork(t *testing.T) { } } -/* -Create some blocks and backdate some of them. -Also create some collections and delete some of them. -Verify block indexes. -*/ +// Create some blocks and backdate some of them. +// Also create some collections and delete some of them. +// Verify block indexes. func TestPutAndGetBlocks(t *testing.T) { defer TearDownDataManagerTest(t) SetupDataManagerTest(t) @@ -531,6 +534,78 @@ func TestRunDatamanagerAsNonAdminUser(t *testing.T) { } } +func TestPutAndGetBlocks_NoErrorDuringSingleRun(t *testing.T) { + testOldBlocksNotDeletedOnDataManagerError(t, "", "", false, false) +} + +func createBadPath(t *testing.T) (badpath string) { + tempdir, err := ioutil.TempDir("", "bad") + if err != nil { + t.Fatalf("Could not create temporary directory for bad path: %v", err) + } + badpath = path.Join(tempdir, "bad") + return +} + +func destroyBadPath(t *testing.T, badpath string) { + tempdir := path.Join(badpath, "..") + err := os.Remove(tempdir) + if err != nil { + t.Fatalf("Could not remove bad path temporary directory %v: %v", tempdir, err) + } +} + +func TestPutAndGetBlocks_ErrorDuringGetCollectionsBadWriteTo(t *testing.T) { + badpath := createBadPath(t) + defer destroyBadPath(t, badpath) + testOldBlocksNotDeletedOnDataManagerError(t, path.Join(badpath, "writetofile"), "", true, true) +} + +func TestPutAndGetBlocks_ErrorDuringGetCollectionsBadHeapProfileFilename(t *testing.T) { + badpath := createBadPath(t) + defer destroyBadPath(t, badpath) + testOldBlocksNotDeletedOnDataManagerError(t, "", path.Join(badpath, "heapprofilefile"), true, true) +} + +// Create some blocks and backdate some of them. +// Run datamanager while producing an error condition. +// Verify that the blocks are hence not deleted. +func testOldBlocksNotDeletedOnDataManagerError(t *testing.T, writeDataTo string, heapProfileFile string, expectError bool, expectOldBlocks bool) { + defer TearDownDataManagerTest(t) + SetupDataManagerTest(t) + + // Put some blocks and backdate them. + var oldUnusedBlockLocators []string + oldUnusedBlockData := "this block will have older mtime" + for i := 0; i < 5; i++ { + oldUnusedBlockLocators = append(oldUnusedBlockLocators, putBlock(t, fmt.Sprintf("%s%d", oldUnusedBlockData, i))) + } + backdateBlocks(t, oldUnusedBlockLocators) + + // Run data manager + summary.WriteDataTo = writeDataTo + collection.HeapProfileFilename = heapProfileFile + + err := singlerun(arv) + if !expectError { + if err != nil { + t.Fatalf("Got an error during datamanager singlerun: %v", err) + } + } else { + if err == nil { + t.Fatalf("Expected error during datamanager singlerun") + } + } + waitUntilQueuesFinishWork(t) + + // Get block indexes and verify that all backdated blocks are not/deleted as expected + if expectOldBlocks { + verifyBlocks(t, nil, oldUnusedBlockLocators, 2) + } else { + verifyBlocks(t, oldUnusedBlockLocators, nil, 2) + } +} + // Create a collection with multiple streams and blocks func createMultiStreamBlockCollection(t *testing.T, data string, numStreams, numBlocks int) (string, []string) { defer switchToken(arvadostest.AdminToken)() @@ -560,71 +635,99 @@ func createMultiStreamBlockCollection(t *testing.T, data string, numStreams, num } var locs []string - for k, _ := range locators { + for k := range locators { locs = append(locs, k) } return collection["uuid"].(string), locs } -/* - Create collection with multiple streams and blocks; backdate the blocks and but do not delete the collection. - Create another collection with multiple streams and blocks; backdate it's blocks and delete the collection. - After datamanager run: expect blocks from the first collection, but none from the second collection. -*/ -func TestPutAndGetCollectionsWithMultipleStreamsAndBlocks(t *testing.T) { +// Create collection with multiple streams and blocks; backdate the blocks and but do not delete the collection. +// Also, create stray block and backdate it. +// After datamanager run: expect blocks from the collection, but not the stray block. +func TestManifestWithMultipleStreamsAndBlocks(t *testing.T) { + testManifestWithMultipleStreamsAndBlocks(t, 100, 10, "", false) +} + +// Same test as TestManifestWithMultipleStreamsAndBlocks with an additional +// keepstore of a service type other than "disk". Only the "disk" type services +// will be indexed by datamanager and hence should work the same way. +func TestManifestWithMultipleStreamsAndBlocks_WithOneUnsupportedKeepServer(t *testing.T) { + testManifestWithMultipleStreamsAndBlocks(t, 2, 2, "testblobstore", false) +} + +// Test datamanager with dry-run. Expect no block to be deleted. +func TestManifestWithMultipleStreamsAndBlocks_DryRun(t *testing.T) { + testManifestWithMultipleStreamsAndBlocks(t, 2, 2, "", true) +} + +func testManifestWithMultipleStreamsAndBlocks(t *testing.T, numStreams, numBlocks int, createExtraKeepServerWithType string, isDryRun bool) { defer TearDownDataManagerTest(t) SetupDataManagerTest(t) // create collection whose blocks will be backdated - collectionWithOldBlocks, oldBlocks := createMultiStreamBlockCollection(t, "old block", 100, 10) + collectionWithOldBlocks, oldBlocks := createMultiStreamBlockCollection(t, "old block", numStreams, numBlocks) if collectionWithOldBlocks == "" { - t.Fatalf("Failed to create collection with 1000 blocks") + t.Fatalf("Failed to create collection with %d blocks", numStreams*numBlocks) } - if len(oldBlocks) != 1000 { + if len(oldBlocks) != numStreams*numBlocks { t.Fatalf("Not all blocks are created: expected %v, found %v", 1000, len(oldBlocks)) } - // create another collection, whose blocks will be backdated and the collection will be deleted - toBeDeletedCollection, toBeDeletedCollectionBlocks := createMultiStreamBlockCollection(t, "new block", 2, 5) - if toBeDeletedCollection == "" { - t.Fatalf("Failed to create collection with 10 blocks") - } - if len(toBeDeletedCollectionBlocks) != 10 { - t.Fatalf("Not all blocks are created: expected %v, found %v", 10, len(toBeDeletedCollectionBlocks)) - } - // create a stray block that will be backdated strayOldBlock := putBlock(t, "this stray block is old") - // create another block that will not be backdated - strayNewBlock := putBlock(t, "this stray block is new") - - expected := []string{} + expected := []string{strayOldBlock} expected = append(expected, oldBlocks...) - expected = append(expected, toBeDeletedCollectionBlocks...) - expected = append(expected, strayOldBlock) - expected = append(expected, strayNewBlock) verifyBlocks(t, nil, expected, 2) // Backdate old blocks; but the collection still references these blocks backdateBlocks(t, oldBlocks) - // Backdate first block from the newer blocks and delete the collection; the rest are still be reachable - backdateBlocks(t, toBeDeletedCollectionBlocks) - deleteCollection(t, toBeDeletedCollection) - // also backdate the stray old block backdateBlocks(t, []string{strayOldBlock}) + // If requested, create an extra keepserver with the given type + // This should be ignored during indexing and hence not change the datamanager outcome + var extraKeepServerUUID string + if createExtraKeepServerWithType != "" { + extraKeepServerUUID = addExtraKeepServer(t, createExtraKeepServerWithType) + defer deleteExtraKeepServer(extraKeepServerUUID) + } + // run datamanager + dryRun = isDryRun dataManagerSingleRun(t) - expected = []string{strayNewBlock} - expected = append(expected, oldBlocks...) + if dryRun { + // verify that all blocks, including strayOldBlock, are still to be found + verifyBlocks(t, nil, expected, 2) + } else { + // verify that strayOldBlock is not to be found, but the collections blocks are still there + verifyBlocks(t, []string{strayOldBlock}, oldBlocks, 2) + } +} - notExpected := []string{strayOldBlock} - notExpected = append(notExpected, toBeDeletedCollectionBlocks...) +// Add one more keepstore with the given service type +func addExtraKeepServer(t *testing.T, serviceType string) string { + defer switchToken(arvadostest.AdminToken)() + + extraKeepService := make(arvadosclient.Dict) + err := arv.Create("keep_services", + arvadosclient.Dict{"keep_service": arvadosclient.Dict{ + "service_host": "localhost", + "service_port": "21321", + "service_ssl_flag": false, + "service_type": serviceType}}, + &extraKeepService) + if err != nil { + t.Fatal(err) + } - verifyBlocks(t, notExpected, expected, 2) + return extraKeepService["uuid"].(string) +} + +func deleteExtraKeepServer(uuid string) { + defer switchToken(arvadostest.AdminToken)() + arv.Delete("keep_services", uuid, nil, nil) }