X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/8a6c7aa5eac21e93d721bf958ca67eb7f216888e..bde564d8ad6f00326c16d7b85ce6dbeec9d338d3:/services/datamanager/datamanager_test.go

diff --git a/services/datamanager/datamanager_test.go b/services/datamanager/datamanager_test.go
index f432c07170..a99ec6b052 100644
--- a/services/datamanager/datamanager_test.go
+++ b/services/datamanager/datamanager_test.go
@@ -12,6 +12,7 @@ import (
 	"net/http"
 	"os"
 	"os/exec"
+	"path"
 	"regexp"
 	"strings"
 	"testing"
@@ -41,7 +42,6 @@ func SetupDataManagerTest(t *testing.T) {
 	keepClient = &keepclient.KeepClient{
 		Arvados:       &arv,
 		Want_replicas: 2,
-		Using_proxy:   true,
 		Client:        &http.Client{},
 	}
 
@@ -538,12 +538,33 @@ func TestPutAndGetBlocks_NoErrorDuringSingleRun(t *testing.T) {
 	testOldBlocksNotDeletedOnDataManagerError(t, "", "", false, false)
 }
 
+func createBadPath(t *testing.T) (badpath string) {
+	tempdir, err := ioutil.TempDir("", "bad")
+	if err != nil {
+		t.Fatalf("Could not create temporary directory for bad path: %v", err)
+	}
+	badpath = path.Join(tempdir, "bad")
+	return
+}
+
+func destroyBadPath(t *testing.T, badpath string) {
+	tempdir := path.Join(badpath, "..")
+	err := os.Remove(tempdir)
+	if err != nil {
+		t.Fatalf("Could not remove bad path temporary directory %v: %v", tempdir, err)
+	}
+}
+
 func TestPutAndGetBlocks_ErrorDuringGetCollectionsBadWriteTo(t *testing.T) {
-	testOldBlocksNotDeletedOnDataManagerError(t, "/badwritetofile", "", true, true)
+	badpath := createBadPath(t)
+	defer destroyBadPath(t, badpath)
+	testOldBlocksNotDeletedOnDataManagerError(t, path.Join(badpath, "writetofile"), "", true, true)
 }
 
 func TestPutAndGetBlocks_ErrorDuringGetCollectionsBadHeapProfileFilename(t *testing.T) {
-	testOldBlocksNotDeletedOnDataManagerError(t, "", "/badheapprofilefile", true, true)
+	badpath := createBadPath(t)
+	defer destroyBadPath(t, badpath)
+	testOldBlocksNotDeletedOnDataManagerError(t, "", path.Join(badpath, "heapprofilefile"), true, true)
 }
 
 // Create some blocks and backdate some of them.
@@ -625,23 +646,31 @@ func createMultiStreamBlockCollection(t *testing.T, data string, numStreams, num
 // Also, create stray block and backdate it.
 // After datamanager run: expect blocks from the collection, but not the stray block.
 func TestManifestWithMultipleStreamsAndBlocks(t *testing.T) {
-	testManifestWithMultipleStreamsAndBlocks(t, false)
+	testManifestWithMultipleStreamsAndBlocks(t, 100, 10, "", false)
 }
 
+// Same test as TestManifestWithMultipleStreamsAndBlocks with an additional
+// keepstore of a service type other than "disk". Only the "disk" type services
+// will be indexed by datamanager and hence should work the same way.
+func TestManifestWithMultipleStreamsAndBlocks_WithOneUnsupportedKeepServer(t *testing.T) {
+	testManifestWithMultipleStreamsAndBlocks(t, 2, 2, "testblobstore", false)
+}
+
+// Test datamanager with dry-run. Expect no block to be deleted.
 func TestManifestWithMultipleStreamsAndBlocks_DryRun(t *testing.T) {
-	testManifestWithMultipleStreamsAndBlocks(t, true)
+	testManifestWithMultipleStreamsAndBlocks(t, 2, 2, "", true)
 }
 
-func testManifestWithMultipleStreamsAndBlocks(t *testing.T, isDryRun bool) {
+func testManifestWithMultipleStreamsAndBlocks(t *testing.T, numStreams, numBlocks int, createExtraKeepServerWithType string, isDryRun bool) {
 	defer TearDownDataManagerTest(t)
 	SetupDataManagerTest(t)
 
 	// create collection whose blocks will be backdated
-	collectionWithOldBlocks, oldBlocks := createMultiStreamBlockCollection(t, "old block", 100, 10)
+	collectionWithOldBlocks, oldBlocks := createMultiStreamBlockCollection(t, "old block", numStreams, numBlocks)
 	if collectionWithOldBlocks == "" {
-		t.Fatalf("Failed to create collection with 1000 blocks")
+		t.Fatalf("Failed to create collection with %d blocks", numStreams*numBlocks)
 	}
-	if len(oldBlocks) != 1000 {
+	if len(oldBlocks) != numStreams*numBlocks {
 		t.Fatalf("Not all blocks are created: expected %v, found %v", 1000, len(oldBlocks))
 	}
 
@@ -658,6 +687,14 @@ func testManifestWithMultipleStreamsAndBlocks(t *testing.T, isDryRun bool) {
 	// also backdate the stray old block
 	backdateBlocks(t, []string{strayOldBlock})
 
+	// If requested, create an extra keepserver with the given type
+	// This should be ignored during indexing and hence not change the datamanager outcome
+	var extraKeepServerUUID string
+	if createExtraKeepServerWithType != "" {
+		extraKeepServerUUID = addExtraKeepServer(t, createExtraKeepServerWithType)
+		defer deleteExtraKeepServer(extraKeepServerUUID)
+	}
+
 	// run datamanager
 	dryRun = isDryRun
 	dataManagerSingleRun(t)
@@ -670,3 +707,27 @@ func testManifestWithMultipleStreamsAndBlocks(t *testing.T, isDryRun bool) {
 		verifyBlocks(t, []string{strayOldBlock}, oldBlocks, 2)
 	}
 }
+
+// Add one more keepstore with the given service type
+func addExtraKeepServer(t *testing.T, serviceType string) string {
+	defer switchToken(arvadostest.AdminToken)()
+
+	extraKeepService := make(arvadosclient.Dict)
+	err := arv.Create("keep_services",
+		arvadosclient.Dict{"keep_service": arvadosclient.Dict{
+			"service_host":     "localhost",
+			"service_port":     "21321",
+			"service_ssl_flag": false,
+			"service_type":     serviceType}},
+		&extraKeepService)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	return extraKeepService["uuid"].(string)
+}
+
+func deleteExtraKeepServer(uuid string) {
+	defer switchToken(arvadostest.AdminToken)()
+	arv.Delete("keep_services", uuid, nil, nil)
+}