X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/244159419c42341baeb388236ad29cc546b7eca1..d78db2b1b518abdac0893e65123504fb85319010:/services/keepstore/handler_test.go

diff --git a/services/keepstore/handler_test.go b/services/keepstore/handler_test.go
index 64a417f9a7..fbf4ef3252 100644
--- a/services/keepstore/handler_test.go
+++ b/services/keepstore/handler_test.go
@@ -13,7 +13,6 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"github.com/gorilla/mux"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -51,9 +50,6 @@ func TestGetHandler(t *testing.T) {
 		t.Error(err)
 	}
 
-	// Set up a REST router for testing the handlers.
-	rest := MakeRESTRouter()
-
 	// Create locators for testing.
 	// Turn on permission settings so we can generate signed locators.
 	enforce_permissions = true
@@ -74,7 +70,7 @@ func TestGetHandler(t *testing.T) {
 
 	// Unauthenticated request, unsigned locator
 	// => OK
-	response := IssueRequest(rest,
+	response := IssueRequest(
 		&RequestTester{
 			method: "GET",
 			uri:    unsigned_locator,
@@ -97,7 +93,7 @@ func TestGetHandler(t *testing.T) {
 
 	// Authenticated request, signed locator
 	// => OK
-	response = IssueRequest(rest, &RequestTester{
+	response = IssueRequest(&RequestTester{
 		method:    "GET",
 		uri:       signed_locator,
 		api_token: known_token,
@@ -114,7 +110,7 @@ func TestGetHandler(t *testing.T) {
 
 	// Authenticated request, unsigned locator
 	// => PermissionError
-	response = IssueRequest(rest, &RequestTester{
+	response = IssueRequest(&RequestTester{
 		method:    "GET",
 		uri:       unsigned_locator,
 		api_token: known_token,
@@ -123,7 +119,7 @@ func TestGetHandler(t *testing.T) {
 
 	// Unauthenticated request, signed locator
 	// => PermissionError
-	response = IssueRequest(rest, &RequestTester{
+	response = IssueRequest(&RequestTester{
 		method: "GET",
 		uri:    signed_locator,
 	})
@@ -133,7 +129,7 @@ func TestGetHandler(t *testing.T) {
 
 	// Authenticated request, expired locator
 	// => ExpiredError
-	response = IssueRequest(rest, &RequestTester{
+	response = IssueRequest(&RequestTester{
 		method:    "GET",
 		uri:       expired_locator,
 		api_token: known_token,
@@ -155,16 +151,13 @@ func TestPutHandler(t *testing.T) {
 	KeepVM = MakeTestVolumeManager(2)
 	defer KeepVM.Quit()
 
-	// Set up a REST router for testing the handlers.
-	rest := MakeRESTRouter()
-
 	// --------------
 	// No server key.
 
 	// Unauthenticated request, no server key
 	// => OK (unsigned response)
 	unsigned_locator := "/" + TEST_HASH
-	response := IssueRequest(rest,
+	response := IssueRequest(
 		&RequestTester{
 			method:       "PUT",
 			uri:          unsigned_locator,
@@ -188,7 +181,7 @@ func TestPutHandler(t *testing.T) {
 
 	// Authenticated PUT, signed locator
 	// => OK (signed response)
-	response = IssueRequest(rest,
+	response = IssueRequest(
 		&RequestTester{
 			method:       "PUT",
 			uri:          unsigned_locator,
@@ -208,7 +201,7 @@ func TestPutHandler(t *testing.T) {
 
 	// Unauthenticated PUT, unsigned locator
 	// => OK
-	response = IssueRequest(rest,
+	response = IssueRequest(
 		&RequestTester{
 			method:       "PUT",
 			uri:          unsigned_locator,
@@ -224,21 +217,15 @@ func TestPutHandler(t *testing.T) {
 }
 
 // Test /index requests:
-//   - enforce_permissions off | unauthenticated /index request
-//   - enforce_permissions off | unauthenticated /index/prefix request
-//   - enforce_permissions off | authenticated /index request        | non-superuser
-//   - enforce_permissions off | authenticated /index/prefix request | non-superuser
-//   - enforce_permissions off | authenticated /index request        | superuser
-//   - enforce_permissions off | authenticated /index/prefix request | superuser
-//   - enforce_permissions on  | unauthenticated /index request
-//   - enforce_permissions on  | unauthenticated /index/prefix request
-//   - enforce_permissions on  | authenticated /index request        | non-superuser
-//   - enforce_permissions on  | authenticated /index/prefix request | non-superuser
-//   - enforce_permissions on  | authenticated /index request        | superuser
-//   - enforce_permissions on  | authenticated /index/prefix request | superuser
+//   - unauthenticated /index request
+//   - unauthenticated /index/prefix request
+//   - authenticated   /index request        | non-superuser
+//   - authenticated   /index/prefix request | non-superuser
+//   - authenticated   /index request        | superuser
+//   - authenticated   /index/prefix request | superuser
 //
 // The only /index requests that should succeed are those issued by the
-// superuser when enforce_permissions = true.
+// superuser. They should pass regardless of the value of enforce_permissions.
 //
 func TestIndexHandler(t *testing.T) {
 	defer teardown()
@@ -255,9 +242,6 @@ func TestIndexHandler(t *testing.T) {
 	vols[0].Put(TEST_HASH+".meta", []byte("metadata"))
 	vols[1].Put(TEST_HASH_2+".meta", []byte("metadata"))
 
-	// Set up a REST router for testing the handlers.
-	rest := MakeRESTRouter()
-
 	data_manager_token = "DATA MANAGER TOKEN"
 
 	unauthenticated_req := &RequestTester{
@@ -289,104 +273,69 @@ func TestIndexHandler(t *testing.T) {
 		api_token: data_manager_token,
 	}
 
-	// ----------------------------
-	// enforce_permissions disabled
-	// All /index requests should fail.
-	enforce_permissions = false
-
-	// unauthenticated /index request
-	// => PermissionError
-	response := IssueRequest(rest, unauthenticated_req)
-	ExpectStatusCode(t,
-		"enforce_permissions off, unauthenticated request",
-		PermissionError.HTTPCode,
-		response)
-
-	// unauthenticated /index/prefix request
-	// => PermissionError
-	response = IssueRequest(rest, unauth_prefix_req)
-	ExpectStatusCode(t,
-		"enforce_permissions off, unauthenticated /index/prefix request",
-		PermissionError.HTTPCode,
-		response)
-
-	// authenticated /index request, non-superuser
-	// => PermissionError
-	response = IssueRequest(rest, authenticated_req)
-	ExpectStatusCode(t,
-		"enforce_permissions off, authenticated request, non-superuser",
-		PermissionError.HTTPCode,
-		response)
-
-	// authenticated /index/prefix request, non-superuser
-	// => PermissionError
-	response = IssueRequest(rest, auth_prefix_req)
-	ExpectStatusCode(t,
-		"enforce_permissions off, authenticated /index/prefix request, non-superuser",
-		PermissionError.HTTPCode,
-		response)
-
-	// authenticated /index request, superuser
-	// => PermissionError
-	response = IssueRequest(rest, superuser_req)
-	ExpectStatusCode(t,
-		"enforce_permissions off, superuser request",
-		PermissionError.HTTPCode,
-		response)
-
-	// superuser /index/prefix request
-	// => PermissionError
-	response = IssueRequest(rest, superuser_prefix_req)
-	ExpectStatusCode(t,
-		"enforce_permissions off, superuser /index/prefix request",
-		PermissionError.HTTPCode,
-		response)
-
-	// ---------------------------
-	// enforce_permissions enabled
+	// -------------------------------------------------------------
 	// Only the superuser should be allowed to issue /index requests.
-	enforce_permissions = true
+
+  // ---------------------------
+  // enforce_permissions enabled
+	// This setting should not affect tests passing.
+  enforce_permissions = true
 
 	// unauthenticated /index request
-	// => PermissionError
-	response = IssueRequest(rest, unauthenticated_req)
+	// => UnauthorizedError
+	response := IssueRequest(unauthenticated_req)
 	ExpectStatusCode(t,
 		"enforce_permissions on, unauthenticated request",
-		PermissionError.HTTPCode,
+		UnauthorizedError.HTTPCode,
 		response)
 
 	// unauthenticated /index/prefix request
-	// => PermissionError
-	response = IssueRequest(rest, unauth_prefix_req)
+	// => UnauthorizedError
+	response = IssueRequest(unauth_prefix_req)
 	ExpectStatusCode(t,
 		"permissions on, unauthenticated /index/prefix request",
-		PermissionError.HTTPCode,
+		UnauthorizedError.HTTPCode,
 		response)
 
 	// authenticated /index request, non-superuser
-	// => PermissionError
-	response = IssueRequest(rest, authenticated_req)
+	// => UnauthorizedError
+	response = IssueRequest(authenticated_req)
 	ExpectStatusCode(t,
 		"permissions on, authenticated request, non-superuser",
-		PermissionError.HTTPCode,
+		UnauthorizedError.HTTPCode,
 		response)
 
 	// authenticated /index/prefix request, non-superuser
-	// => PermissionError
-	response = IssueRequest(rest, auth_prefix_req)
+	// => UnauthorizedError
+	response = IssueRequest(auth_prefix_req)
 	ExpectStatusCode(t,
 		"permissions on, authenticated /index/prefix request, non-superuser",
-		PermissionError.HTTPCode,
+		UnauthorizedError.HTTPCode,
 		response)
 
 	// superuser /index request
 	// => OK
-	response = IssueRequest(rest, superuser_req)
+	response = IssueRequest(superuser_req)
+	ExpectStatusCode(t,
+		"permissions on, superuser request",
+		http.StatusOK,
+		response)
+
+	// ----------------------------
+	// enforce_permissions disabled
+	// Valid Request should still pass.
+	enforce_permissions = false
+
+	// superuser /index request
+	// => OK
+	response = IssueRequest(superuser_req)
 	ExpectStatusCode(t,
 		"permissions on, superuser request",
 		http.StatusOK,
 		response)
 
+
+
 	expected := `^` + TEST_HASH + `\+\d+ \d+\n` +
 		TEST_HASH_2 + `\+\d+ \d+\n$`
 	match, _ := regexp.MatchString(expected, response.Body.String())
@@ -398,7 +347,7 @@ func TestIndexHandler(t *testing.T) {
 
 	// superuser /index/prefix request
 	// => OK
-	response = IssueRequest(rest, superuser_prefix_req)
+	response = IssueRequest(superuser_prefix_req)
 	ExpectStatusCode(t,
 		"permissions on, superuser request",
 		http.StatusOK,
@@ -451,8 +400,10 @@ func TestDeleteHandler(t *testing.T) {
 	vols := KeepVM.Volumes()
 	vols[0].Put(TEST_HASH, TEST_BLOCK)
 
-	// Set up a REST router for testing the handlers.
-	rest := MakeRESTRouter()
+	// Explicitly set the permission_ttl to 0 for these
+	// tests, to ensure the MockVolume deletes the blocks
+	// even though they have just been created.
+	permission_ttl = time.Duration(0)
 
 	var user_token = "NOT DATA MANAGER TOKEN"
 	data_manager_token = "DATA MANAGER TOKEN"
@@ -482,14 +433,14 @@ func TestDeleteHandler(t *testing.T) {
 
 	// Unauthenticated request returns PermissionError.
 	var response *httptest.ResponseRecorder
-	response = IssueRequest(rest, unauth_req)
+	response = IssueRequest(unauth_req)
 	ExpectStatusCode(t,
 		"unauthenticated request",
 		PermissionError.HTTPCode,
 		response)
 
 	// Authenticated non-admin request returns PermissionError.
-	response = IssueRequest(rest, user_req)
+	response = IssueRequest(user_req)
 	ExpectStatusCode(t,
 		"authenticated non-admin request",
 		PermissionError.HTTPCode,
@@ -502,7 +453,7 @@ func TestDeleteHandler(t *testing.T) {
 	}
 	var response_dc, expected_dc deletecounter
 
-	response = IssueRequest(rest, superuser_nonexistent_block_req)
+	response = IssueRequest(superuser_nonexistent_block_req)
 	ExpectStatusCode(t,
 		"data manager request, nonexistent block",
 		http.StatusNotFound,
@@ -510,7 +461,7 @@ func TestDeleteHandler(t *testing.T) {
 
 	// Authenticated admin request for existing block while never_delete is set.
 	never_delete = true
-	response = IssueRequest(rest, superuser_existing_block_req)
+	response = IssueRequest(superuser_existing_block_req)
 	ExpectStatusCode(t,
 		"authenticated request, existing block, method disabled",
 		MethodDisabledError.HTTPCode,
@@ -518,7 +469,7 @@ func TestDeleteHandler(t *testing.T) {
 	never_delete = false
 
 	// Authenticated admin request for existing block.
-	response = IssueRequest(rest, superuser_existing_block_req)
+	response = IssueRequest(superuser_existing_block_req)
 	ExpectStatusCode(t,
 		"data manager request, existing block",
 		http.StatusOK,
@@ -536,6 +487,235 @@ func TestDeleteHandler(t *testing.T) {
 	if !block_deleted {
 		t.Error("superuser_existing_block_req: block not deleted")
 	}
+
+	// A DELETE request on a block newer than permission_ttl should return
+	// success but leave the block on the volume.
+	vols[0].Put(TEST_HASH, TEST_BLOCK)
+	permission_ttl = time.Duration(1) * time.Hour
+
+	response = IssueRequest(superuser_existing_block_req)
+	ExpectStatusCode(t,
+		"data manager request, existing block",
+		http.StatusOK,
+		response)
+	// Expect response {"copies_deleted":1,"copies_failed":0}
+	expected_dc = deletecounter{1, 0}
+	json.NewDecoder(response.Body).Decode(&response_dc)
+	if response_dc != expected_dc {
+		t.Errorf("superuser_existing_block_req\nexpected: %+v\nreceived: %+v",
+			expected_dc, response_dc)
+	}
+	// Confirm the block has NOT been deleted.
+	_, err = vols[0].Get(TEST_HASH)
+	if err != nil {
+		t.Errorf("testing delete on new block: %s\n", err)
+	}
+}
+
+// TestPullHandler
+//
+// Test handling of the PUT /pull statement.
+//
+// Cases tested: syntactically valid and invalid pull lists, from the
+// data manager and from unprivileged users:
+//
+//   1. Valid pull list from an ordinary user
+//      (expected result: 401 Unauthorized)
+//
+//   2. Invalid pull request from an ordinary user
+//      (expected result: 401 Unauthorized)
+//
+//   3. Valid pull request from the data manager
+//      (expected result: 200 OK with request body "Received 3 pull
+//      requests"
+//
+//   4. Invalid pull request from the data manager
+//      (expected result: 400 Bad Request)
+//
+// Test that in the end, the pull manager received a good pull list with
+// the expected number of requests.
+//
+// TODO(twp): test concurrency: launch 100 goroutines to update the
+// pull list simultaneously.  Make sure that none of them return 400
+// Bad Request and that pullq.GetList() returns a valid list.
+//
+func TestPullHandler(t *testing.T) {
+	defer teardown()
+
+	var user_token = "USER TOKEN"
+	data_manager_token = "DATA MANAGER TOKEN"
+
+	good_json := []byte(`[
+		{
+			"locator":"locator_with_two_servers",
+			"servers":[
+				"server1",
+				"server2"
+		 	]
+		},
+		{
+			"locator":"locator_with_no_servers",
+			"servers":[]
+		},
+		{
+			"locator":"",
+			"servers":["empty_locator"]
+		}
+	]`)
+
+	bad_json := []byte(`{ "key":"I'm a little teapot" }`)
+
+	type pullTest struct {
+		name          string
+		req           RequestTester
+		response_code int
+		response_body string
+	}
+	var testcases = []pullTest{
+		{
+			"Valid pull list from an ordinary user",
+			RequestTester{"/pull", user_token, "PUT", good_json},
+			http.StatusUnauthorized,
+			"Unauthorized\n",
+		},
+		{
+			"Invalid pull request from an ordinary user",
+			RequestTester{"/pull", user_token, "PUT", bad_json},
+			http.StatusUnauthorized,
+			"Unauthorized\n",
+		},
+		{
+			"Valid pull request from the data manager",
+			RequestTester{"/pull", data_manager_token, "PUT", good_json},
+			http.StatusOK,
+			"Received 3 pull requests\n",
+		},
+		{
+			"Invalid pull request from the data manager",
+			RequestTester{"/pull", data_manager_token, "PUT", bad_json},
+			http.StatusBadRequest,
+			"Bad Request\n",
+		},
+	}
+
+	for _, tst := range testcases {
+		response := IssueRequest(&tst.req)
+		ExpectStatusCode(t, tst.name, tst.response_code, response)
+		ExpectBody(t, tst.name, tst.response_body, response)
+	}
+
+	// The Keep pull manager should have received one good list with 3
+	// requests on it.
+	for i := 0; i < 3; i++ {
+		item := <-pullq.NextItem
+		if _, ok := item.(PullRequest); !ok {
+			t.Errorf("item %v could not be parsed as a PullRequest", item)
+		}
+	}
+
+	expectChannelEmpty(t, pullq.NextItem)
+}
+
+// TestTrashHandler
+//
+// Test cases:
+//
+// Cases tested: syntactically valid and invalid trash lists, from the
+// data manager and from unprivileged users:
+//
+//   1. Valid trash list from an ordinary user
+//      (expected result: 401 Unauthorized)
+//
+//   2. Invalid trash list from an ordinary user
+//      (expected result: 401 Unauthorized)
+//
+//   3. Valid trash list from the data manager
+//      (expected result: 200 OK with request body "Received 3 trash
+//      requests"
+//
+//   4. Invalid trash list from the data manager
+//      (expected result: 400 Bad Request)
+//
+// Test that in the end, the trash collector received a good list
+// trash list with the expected number of requests.
+//
+// TODO(twp): test concurrency: launch 100 goroutines to update the
+// pull list simultaneously.  Make sure that none of them return 400
+// Bad Request and that replica.Dump() returns a valid list.
+//
+func TestTrashHandler(t *testing.T) {
+	defer teardown()
+
+	var user_token = "USER TOKEN"
+	data_manager_token = "DATA MANAGER TOKEN"
+
+	good_json := []byte(`[
+		{
+			"locator":"block1",
+			"block_mtime":1409082153
+		},
+		{
+			"locator":"block2",
+			"block_mtime":1409082153
+		},
+		{
+			"locator":"block3",
+			"block_mtime":1409082153
+		}
+	]`)
+
+	bad_json := []byte(`I am not a valid JSON string`)
+
+	type trashTest struct {
+		name          string
+		req           RequestTester
+		response_code int
+		response_body string
+	}
+
+	var testcases = []trashTest{
+		{
+			"Valid trash list from an ordinary user",
+			RequestTester{"/trash", user_token, "PUT", good_json},
+			http.StatusUnauthorized,
+			"Unauthorized\n",
+		},
+		{
+			"Invalid trash list from an ordinary user",
+			RequestTester{"/trash", user_token, "PUT", bad_json},
+			http.StatusUnauthorized,
+			"Unauthorized\n",
+		},
+		{
+			"Valid trash list from the data manager",
+			RequestTester{"/trash", data_manager_token, "PUT", good_json},
+			http.StatusOK,
+			"Received 3 trash requests\n",
+		},
+		{
+			"Invalid trash list from the data manager",
+			RequestTester{"/trash", data_manager_token, "PUT", bad_json},
+			http.StatusBadRequest,
+			"Bad Request\n",
+		},
+	}
+
+	for _, tst := range testcases {
+		response := IssueRequest(&tst.req)
+		ExpectStatusCode(t, tst.name, tst.response_code, response)
+		ExpectBody(t, tst.name, tst.response_body, response)
+	}
+
+	// The trash collector should have received one good list with 3
+	// requests on it.
+	for i := 0; i < 3; i++ {
+		item := <-trashq.NextItem
+		if _, ok := item.(TrashRequest); !ok {
+			t.Errorf("item %v could not be parsed as a TrashRequest", item)
+		}
+	}
+
+	expectChannelEmpty(t, trashq.NextItem)
 }
 
 // ====================
@@ -543,15 +723,16 @@ func TestDeleteHandler(t *testing.T) {
 // ====================
 
 // IssueTestRequest executes an HTTP request described by rt, to a
-// specified REST router.  It returns the HTTP response to the request.
-func IssueRequest(router *mux.Router, rt *RequestTester) *httptest.ResponseRecorder {
+// REST router.  It returns the HTTP response to the request.
+func IssueRequest(rt *RequestTester) *httptest.ResponseRecorder {
 	response := httptest.NewRecorder()
 	body := bytes.NewReader(rt.request_body)
 	req, _ := http.NewRequest(rt.method, rt.uri, body)
 	if rt.api_token != "" {
 		req.Header.Set("Authorization", "OAuth2 "+rt.api_token)
 	}
-	router.ServeHTTP(response, req)
+  routerWrapper := MakeRESTRouterWrapper()
+  routerWrapper.ServeHTTP(response, req)
 	return response
 }