4194: improve logging when large block is retrieved.
[arvados.git] / services / keepstore / handler_test.go
index 200e1b17930ebcad9193309b8c039b74585af318..32479d829bec0194a3c8548152fdad29eb8461db 100644 (file)
@@ -224,21 +224,15 @@ func TestPutHandler(t *testing.T) {
 }
 
 // Test /index requests:
-//   - enforce_permissions off | unauthenticated /index request
-//   - enforce_permissions off | unauthenticated /index/prefix request
-//   - enforce_permissions off | authenticated /index request        | non-superuser
-//   - enforce_permissions off | authenticated /index/prefix request | non-superuser
-//   - enforce_permissions off | authenticated /index request        | superuser
-//   - enforce_permissions off | authenticated /index/prefix request | superuser
-//   - enforce_permissions on  | unauthenticated /index request
-//   - enforce_permissions on  | unauthenticated /index/prefix request
-//   - enforce_permissions on  | authenticated /index request        | non-superuser
-//   - enforce_permissions on  | authenticated /index/prefix request | non-superuser
-//   - enforce_permissions on  | authenticated /index request        | superuser
-//   - enforce_permissions on  | authenticated /index/prefix request | superuser
+//   - unauthenticated /index request
+//   - unauthenticated /index/prefix request
+//   - authenticated   /index request        | non-superuser
+//   - authenticated   /index/prefix request | non-superuser
+//   - authenticated   /index request        | superuser
+//   - authenticated   /index/prefix request | superuser
 //
 // The only /index requests that should succeed are those issued by the
-// superuser when enforce_permissions = true.
+// superuser. They should pass regardless of the value of enforce_permissions.
 //
 func TestIndexHandler(t *testing.T) {
        defer teardown()
@@ -289,95 +283,58 @@ func TestIndexHandler(t *testing.T) {
                api_token: data_manager_token,
        }
 
-       // ----------------------------
-       // enforce_permissions disabled
-       // All /index requests should fail.
-       enforce_permissions = false
+       // -------------------------------------------------------------
+       // Only the superuser should be allowed to issue /index requests.
+
+  // ---------------------------
+  // enforce_permissions enabled
+       // This setting should not affect tests passing.
+  enforce_permissions = true
 
        // unauthenticated /index request
-       // => PermissionError
+       // => UnauthorizedError
        response := IssueRequest(rest, unauthenticated_req)
        ExpectStatusCode(t,
-               "enforce_permissions off, unauthenticated request",
-               PermissionError.HTTPCode,
+               "enforce_permissions on, unauthenticated request",
+               UnauthorizedError.HTTPCode,
                response)
 
        // unauthenticated /index/prefix request
-       // => PermissionError
+       // => UnauthorizedError
        response = IssueRequest(rest, unauth_prefix_req)
        ExpectStatusCode(t,
-               "enforce_permissions off, unauthenticated /index/prefix request",
-               PermissionError.HTTPCode,
+               "permissions on, unauthenticated /index/prefix request",
+               UnauthorizedError.HTTPCode,
                response)
 
        // authenticated /index request, non-superuser
-       // => PermissionError
+       // => UnauthorizedError
        response = IssueRequest(rest, authenticated_req)
        ExpectStatusCode(t,
-               "enforce_permissions off, authenticated request, non-superuser",
-               PermissionError.HTTPCode,
+               "permissions on, authenticated request, non-superuser",
+               UnauthorizedError.HTTPCode,
                response)
 
        // authenticated /index/prefix request, non-superuser
-       // => PermissionError
+       // => UnauthorizedError
        response = IssueRequest(rest, auth_prefix_req)
        ExpectStatusCode(t,
-               "enforce_permissions off, authenticated /index/prefix request, non-superuser",
-               PermissionError.HTTPCode,
+               "permissions on, authenticated /index/prefix request, non-superuser",
+               UnauthorizedError.HTTPCode,
                response)
 
-       // authenticated /index request, superuser
-       // => PermissionError
+       // superuser /index request
+       // => OK
        response = IssueRequest(rest, superuser_req)
        ExpectStatusCode(t,
-               "enforce_permissions off, superuser request",
-               PermissionError.HTTPCode,
-               response)
-
-       // superuser /index/prefix request
-       // => PermissionError
-       response = IssueRequest(rest, superuser_prefix_req)
-       ExpectStatusCode(t,
-               "enforce_permissions off, superuser /index/prefix request",
-               PermissionError.HTTPCode,
-               response)
-
-       // ---------------------------
-       // enforce_permissions enabled
-       // Only the superuser should be allowed to issue /index requests.
-       enforce_permissions = true
-
-       // unauthenticated /index request
-       // => PermissionError
-       response = IssueRequest(rest, unauthenticated_req)
-       ExpectStatusCode(t,
-               "enforce_permissions on, unauthenticated request",
-               PermissionError.HTTPCode,
-               response)
-
-       // unauthenticated /index/prefix request
-       // => PermissionError
-       response = IssueRequest(rest, unauth_prefix_req)
-       ExpectStatusCode(t,
-               "permissions on, unauthenticated /index/prefix request",
-               PermissionError.HTTPCode,
-               response)
-
-       // authenticated /index request, non-superuser
-       // => PermissionError
-       response = IssueRequest(rest, authenticated_req)
-       ExpectStatusCode(t,
-               "permissions on, authenticated request, non-superuser",
-               PermissionError.HTTPCode,
+               "permissions on, superuser request",
+               http.StatusOK,
                response)
 
-       // authenticated /index/prefix request, non-superuser
-       // => PermissionError
-       response = IssueRequest(rest, auth_prefix_req)
-       ExpectStatusCode(t,
-               "permissions on, authenticated /index/prefix request, non-superuser",
-               PermissionError.HTTPCode,
-               response)
+       // ----------------------------
+       // enforce_permissions disabled
+       // Valid Request should still pass.
+       enforce_permissions = false
 
        // superuser /index request
        // => OK
@@ -387,6 +344,8 @@ func TestIndexHandler(t *testing.T) {
                http.StatusOK,
                response)
 
+
+
        expected := `^` + TEST_HASH + `\+\d+ \d+\n` +
                TEST_HASH_2 + `\+\d+ \d+\n$`
        match, _ := regexp.MatchString(expected, response.Body.String())
@@ -566,6 +525,218 @@ func TestDeleteHandler(t *testing.T) {
        }
 }
 
+// TestPullHandler
+//
+// Test handling of the PUT /pull statement.
+//
+// Cases tested: syntactically valid and invalid pull lists, from the
+// data manager and from unprivileged users:
+//
+//   1. Valid pull list from an ordinary user
+//      (expected result: 401 Unauthorized)
+//
+//   2. Invalid pull request from an ordinary user
+//      (expected result: 401 Unauthorized)
+//
+//   3. Valid pull request from the data manager
+//      (expected result: 200 OK with request body "Received 3 pull
+//      requests"
+//
+//   4. Invalid pull request from the data manager
+//      (expected result: 400 Bad Request)
+//
+// Test that in the end, the pull manager received a good pull list with
+// the expected number of requests.
+//
+// TODO(twp): test concurrency: launch 100 goroutines to update the
+// pull list simultaneously.  Make sure that none of them return 400
+// Bad Request and that pullq.GetList() returns a valid list.
+//
+func TestPullHandler(t *testing.T) {
+       defer teardown()
+
+       // Set up a REST router for testing the handlers.
+       rest := MakeRESTRouter()
+
+       var user_token = "USER TOKEN"
+       data_manager_token = "DATA MANAGER TOKEN"
+
+       good_json := []byte(`[
+               {
+                       "locator":"locator_with_two_servers",
+                       "servers":[
+                               "server1",
+                               "server2"
+                       ]
+               },
+               {
+                       "locator":"locator_with_no_servers",
+                       "servers":[]
+               },
+               {
+                       "locator":"",
+                       "servers":["empty_locator"]
+               }
+       ]`)
+
+       bad_json := []byte(`{ "key":"I'm a little teapot" }`)
+
+       type pullTest struct {
+               name          string
+               req           RequestTester
+               response_code int
+               response_body string
+       }
+       var testcases = []pullTest{
+               {
+                       "Valid pull list from an ordinary user",
+                       RequestTester{"/pull", user_token, "PUT", good_json},
+                       http.StatusUnauthorized,
+                       "Unauthorized\n",
+               },
+               {
+                       "Invalid pull request from an ordinary user",
+                       RequestTester{"/pull", user_token, "PUT", bad_json},
+                       http.StatusUnauthorized,
+                       "Unauthorized\n",
+               },
+               {
+                       "Valid pull request from the data manager",
+                       RequestTester{"/pull", data_manager_token, "PUT", good_json},
+                       http.StatusOK,
+                       "Received 3 pull requests\n",
+               },
+               {
+                       "Invalid pull request from the data manager",
+                       RequestTester{"/pull", data_manager_token, "PUT", bad_json},
+                       http.StatusBadRequest,
+                       "Bad Request\n",
+               },
+       }
+
+       for _, tst := range testcases {
+               response := IssueRequest(rest, &tst.req)
+               ExpectStatusCode(t, tst.name, tst.response_code, response)
+               ExpectBody(t, tst.name, tst.response_body, response)
+       }
+
+       // The Keep pull manager should have received one good list with 3
+       // requests on it.
+       for i := 0; i < 3; i++ {
+               item := <-pullq.NextItem
+               if _, ok := item.(PullRequest); !ok {
+                       t.Errorf("item %v could not be parsed as a PullRequest", item)
+               }
+       }
+
+       expectChannelEmpty(t, pullq.NextItem)
+}
+
+// TestTrashHandler
+//
+// Test cases:
+//
+// Cases tested: syntactically valid and invalid trash lists, from the
+// data manager and from unprivileged users:
+//
+//   1. Valid trash list from an ordinary user
+//      (expected result: 401 Unauthorized)
+//
+//   2. Invalid trash list from an ordinary user
+//      (expected result: 401 Unauthorized)
+//
+//   3. Valid trash list from the data manager
+//      (expected result: 200 OK with request body "Received 3 trash
+//      requests"
+//
+//   4. Invalid trash list from the data manager
+//      (expected result: 400 Bad Request)
+//
+// Test that in the end, the trash collector received a good list
+// trash list with the expected number of requests.
+//
+// TODO(twp): test concurrency: launch 100 goroutines to update the
+// pull list simultaneously.  Make sure that none of them return 400
+// Bad Request and that replica.Dump() returns a valid list.
+//
+func TestTrashHandler(t *testing.T) {
+       defer teardown()
+
+       // Set up a REST router for testing the handlers.
+       rest := MakeRESTRouter()
+
+       var user_token = "USER TOKEN"
+       data_manager_token = "DATA MANAGER TOKEN"
+
+       good_json := []byte(`[
+               {
+                       "locator":"block1",
+                       "block_mtime":1409082153
+               },
+               {
+                       "locator":"block2",
+                       "block_mtime":1409082153
+               },
+               {
+                       "locator":"block3",
+                       "block_mtime":1409082153
+               }
+       ]`)
+
+       bad_json := []byte(`I am not a valid JSON string`)
+
+       type trashTest struct {
+               name          string
+               req           RequestTester
+               response_code int
+               response_body string
+       }
+
+       var testcases = []trashTest{
+               {
+                       "Valid trash list from an ordinary user",
+                       RequestTester{"/trash", user_token, "PUT", good_json},
+                       http.StatusUnauthorized,
+                       "Unauthorized\n",
+               },
+               {
+                       "Invalid trash list from an ordinary user",
+                       RequestTester{"/trash", user_token, "PUT", bad_json},
+                       http.StatusUnauthorized,
+                       "Unauthorized\n",
+               },
+               {
+                       "Valid trash list from the data manager",
+                       RequestTester{"/trash", data_manager_token, "PUT", good_json},
+                       http.StatusOK,
+                       "Received 3 trash requests\n",
+               },
+               {
+                       "Invalid trash list from the data manager",
+                       RequestTester{"/trash", data_manager_token, "PUT", bad_json},
+                       http.StatusBadRequest,
+                       "Bad Request\n",
+               },
+       }
+
+       for _, tst := range testcases {
+               response := IssueRequest(rest, &tst.req)
+               ExpectStatusCode(t, tst.name, tst.response_code, response)
+               ExpectBody(t, tst.name, tst.response_body, response)
+       }
+
+       // The trash collector should have received one good list with 3
+       // requests on it.
+       for i := 0; i < 3; i++ {
+               item := <-trashq.NextItem
+               if _, ok := item.(TrashRequest); !ok {
+                       t.Errorf("item %v could not be parsed as a TrashRequest", item)
+               }
+       }
+
+       expectChannelEmpty(t, trashq.NextItem)
+}
+
 // ====================
 // Helper functions
 // ====================
@@ -579,7 +750,8 @@ func IssueRequest(router *mux.Router, rt *RequestTester) *httptest.ResponseRecor
        if rt.api_token != "" {
                req.Header.Set("Authorization", "OAuth2 "+rt.api_token)
        }
-       router.ServeHTTP(response, req)
+  routerWrapper := WrapRESTRouter{router}
+  routerWrapper.ServeHTTP(response, req)
        return response
 }