14287: Ensure timestamps in responses have 9 digits of nanoseconds.

[arvados.git] / lib / controller / federation_test.go

diff --git a/lib/controller/federation_test.go b/lib/controller/federation_test.go
index 1c859cfc515d142a0289610e402e725e07bfebb1..d689bb00526d0dd701ec77316ff07727ca89e461 100644 (file)
--- a/lib/controller/federation_test.go
+++ b/lib/controller/federation_test.go
@@ -39,7 +39,8 @@ type FederationSuite struct {
        // provided by the integration test environment.
        remoteServer *httpserver.Server
        // remoteMock ("zmock") appends each incoming request to
-       // remoteMockRequests, and returns an empty 200 response.
+       // remoteMockRequests, and returns 200 with an empty JSON
+       // object.
        remoteMock         *httpserver.Server
        remoteMockRequests []http.Request
 }
@@ -55,14 +56,13 @@ func (s *FederationSuite) SetUpTest(c *check.C) {
        c.Assert(s.remoteMock.Start(), check.IsNil)
 
        cluster := &arvados.Cluster{
-               ClusterID:  "zhome",
-               PostgreSQL: integrationTestCluster().PostgreSQL,
-               TLS:        arvados.TLS{Insecure: true},
-               API: arvados.API{
-                       MaxItemsPerResponse:     1000,
-                       MaxRequestAmplification: 4,
-               },
+               ClusterID:                 "zhome",
+               PostgreSQL:                integrationTestCluster().PostgreSQL,
+               EnableBetaController14287: enableBetaController14287,
        }
+       cluster.TLS.Insecure = true
+       cluster.API.MaxItemsPerResponse = 1000
+       cluster.API.MaxRequestAmplification = 4
        arvadostest.SetServiceURL(&cluster.Services.RailsAPI, "http://localhost:1/")
        arvadostest.SetServiceURL(&cluster.Services.Controller, "http://localhost:/")
        s.testHandler = &Handler{Cluster: cluster}
@@ -93,6 +93,8 @@ func (s *FederationSuite) remoteMockHandler(w http.ResponseWriter, req *http.Req
        req.Body.Close()
        req.Body = ioutil.NopCloser(b)
        s.remoteMockRequests = append(s.remoteMockRequests, *req)
+       // Repond 200 with a valid JSON object
+       fmt.Fprint(w, "{}")
 }
 
 func (s *FederationSuite) TearDownTest(c *check.C) {
@@ -104,15 +106,15 @@ func (s *FederationSuite) TearDownTest(c *check.C) {
        }
 }
 
-func (s *FederationSuite) testRequest(req *http.Request) *http.Response {
+func (s *FederationSuite) testRequest(req *http.Request) *httptest.ResponseRecorder {
        resp := httptest.NewRecorder()
        s.testServer.Server.Handler.ServeHTTP(resp, req)
-       return resp.Result()
+       return resp
 }
 
 func (s *FederationSuite) TestLocalRequest(c *check.C) {
        req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zhome-", 1), nil)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        s.checkHandledLocally(c, resp)
 }
 
@@ -127,7 +129,7 @@ func (s *FederationSuite) checkHandledLocally(c *check.C, resp *http.Response) {
 
 func (s *FederationSuite) TestNoAuth(c *check.C) {
        req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
        s.checkJSONErrorMatches(c, resp, `Not logged in`)
 }
@@ -135,7 +137,7 @@ func (s *FederationSuite) TestNoAuth(c *check.C) {
 func (s *FederationSuite) TestBadAuth(c *check.C) {
        req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
        req.Header.Set("Authorization", "Bearer aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
        s.checkJSONErrorMatches(c, resp, `Not logged in`)
 }
@@ -143,7 +145,7 @@ func (s *FederationSuite) TestBadAuth(c *check.C) {
 func (s *FederationSuite) TestNoAccess(c *check.C) {
        req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.SpectatorToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
        s.checkJSONErrorMatches(c, resp, `.*not found`)
 }
@@ -151,7 +153,7 @@ func (s *FederationSuite) TestNoAccess(c *check.C) {
 func (s *FederationSuite) TestGetUnknownRemote(c *check.C) {
        req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zz404-", 1), nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
        s.checkJSONErrorMatches(c, resp, `.*no proxy available for cluster zz404`)
 }
@@ -163,7 +165,7 @@ func (s *FederationSuite) TestRemoteError(c *check.C) {
 
        req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusBadGateway)
        s.checkJSONErrorMatches(c, resp, `.*HTTP response to HTTPS client`)
 }
@@ -171,7 +173,7 @@ func (s *FederationSuite) TestRemoteError(c *check.C) {
 func (s *FederationSuite) TestGetRemoteWorkflow(c *check.C) {
        req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        var wf arvados.Workflow
        c.Check(json.NewDecoder(resp.Body).Decode(&wf), check.IsNil)
@@ -182,7 +184,7 @@ func (s *FederationSuite) TestGetRemoteWorkflow(c *check.C) {
 func (s *FederationSuite) TestOptionsMethod(c *check.C) {
        req := httptest.NewRequest("OPTIONS", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
        req.Header.Set("Origin", "https://example.com")
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        body, err := ioutil.ReadAll(resp.Body)
        c.Check(err, check.IsNil)
@@ -198,7 +200,7 @@ func (s *FederationSuite) TestOptionsMethod(c *check.C) {
 
 func (s *FederationSuite) TestRemoteWithTokenInQuery(c *check.C) {
        req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1)+"?api_token="+arvadostest.ActiveToken, nil)
-       s.testRequest(req)
+       s.testRequest(req).Result()
        c.Assert(s.remoteMockRequests, check.HasLen, 1)
        pr := s.remoteMockRequests[0]
        // Token is salted and moved from query to Authorization header.
@@ -207,28 +209,51 @@ func (s *FederationSuite) TestRemoteWithTokenInQuery(c *check.C) {
 }
 
 func (s *FederationSuite) TestLocalTokenSalted(c *check.C) {
-       req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1), nil)
-       req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       s.testRequest(req)
-       c.Assert(s.remoteMockRequests, check.HasLen, 1)
-       pr := s.remoteMockRequests[0]
-       // The salted token here has a "zzzzz-" UUID instead of a
-       // "ztest-" UUID because ztest's local database has the
-       // "zzzzz-" test fixtures. The "secret" part is HMAC(sha1,
-       // arvadostest.ActiveToken, "zmock") = "7fd3...".
-       c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/7fd31b61f39c0e82a4155592163218272cedacdc")
+       defer s.localServiceReturns404(c).Close()
+       for _, path := range []string{
+               // During the transition to the strongly typed
+               // controller implementation (#14287), workflows and
+               // collections test different code paths.
+               "/arvados/v1/workflows/" + strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1),
+               "/arvados/v1/collections/" + strings.Replace(arvadostest.UserAgreementCollection, "zzzzz-", "zmock-", 1),
+       } {
+               c.Log("testing path ", path)
+               s.remoteMockRequests = nil
+               req := httptest.NewRequest("GET", path, nil)
+               req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+               s.testRequest(req).Result()
+               c.Assert(s.remoteMockRequests, check.HasLen, 1)
+               pr := s.remoteMockRequests[0]
+               // The salted token here has a "zzzzz-" UUID instead of a
+               // "ztest-" UUID because ztest's local database has the
+               // "zzzzz-" test fixtures. The "secret" part is HMAC(sha1,
+               // arvadostest.ActiveToken, "zmock") = "7fd3...".
+               c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/7fd31b61f39c0e82a4155592163218272cedacdc")
+       }
 }
 
 func (s *FederationSuite) TestRemoteTokenNotSalted(c *check.C) {
+       defer s.localServiceReturns404(c).Close()
        // remoteToken can be any v1 token that doesn't appear in
        // ztest's local db.
        remoteToken := "abcdef00000000000000000000000000000000000000000000"
-       req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1), nil)
-       req.Header.Set("Authorization", "Bearer "+remoteToken)
-       s.testRequest(req)
-       c.Assert(s.remoteMockRequests, check.HasLen, 1)
-       pr := s.remoteMockRequests[0]
-       c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer "+remoteToken)
+
+       for _, path := range []string{
+               // During the transition to the strongly typed
+               // controller implementation (#14287), workflows and
+               // collections test different code paths.
+               "/arvados/v1/workflows/" + strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1),
+               "/arvados/v1/collections/" + strings.Replace(arvadostest.UserAgreementCollection, "zzzzz-", "zmock-", 1),
+       } {
+               c.Log("testing path ", path)
+               s.remoteMockRequests = nil
+               req := httptest.NewRequest("GET", path, nil)
+               req.Header.Set("Authorization", "Bearer "+remoteToken)
+               s.testRequest(req).Result()
+               c.Assert(s.remoteMockRequests, check.HasLen, 1)
+               pr := s.remoteMockRequests[0]
+               c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer "+remoteToken)
+       }
 }
 
 func (s *FederationSuite) TestWorkflowCRUD(c *check.C) {
@@ -270,7 +295,7 @@ func (s *FederationSuite) TestWorkflowCRUD(c *check.C) {
                req := httptest.NewRequest(method, "/arvados/v1/workflows/"+wf.UUID, strings.NewReader(form.Encode()))
                req.Header.Set("Content-type", "application/x-www-form-urlencoded")
                req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-               resp := s.testRequest(req)
+               resp := s.testRequest(req).Result()
                s.checkResponseOK(c, resp)
                err := json.NewDecoder(resp.Body).Decode(&wf)
                c.Check(err, check.IsNil)
@@ -280,7 +305,7 @@ func (s *FederationSuite) TestWorkflowCRUD(c *check.C) {
        {
                req := httptest.NewRequest("DELETE", "/arvados/v1/workflows/"+wf.UUID, nil)
                req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-               resp := s.testRequest(req)
+               resp := s.testRequest(req).Result()
                s.checkResponseOK(c, resp)
                err := json.NewDecoder(resp.Body).Decode(&wf)
                c.Check(err, check.IsNil)
@@ -288,7 +313,7 @@ func (s *FederationSuite) TestWorkflowCRUD(c *check.C) {
        {
                req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+wf.UUID, nil)
                req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-               resp := s.testRequest(req)
+               resp := s.testRequest(req).Result()
                c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
        }
 }
@@ -322,7 +347,15 @@ func (s *FederationSuite) localServiceHandler(c *check.C, h http.Handler) *https
 
 func (s *FederationSuite) localServiceReturns404(c *check.C) *httpserver.Server {
        return s.localServiceHandler(c, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
-               w.WriteHeader(404)
+               if req.URL.Path == "/arvados/v1/api_client_authorizations/current" {
+                       if req.Header.Get("Authorization") == "Bearer "+arvadostest.ActiveToken {
+                               json.NewEncoder(w).Encode(arvados.APIClientAuthorization{UUID: arvadostest.ActiveTokenUUID, APIToken: arvadostest.ActiveToken})
+                       } else {
+                               w.WriteHeader(http.StatusUnauthorized)
+                       }
+               } else {
+                       w.WriteHeader(404)
+               }
        }))
 }
 
@@ -334,7 +367,7 @@ func (s *FederationSuite) TestGetLocalCollection(c *check.C) {
 
        req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementCollection, nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
 
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        var col arvados.Collection
@@ -351,7 +384,7 @@ func (s *FederationSuite) TestGetLocalCollection(c *check.C) {
        }).Encode()))
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
        req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
-       resp = s.testRequest(req)
+       resp = s.testRequest(req).Result()
 
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        col = arvados.Collection{}
@@ -367,7 +400,7 @@ func (s *FederationSuite) TestGetRemoteCollection(c *check.C) {
 
        req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementCollection, nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        var col arvados.Collection
        c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
@@ -382,7 +415,7 @@ func (s *FederationSuite) TestGetRemoteCollectionError(c *check.C) {
 
        req := httptest.NewRequest("GET", "/arvados/v1/collections/zzzzz-4zz18-fakefakefakefak", nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 }
 
@@ -404,7 +437,7 @@ func (s *FederationSuite) TestGetLocalCollectionByPDH(c *check.C) {
 
        req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
 
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        var col arvados.Collection
@@ -420,7 +453,7 @@ func (s *FederationSuite) TestGetRemoteCollectionByPDH(c *check.C) {
 
        req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
 
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 
@@ -438,7 +471,7 @@ func (s *FederationSuite) TestGetCollectionByPDHError(c *check.C) {
        req := httptest.NewRequest("GET", "/arvados/v1/collections/99999999999999999999999999999999+99", nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        defer resp.Body.Close()
 
        c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
@@ -477,7 +510,7 @@ func (s *FederationSuite) TestGetCollectionByPDHErrorBadHash(c *check.C) {
        req := httptest.NewRequest("GET", "/arvados/v1/collections/99999999999999999999999999999999+99", nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        defer resp.Body.Close()
 
        c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
@@ -488,7 +521,7 @@ func (s *FederationSuite) TestSaltedTokenGetCollectionByPDH(c *check.C) {
 
        req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
        req.Header.Set("Authorization", "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/282d7d172b6cfdce364c5ed12ddf7417b2d00065")
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
 
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        var col arvados.Collection
@@ -504,7 +537,7 @@ func (s *FederationSuite) TestSaltedTokenGetCollectionByPDHError(c *check.C) {
 
        req := httptest.NewRequest("GET", "/arvados/v1/collections/99999999999999999999999999999999+99", nil)
        req.Header.Set("Authorization", "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/282d7d172b6cfdce364c5ed12ddf7417b2d00065")
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
 
        c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 }
@@ -513,7 +546,7 @@ func (s *FederationSuite) TestGetRemoteContainerRequest(c *check.C) {
        defer s.localServiceReturns404(c).Close()
        req := httptest.NewRequest("GET", "/arvados/v1/container_requests/"+arvadostest.QueuedContainerRequestUUID, nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        var cr arvados.ContainerRequest
        c.Check(json.NewDecoder(resp.Body).Decode(&cr), check.IsNil)
@@ -528,7 +561,7 @@ func (s *FederationSuite) TestUpdateRemoteContainerRequest(c *check.C) {
                        strings.NewReader(fmt.Sprintf(`{"container_request": {"priority": %d}}`, pri)))
                req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
                req.Header.Set("Content-type", "application/json")
-               resp := s.testRequest(req)
+               resp := s.testRequest(req).Result()
                c.Check(resp.StatusCode, check.Equals, http.StatusOK)
                var cr arvados.ContainerRequest
                c.Check(json.NewDecoder(resp.Body).Decode(&cr), check.IsNil)
@@ -556,7 +589,7 @@ func (s *FederationSuite) TestCreateRemoteContainerRequest(c *check.C) {
 `))
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
        req.Header.Set("Content-type", "application/json")
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        var cr arvados.ContainerRequest
        c.Check(json.NewDecoder(resp.Body).Decode(&cr), check.IsNil)
@@ -588,7 +621,7 @@ func (s *FederationSuite) TestCreateRemoteContainerRequestCheckRuntimeToken(c *c
        arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
        s.testHandler.Cluster.ClusterID = "zzzzz"
 
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        var cr struct {
                arvados.ContainerRequest `json:"container_request"`
@@ -619,7 +652,7 @@ func (s *FederationSuite) TestCreateRemoteContainerRequestCheckSetRuntimeToken(c
 `))
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
        req.Header.Set("Content-type", "application/json")
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        var cr struct {
                arvados.ContainerRequest `json:"container_request"`
@@ -648,7 +681,7 @@ func (s *FederationSuite) TestCreateRemoteContainerRequestRuntimeTokenFromAuth(c
 `))
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2+"/zzzzz-dz642-parentcontainer")
        req.Header.Set("Content-type", "application/json")
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        var cr struct {
                arvados.ContainerRequest `json:"container_request"`
@@ -674,7 +707,7 @@ func (s *FederationSuite) TestCreateRemoteContainerRequestError(c *check.C) {
 `))
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
        req.Header.Set("Content-type", "application/json")
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 }
 
@@ -683,7 +716,7 @@ func (s *FederationSuite) TestGetRemoteContainer(c *check.C) {
        req := httptest.NewRequest("GET", "/arvados/v1/containers/"+arvadostest.QueuedContainerUUID, nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
        resp := s.testRequest(req)
-       c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+       c.Check(resp.Code, check.Equals, http.StatusOK)
        var cn arvados.Container
        c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
        c.Check(cn.UUID, check.Equals, arvadostest.QueuedContainerUUID)
@@ -694,10 +727,11 @@ func (s *FederationSuite) TestListRemoteContainer(c *check.C) {
        req := httptest.NewRequest("GET", "/arvados/v1/containers?count=none&filters="+
                url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v"]]]`, arvadostest.QueuedContainerUUID)), nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        var cn arvados.ContainerList
        c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
+       c.Assert(cn.Items, check.HasLen, 1)
        c.Check(cn.Items[0].UUID, check.Equals, arvadostest.QueuedContainerUUID)
 }
 
@@ -714,7 +748,7 @@ func (s *FederationSuite) TestListMultiRemoteContainers(c *check.C) {
                url.QueryEscape(`["uuid", "command"]`)),
                nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        var cn arvados.ContainerList
        c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
@@ -737,7 +771,7 @@ func (s *FederationSuite) TestListMultiRemoteContainerError(c *check.C) {
                url.QueryEscape(`["uuid", "command"]`)),
                nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusBadGateway)
        s.checkJSONErrorMatches(c, resp, `error fetching from zhome \(404 Not Found\): EOF`)
 }
@@ -763,7 +797,7 @@ func (s *FederationSuite) TestListMultiRemoteContainersPaged(c *check.C) {
                        arvadostest.QueuedContainerUUID))),
                nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        c.Check(callCount, check.Equals, 2)
        var cn arvados.ContainerList
@@ -799,7 +833,7 @@ func (s *FederationSuite) TestListMultiRemoteContainersMissing(c *check.C) {
                        arvadostest.QueuedContainerUUID))),
                nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        c.Check(callCount, check.Equals, 2)
        var cn arvados.ContainerList
@@ -820,7 +854,7 @@ func (s *FederationSuite) TestListMultiRemoteContainerPageSizeError(c *check.C)
                        arvadostest.QueuedContainerUUID))),
                nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
        s.checkJSONErrorMatches(c, resp, `Federated multi-object request for 2 objects which is more than max page size 1.`)
 }
@@ -831,7 +865,7 @@ func (s *FederationSuite) TestListMultiRemoteContainerLimitError(c *check.C) {
                        arvadostest.QueuedContainerUUID))),
                nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
        s.checkJSONErrorMatches(c, resp, `Federated multi-object may not provide 'limit', 'offset' or 'order'.`)
 }
@@ -842,7 +876,7 @@ func (s *FederationSuite) TestListMultiRemoteContainerOffsetError(c *check.C) {
                        arvadostest.QueuedContainerUUID))),
                nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
        s.checkJSONErrorMatches(c, resp, `Federated multi-object may not provide 'limit', 'offset' or 'order'.`)
 }
@@ -853,7 +887,7 @@ func (s *FederationSuite) TestListMultiRemoteContainerOrderError(c *check.C) {
                        arvadostest.QueuedContainerUUID))),
                nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
        s.checkJSONErrorMatches(c, resp, `Federated multi-object may not provide 'limit', 'offset' or 'order'.`)
 }
@@ -865,7 +899,7 @@ func (s *FederationSuite) TestListMultiRemoteContainerSelectError(c *check.C) {
                url.QueryEscape(`["command"]`)),
                nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
        s.checkJSONErrorMatches(c, resp, `Federated multi-object request must include 'uuid' in 'select'`)
 }