14287: Ensure timestamps in responses have 9 digits of nanoseconds.

[arvados.git] / lib / controller / federation_test.go

diff --git a/lib/controller/federation_test.go b/lib/controller/federation_test.go
index 62916acd2ac10be14d90d4e02e2703e77949e32b..d689bb00526d0dd701ec77316ff07727ca89e461 100644 (file)
--- a/lib/controller/federation_test.go
+++ b/lib/controller/federation_test.go
@@ -39,7 +39,8 @@ type FederationSuite struct {
        // provided by the integration test environment.
        remoteServer *httpserver.Server
        // remoteMock ("zmock") appends each incoming request to
-       // remoteMockRequests, and returns an empty 200 response.
+       // remoteMockRequests, and returns 200 with an empty JSON
+       // object.
        remoteMock         *httpserver.Server
        remoteMockRequests []http.Request
 }
@@ -54,25 +55,21 @@ func (s *FederationSuite) SetUpTest(c *check.C) {
        s.remoteMock.Server.Handler = http.HandlerFunc(s.remoteMockHandler)
        c.Assert(s.remoteMock.Start(), check.IsNil)
 
-       nodeProfile := arvados.NodeProfile{
-               Controller: arvados.SystemServiceInstance{Listen: ":"},
-               RailsAPI:   arvados.SystemServiceInstance{Listen: ":1"}, // local reqs will error "connection refused"
+       cluster := &arvados.Cluster{
+               ClusterID:                 "zhome",
+               PostgreSQL:                integrationTestCluster().PostgreSQL,
+               EnableBetaController14287: enableBetaController14287,
        }
-       s.testHandler = &Handler{Cluster: &arvados.Cluster{
-               ClusterID:  "zhome",
-               PostgreSQL: integrationTestCluster().PostgreSQL,
-               NodeProfiles: map[string]arvados.NodeProfile{
-                       "*": nodeProfile,
-               },
-               RequestLimits: arvados.RequestLimits{
-                       MaxItemsPerResponse:            1000,
-                       MultiClusterRequestConcurrency: 4,
-               },
-       }, NodeProfile: &nodeProfile}
+       cluster.TLS.Insecure = true
+       cluster.API.MaxItemsPerResponse = 1000
+       cluster.API.MaxRequestAmplification = 4
+       arvadostest.SetServiceURL(&cluster.Services.RailsAPI, "http://localhost:1/")
+       arvadostest.SetServiceURL(&cluster.Services.Controller, "http://localhost:/")
+       s.testHandler = &Handler{Cluster: cluster}
        s.testServer = newServerFromIntegrationTestEnv(c)
        s.testServer.Server.Handler = httpserver.AddRequestIDs(httpserver.LogRequests(s.log, s.testHandler))
 
-       s.testHandler.Cluster.RemoteClusters = map[string]arvados.RemoteCluster{
+       cluster.RemoteClusters = map[string]arvados.RemoteCluster{
                "zzzzz": {
                        Host:   s.remoteServer.Addr,
                        Proxy:  true,
@@ -96,6 +93,8 @@ func (s *FederationSuite) remoteMockHandler(w http.ResponseWriter, req *http.Req
        req.Body.Close()
        req.Body = ioutil.NopCloser(b)
        s.remoteMockRequests = append(s.remoteMockRequests, *req)
+       // Repond 200 with a valid JSON object
+       fmt.Fprint(w, "{}")
 }
 
 func (s *FederationSuite) TearDownTest(c *check.C) {
@@ -107,15 +106,15 @@ func (s *FederationSuite) TearDownTest(c *check.C) {
        }
 }
 
-func (s *FederationSuite) testRequest(req *http.Request) *http.Response {
+func (s *FederationSuite) testRequest(req *http.Request) *httptest.ResponseRecorder {
        resp := httptest.NewRecorder()
        s.testServer.Server.Handler.ServeHTTP(resp, req)
-       return resp.Result()
+       return resp
 }
 
 func (s *FederationSuite) TestLocalRequest(c *check.C) {
        req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zhome-", 1), nil)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        s.checkHandledLocally(c, resp)
 }
 
@@ -130,7 +129,7 @@ func (s *FederationSuite) checkHandledLocally(c *check.C, resp *http.Response) {
 
 func (s *FederationSuite) TestNoAuth(c *check.C) {
        req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
        s.checkJSONErrorMatches(c, resp, `Not logged in`)
 }
@@ -138,7 +137,7 @@ func (s *FederationSuite) TestNoAuth(c *check.C) {
 func (s *FederationSuite) TestBadAuth(c *check.C) {
        req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
        req.Header.Set("Authorization", "Bearer aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
        s.checkJSONErrorMatches(c, resp, `Not logged in`)
 }
@@ -146,7 +145,7 @@ func (s *FederationSuite) TestBadAuth(c *check.C) {
 func (s *FederationSuite) TestNoAccess(c *check.C) {
        req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.SpectatorToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
        s.checkJSONErrorMatches(c, resp, `.*not found`)
 }
@@ -154,7 +153,7 @@ func (s *FederationSuite) TestNoAccess(c *check.C) {
 func (s *FederationSuite) TestGetUnknownRemote(c *check.C) {
        req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zz404-", 1), nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
        s.checkJSONErrorMatches(c, resp, `.*no proxy available for cluster zz404`)
 }
@@ -166,7 +165,7 @@ func (s *FederationSuite) TestRemoteError(c *check.C) {
 
        req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusBadGateway)
        s.checkJSONErrorMatches(c, resp, `.*HTTP response to HTTPS client`)
 }
@@ -174,7 +173,7 @@ func (s *FederationSuite) TestRemoteError(c *check.C) {
 func (s *FederationSuite) TestGetRemoteWorkflow(c *check.C) {
        req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        var wf arvados.Workflow
        c.Check(json.NewDecoder(resp.Body).Decode(&wf), check.IsNil)
@@ -185,7 +184,7 @@ func (s *FederationSuite) TestGetRemoteWorkflow(c *check.C) {
 func (s *FederationSuite) TestOptionsMethod(c *check.C) {
        req := httptest.NewRequest("OPTIONS", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
        req.Header.Set("Origin", "https://example.com")
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        body, err := ioutil.ReadAll(resp.Body)
        c.Check(err, check.IsNil)
@@ -201,7 +200,7 @@ func (s *FederationSuite) TestOptionsMethod(c *check.C) {
 
 func (s *FederationSuite) TestRemoteWithTokenInQuery(c *check.C) {
        req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1)+"?api_token="+arvadostest.ActiveToken, nil)
-       s.testRequest(req)
+       s.testRequest(req).Result()
        c.Assert(s.remoteMockRequests, check.HasLen, 1)
        pr := s.remoteMockRequests[0]
        // Token is salted and moved from query to Authorization header.
@@ -210,28 +209,51 @@ func (s *FederationSuite) TestRemoteWithTokenInQuery(c *check.C) {
 }
 
 func (s *FederationSuite) TestLocalTokenSalted(c *check.C) {
-       req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1), nil)
-       req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       s.testRequest(req)
-       c.Assert(s.remoteMockRequests, check.HasLen, 1)
-       pr := s.remoteMockRequests[0]
-       // The salted token here has a "zzzzz-" UUID instead of a
-       // "ztest-" UUID because ztest's local database has the
-       // "zzzzz-" test fixtures. The "secret" part is HMAC(sha1,
-       // arvadostest.ActiveToken, "zmock") = "7fd3...".
-       c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/7fd31b61f39c0e82a4155592163218272cedacdc")
+       defer s.localServiceReturns404(c).Close()
+       for _, path := range []string{
+               // During the transition to the strongly typed
+               // controller implementation (#14287), workflows and
+               // collections test different code paths.
+               "/arvados/v1/workflows/" + strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1),
+               "/arvados/v1/collections/" + strings.Replace(arvadostest.UserAgreementCollection, "zzzzz-", "zmock-", 1),
+       } {
+               c.Log("testing path ", path)
+               s.remoteMockRequests = nil
+               req := httptest.NewRequest("GET", path, nil)
+               req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+               s.testRequest(req).Result()
+               c.Assert(s.remoteMockRequests, check.HasLen, 1)
+               pr := s.remoteMockRequests[0]
+               // The salted token here has a "zzzzz-" UUID instead of a
+               // "ztest-" UUID because ztest's local database has the
+               // "zzzzz-" test fixtures. The "secret" part is HMAC(sha1,
+               // arvadostest.ActiveToken, "zmock") = "7fd3...".
+               c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/7fd31b61f39c0e82a4155592163218272cedacdc")
+       }
 }
 
 func (s *FederationSuite) TestRemoteTokenNotSalted(c *check.C) {
+       defer s.localServiceReturns404(c).Close()
        // remoteToken can be any v1 token that doesn't appear in
        // ztest's local db.
        remoteToken := "abcdef00000000000000000000000000000000000000000000"
-       req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1), nil)
-       req.Header.Set("Authorization", "Bearer "+remoteToken)
-       s.testRequest(req)
-       c.Assert(s.remoteMockRequests, check.HasLen, 1)
-       pr := s.remoteMockRequests[0]
-       c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer "+remoteToken)
+
+       for _, path := range []string{
+               // During the transition to the strongly typed
+               // controller implementation (#14287), workflows and
+               // collections test different code paths.
+               "/arvados/v1/workflows/" + strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1),
+               "/arvados/v1/collections/" + strings.Replace(arvadostest.UserAgreementCollection, "zzzzz-", "zmock-", 1),
+       } {
+               c.Log("testing path ", path)
+               s.remoteMockRequests = nil
+               req := httptest.NewRequest("GET", path, nil)
+               req.Header.Set("Authorization", "Bearer "+remoteToken)
+               s.testRequest(req).Result()
+               c.Assert(s.remoteMockRequests, check.HasLen, 1)
+               pr := s.remoteMockRequests[0]
+               c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer "+remoteToken)
+       }
 }
 
 func (s *FederationSuite) TestWorkflowCRUD(c *check.C) {
@@ -273,7 +295,7 @@ func (s *FederationSuite) TestWorkflowCRUD(c *check.C) {
                req := httptest.NewRequest(method, "/arvados/v1/workflows/"+wf.UUID, strings.NewReader(form.Encode()))
                req.Header.Set("Content-type", "application/x-www-form-urlencoded")
                req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-               resp := s.testRequest(req)
+               resp := s.testRequest(req).Result()
                s.checkResponseOK(c, resp)
                err := json.NewDecoder(resp.Body).Decode(&wf)
                c.Check(err, check.IsNil)
@@ -283,7 +305,7 @@ func (s *FederationSuite) TestWorkflowCRUD(c *check.C) {
        {
                req := httptest.NewRequest("DELETE", "/arvados/v1/workflows/"+wf.UUID, nil)
                req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-               resp := s.testRequest(req)
+               resp := s.testRequest(req).Result()
                s.checkResponseOK(c, resp)
                err := json.NewDecoder(resp.Body).Decode(&wf)
                c.Check(err, check.IsNil)
@@ -291,7 +313,7 @@ func (s *FederationSuite) TestWorkflowCRUD(c *check.C) {
        {
                req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+wf.UUID, nil)
                req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-               resp := s.testRequest(req)
+               resp := s.testRequest(req).Result()
                c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
        }
 }
@@ -318,39 +340,34 @@ func (s *FederationSuite) localServiceHandler(c *check.C, h http.Handler) *https
                        Handler: h,
                },
        }
-
        c.Assert(srv.Start(), check.IsNil)
-
-       np := arvados.NodeProfile{
-               Controller: arvados.SystemServiceInstance{Listen: ":"},
-               RailsAPI: arvados.SystemServiceInstance{Listen: srv.Addr,
-                       TLS: false, Insecure: true}}
-       s.testHandler.Cluster.NodeProfiles["*"] = np
-       s.testHandler.NodeProfile = &np
-
+       arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "http://"+srv.Addr)
        return srv
 }
 
 func (s *FederationSuite) localServiceReturns404(c *check.C) *httpserver.Server {
        return s.localServiceHandler(c, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
-               w.WriteHeader(404)
+               if req.URL.Path == "/arvados/v1/api_client_authorizations/current" {
+                       if req.Header.Get("Authorization") == "Bearer "+arvadostest.ActiveToken {
+                               json.NewEncoder(w).Encode(arvados.APIClientAuthorization{UUID: arvadostest.ActiveTokenUUID, APIToken: arvadostest.ActiveToken})
+                       } else {
+                               w.WriteHeader(http.StatusUnauthorized)
+                       }
+               } else {
+                       w.WriteHeader(404)
+               }
        }))
 }
 
 func (s *FederationSuite) TestGetLocalCollection(c *check.C) {
-       np := arvados.NodeProfile{
-               Controller: arvados.SystemServiceInstance{Listen: ":"},
-               RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"),
-                       TLS: true, Insecure: true}}
        s.testHandler.Cluster.ClusterID = "zzzzz"
-       s.testHandler.Cluster.NodeProfiles["*"] = np
-       s.testHandler.NodeProfile = &np
+       arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
 
        // HTTP GET
 
        req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementCollection, nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
 
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        var col arvados.Collection
@@ -367,7 +384,7 @@ func (s *FederationSuite) TestGetLocalCollection(c *check.C) {
        }).Encode()))
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
        req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
-       resp = s.testRequest(req)
+       resp = s.testRequest(req).Result()
 
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        col = arvados.Collection{}
@@ -383,7 +400,7 @@ func (s *FederationSuite) TestGetRemoteCollection(c *check.C) {
 
        req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementCollection, nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        var col arvados.Collection
        c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
@@ -398,7 +415,7 @@ func (s *FederationSuite) TestGetRemoteCollectionError(c *check.C) {
 
        req := httptest.NewRequest("GET", "/arvados/v1/collections/zzzzz-4zz18-fakefakefakefak", nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 }
 
@@ -416,16 +433,11 @@ func (s *FederationSuite) TestSignedLocatorPattern(c *check.C) {
 }
 
 func (s *FederationSuite) TestGetLocalCollectionByPDH(c *check.C) {
-       np := arvados.NodeProfile{
-               Controller: arvados.SystemServiceInstance{Listen: ":"},
-               RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"),
-                       TLS: true, Insecure: true}}
-       s.testHandler.Cluster.NodeProfiles["*"] = np
-       s.testHandler.NodeProfile = &np
+       arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
 
        req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
 
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        var col arvados.Collection
@@ -441,7 +453,7 @@ func (s *FederationSuite) TestGetRemoteCollectionByPDH(c *check.C) {
 
        req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
 
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 
@@ -459,7 +471,7 @@ func (s *FederationSuite) TestGetCollectionByPDHError(c *check.C) {
        req := httptest.NewRequest("GET", "/arvados/v1/collections/99999999999999999999999999999999+99", nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        defer resp.Body.Close()
 
        c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
@@ -498,23 +510,18 @@ func (s *FederationSuite) TestGetCollectionByPDHErrorBadHash(c *check.C) {
        req := httptest.NewRequest("GET", "/arvados/v1/collections/99999999999999999999999999999999+99", nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        defer resp.Body.Close()
 
        c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 }
 
 func (s *FederationSuite) TestSaltedTokenGetCollectionByPDH(c *check.C) {
-       np := arvados.NodeProfile{
-               Controller: arvados.SystemServiceInstance{Listen: ":"},
-               RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"),
-                       TLS: true, Insecure: true}}
-       s.testHandler.Cluster.NodeProfiles["*"] = np
-       s.testHandler.NodeProfile = &np
+       arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
 
        req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
        req.Header.Set("Authorization", "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/282d7d172b6cfdce364c5ed12ddf7417b2d00065")
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
 
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        var col arvados.Collection
@@ -526,16 +533,11 @@ func (s *FederationSuite) TestSaltedTokenGetCollectionByPDH(c *check.C) {
 }
 
 func (s *FederationSuite) TestSaltedTokenGetCollectionByPDHError(c *check.C) {
-       np := arvados.NodeProfile{
-               Controller: arvados.SystemServiceInstance{Listen: ":"},
-               RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"),
-                       TLS: true, Insecure: true}}
-       s.testHandler.Cluster.NodeProfiles["*"] = np
-       s.testHandler.NodeProfile = &np
+       arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
 
        req := httptest.NewRequest("GET", "/arvados/v1/collections/99999999999999999999999999999999+99", nil)
        req.Header.Set("Authorization", "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/282d7d172b6cfdce364c5ed12ddf7417b2d00065")
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
 
        c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 }
@@ -544,7 +546,7 @@ func (s *FederationSuite) TestGetRemoteContainerRequest(c *check.C) {
        defer s.localServiceReturns404(c).Close()
        req := httptest.NewRequest("GET", "/arvados/v1/container_requests/"+arvadostest.QueuedContainerRequestUUID, nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        var cr arvados.ContainerRequest
        c.Check(json.NewDecoder(resp.Body).Decode(&cr), check.IsNil)
@@ -559,7 +561,7 @@ func (s *FederationSuite) TestUpdateRemoteContainerRequest(c *check.C) {
                        strings.NewReader(fmt.Sprintf(`{"container_request": {"priority": %d}}`, pri)))
                req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
                req.Header.Set("Content-type", "application/json")
-               resp := s.testRequest(req)
+               resp := s.testRequest(req).Result()
                c.Check(resp.StatusCode, check.Equals, http.StatusOK)
                var cr arvados.ContainerRequest
                c.Check(json.NewDecoder(resp.Body).Decode(&cr), check.IsNil)
@@ -587,7 +589,7 @@ func (s *FederationSuite) TestCreateRemoteContainerRequest(c *check.C) {
 `))
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
        req.Header.Set("Content-type", "application/json")
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        var cr arvados.ContainerRequest
        c.Check(json.NewDecoder(resp.Body).Decode(&cr), check.IsNil)
@@ -616,15 +618,10 @@ func (s *FederationSuite) TestCreateRemoteContainerRequestCheckRuntimeToken(c *c
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
        req.Header.Set("Content-type", "application/json")
 
-       np := arvados.NodeProfile{
-               Controller: arvados.SystemServiceInstance{Listen: ":"},
-               RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"),
-                       TLS: true, Insecure: true}}
+       arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
        s.testHandler.Cluster.ClusterID = "zzzzz"
-       s.testHandler.Cluster.NodeProfiles["*"] = np
-       s.testHandler.NodeProfile = &np
 
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        var cr struct {
                arvados.ContainerRequest `json:"container_request"`
@@ -655,7 +652,7 @@ func (s *FederationSuite) TestCreateRemoteContainerRequestCheckSetRuntimeToken(c
 `))
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
        req.Header.Set("Content-type", "application/json")
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        var cr struct {
                arvados.ContainerRequest `json:"container_request"`
@@ -684,7 +681,7 @@ func (s *FederationSuite) TestCreateRemoteContainerRequestRuntimeTokenFromAuth(c
 `))
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2+"/zzzzz-dz642-parentcontainer")
        req.Header.Set("Content-type", "application/json")
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        var cr struct {
                arvados.ContainerRequest `json:"container_request"`
@@ -710,7 +707,7 @@ func (s *FederationSuite) TestCreateRemoteContainerRequestError(c *check.C) {
 `))
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
        req.Header.Set("Content-type", "application/json")
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 }
 
@@ -719,7 +716,7 @@ func (s *FederationSuite) TestGetRemoteContainer(c *check.C) {
        req := httptest.NewRequest("GET", "/arvados/v1/containers/"+arvadostest.QueuedContainerUUID, nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
        resp := s.testRequest(req)
-       c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+       c.Check(resp.Code, check.Equals, http.StatusOK)
        var cn arvados.Container
        c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
        c.Check(cn.UUID, check.Equals, arvadostest.QueuedContainerUUID)
@@ -730,10 +727,11 @@ func (s *FederationSuite) TestListRemoteContainer(c *check.C) {
        req := httptest.NewRequest("GET", "/arvados/v1/containers?count=none&filters="+
                url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v"]]]`, arvadostest.QueuedContainerUUID)), nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        var cn arvados.ContainerList
        c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
+       c.Assert(cn.Items, check.HasLen, 1)
        c.Check(cn.Items[0].UUID, check.Equals, arvadostest.QueuedContainerUUID)
 }
 
@@ -750,7 +748,7 @@ func (s *FederationSuite) TestListMultiRemoteContainers(c *check.C) {
                url.QueryEscape(`["uuid", "command"]`)),
                nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        var cn arvados.ContainerList
        c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
@@ -773,7 +771,7 @@ func (s *FederationSuite) TestListMultiRemoteContainerError(c *check.C) {
                url.QueryEscape(`["uuid", "command"]`)),
                nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusBadGateway)
        s.checkJSONErrorMatches(c, resp, `error fetching from zhome \(404 Not Found\): EOF`)
 }
@@ -799,7 +797,7 @@ func (s *FederationSuite) TestListMultiRemoteContainersPaged(c *check.C) {
                        arvadostest.QueuedContainerUUID))),
                nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        c.Check(callCount, check.Equals, 2)
        var cn arvados.ContainerList
@@ -835,7 +833,7 @@ func (s *FederationSuite) TestListMultiRemoteContainersMissing(c *check.C) {
                        arvadostest.QueuedContainerUUID))),
                nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        c.Check(callCount, check.Equals, 2)
        var cn arvados.ContainerList
@@ -850,13 +848,13 @@ func (s *FederationSuite) TestListMultiRemoteContainersMissing(c *check.C) {
 }
 
 func (s *FederationSuite) TestListMultiRemoteContainerPageSizeError(c *check.C) {
-       s.testHandler.Cluster.RequestLimits.MaxItemsPerResponse = 1
+       s.testHandler.Cluster.API.MaxItemsPerResponse = 1
        req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s",
                url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
                        arvadostest.QueuedContainerUUID))),
                nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
        s.checkJSONErrorMatches(c, resp, `Federated multi-object request for 2 objects which is more than max page size 1.`)
 }
@@ -867,7 +865,7 @@ func (s *FederationSuite) TestListMultiRemoteContainerLimitError(c *check.C) {
                        arvadostest.QueuedContainerUUID))),
                nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
        s.checkJSONErrorMatches(c, resp, `Federated multi-object may not provide 'limit', 'offset' or 'order'.`)
 }
@@ -878,7 +876,7 @@ func (s *FederationSuite) TestListMultiRemoteContainerOffsetError(c *check.C) {
                        arvadostest.QueuedContainerUUID))),
                nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
        s.checkJSONErrorMatches(c, resp, `Federated multi-object may not provide 'limit', 'offset' or 'order'.`)
 }
@@ -889,7 +887,7 @@ func (s *FederationSuite) TestListMultiRemoteContainerOrderError(c *check.C) {
                        arvadostest.QueuedContainerUUID))),
                nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
        s.checkJSONErrorMatches(c, resp, `Federated multi-object may not provide 'limit', 'offset' or 'order'.`)
 }
@@ -901,7 +899,7 @@ func (s *FederationSuite) TestListMultiRemoteContainerSelectError(c *check.C) {
                url.QueryEscape(`["command"]`)),
                nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
        s.checkJSONErrorMatches(c, resp, `Federated multi-object request must include 'uuid' in 'select'`)
 }