Merge branch 'patch-1' of https://github.com/mr-c/arvados into mr-c-patch-1

[arvados.git] / lib / controller / federation_test.go

diff --git a/lib/controller/federation_test.go b/lib/controller/federation_test.go
index 62916acd2ac10be14d90d4e02e2703e77949e32b..6a9ad8c15f3db2132bf5c122d8ae639764dbfff7 100644 (file)
--- a/lib/controller/federation_test.go
+++ b/lib/controller/federation_test.go
@@ -6,6 +6,7 @@ package controller
 
 import (
        "bytes"
+       "context"
        "encoding/json"
        "fmt"
        "io"
@@ -17,11 +18,11 @@ import (
        "strings"
        "time"
 
-       "git.curoverse.com/arvados.git/sdk/go/arvados"
-       "git.curoverse.com/arvados.git/sdk/go/arvadostest"
-       "git.curoverse.com/arvados.git/sdk/go/ctxlog"
-       "git.curoverse.com/arvados.git/sdk/go/httpserver"
-       "git.curoverse.com/arvados.git/sdk/go/keepclient"
+       "git.arvados.org/arvados.git/sdk/go/arvados"
+       "git.arvados.org/arvados.git/sdk/go/arvadostest"
+       "git.arvados.org/arvados.git/sdk/go/ctxlog"
+       "git.arvados.org/arvados.git/sdk/go/httpserver"
+       "git.arvados.org/arvados.git/sdk/go/keepclient"
        "github.com/sirupsen/logrus"
        check "gopkg.in/check.v1"
 )
@@ -39,7 +40,8 @@ type FederationSuite struct {
        // provided by the integration test environment.
        remoteServer *httpserver.Server
        // remoteMock ("zmock") appends each incoming request to
-       // remoteMockRequests, and returns an empty 200 response.
+       // remoteMockRequests, and returns 200 with an empty JSON
+       // object.
        remoteMock         *httpserver.Server
        remoteMockRequests []http.Request
 }
@@ -54,25 +56,24 @@ func (s *FederationSuite) SetUpTest(c *check.C) {
        s.remoteMock.Server.Handler = http.HandlerFunc(s.remoteMockHandler)
        c.Assert(s.remoteMock.Start(), check.IsNil)
 
-       nodeProfile := arvados.NodeProfile{
-               Controller: arvados.SystemServiceInstance{Listen: ":"},
-               RailsAPI:   arvados.SystemServiceInstance{Listen: ":1"}, // local reqs will error "connection refused"
+       cluster := &arvados.Cluster{
+               ClusterID:        "zhome",
+               PostgreSQL:       integrationTestCluster().PostgreSQL,
+               ForceLegacyAPI14: forceLegacyAPI14,
        }
-       s.testHandler = &Handler{Cluster: &arvados.Cluster{
-               ClusterID:  "zhome",
-               PostgreSQL: integrationTestCluster().PostgreSQL,
-               NodeProfiles: map[string]arvados.NodeProfile{
-                       "*": nodeProfile,
-               },
-               RequestLimits: arvados.RequestLimits{
-                       MaxItemsPerResponse:            1000,
-                       MultiClusterRequestConcurrency: 4,
-               },
-       }, NodeProfile: &nodeProfile}
+       cluster.TLS.Insecure = true
+       cluster.API.MaxItemsPerResponse = 1000
+       cluster.API.MaxRequestAmplification = 4
+       cluster.API.RequestTimeout = arvados.Duration(5 * time.Minute)
+       arvadostest.SetServiceURL(&cluster.Services.RailsAPI, "http://localhost:1/")
+       arvadostest.SetServiceURL(&cluster.Services.Controller, "http://localhost:/")
+       s.testHandler = &Handler{Cluster: cluster}
        s.testServer = newServerFromIntegrationTestEnv(c)
-       s.testServer.Server.Handler = httpserver.AddRequestIDs(httpserver.LogRequests(s.log, s.testHandler))
+       s.testServer.Server.Handler = httpserver.HandlerWithContext(
+               ctxlog.Context(context.Background(), s.log),
+               httpserver.AddRequestIDs(httpserver.LogRequests(s.testHandler)))
 
-       s.testHandler.Cluster.RemoteClusters = map[string]arvados.RemoteCluster{
+       cluster.RemoteClusters = map[string]arvados.RemoteCluster{
                "zzzzz": {
                        Host:   s.remoteServer.Addr,
                        Proxy:  true,
@@ -83,6 +84,9 @@ func (s *FederationSuite) SetUpTest(c *check.C) {
                        Proxy:  true,
                        Scheme: "http",
                },
+               "*": {
+                       Scheme: "https",
+               },
        }
 
        c.Assert(s.testServer.Start(), check.IsNil)
@@ -96,6 +100,8 @@ func (s *FederationSuite) remoteMockHandler(w http.ResponseWriter, req *http.Req
        req.Body.Close()
        req.Body = ioutil.NopCloser(b)
        s.remoteMockRequests = append(s.remoteMockRequests, *req)
+       // Repond 200 with a valid JSON object
+       fmt.Fprint(w, "{}")
 }
 
 func (s *FederationSuite) TearDownTest(c *check.C) {
@@ -107,15 +113,15 @@ func (s *FederationSuite) TearDownTest(c *check.C) {
        }
 }
 
-func (s *FederationSuite) testRequest(req *http.Request) *http.Response {
+func (s *FederationSuite) testRequest(req *http.Request) *httptest.ResponseRecorder {
        resp := httptest.NewRecorder()
        s.testServer.Server.Handler.ServeHTTP(resp, req)
-       return resp.Result()
+       return resp
 }
 
 func (s *FederationSuite) TestLocalRequest(c *check.C) {
        req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zhome-", 1), nil)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        s.checkHandledLocally(c, resp)
 }
 
@@ -130,31 +136,31 @@ func (s *FederationSuite) checkHandledLocally(c *check.C, resp *http.Response) {
 
 func (s *FederationSuite) TestNoAuth(c *check.C) {
        req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
-       s.checkJSONErrorMatches(c, resp, `Not logged in`)
+       s.checkJSONErrorMatches(c, resp, `Not logged in.*`)
 }
 
 func (s *FederationSuite) TestBadAuth(c *check.C) {
        req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
        req.Header.Set("Authorization", "Bearer aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
-       s.checkJSONErrorMatches(c, resp, `Not logged in`)
+       s.checkJSONErrorMatches(c, resp, `Not logged in.*`)
 }
 
 func (s *FederationSuite) TestNoAccess(c *check.C) {
        req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.SpectatorToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
-       s.checkJSONErrorMatches(c, resp, `.*not found`)
+       s.checkJSONErrorMatches(c, resp, `.*not found.*`)
 }
 
 func (s *FederationSuite) TestGetUnknownRemote(c *check.C) {
        req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zz404-", 1), nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
        s.checkJSONErrorMatches(c, resp, `.*no proxy available for cluster zz404`)
 }
@@ -166,7 +172,7 @@ func (s *FederationSuite) TestRemoteError(c *check.C) {
 
        req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusBadGateway)
        s.checkJSONErrorMatches(c, resp, `.*HTTP response to HTTPS client`)
 }
@@ -174,7 +180,7 @@ func (s *FederationSuite) TestRemoteError(c *check.C) {
 func (s *FederationSuite) TestGetRemoteWorkflow(c *check.C) {
        req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        var wf arvados.Workflow
        c.Check(json.NewDecoder(resp.Body).Decode(&wf), check.IsNil)
@@ -185,7 +191,7 @@ func (s *FederationSuite) TestGetRemoteWorkflow(c *check.C) {
 func (s *FederationSuite) TestOptionsMethod(c *check.C) {
        req := httptest.NewRequest("OPTIONS", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
        req.Header.Set("Origin", "https://example.com")
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        body, err := ioutil.ReadAll(resp.Body)
        c.Check(err, check.IsNil)
@@ -201,7 +207,7 @@ func (s *FederationSuite) TestOptionsMethod(c *check.C) {
 
 func (s *FederationSuite) TestRemoteWithTokenInQuery(c *check.C) {
        req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1)+"?api_token="+arvadostest.ActiveToken, nil)
-       s.testRequest(req)
+       s.testRequest(req).Result()
        c.Assert(s.remoteMockRequests, check.HasLen, 1)
        pr := s.remoteMockRequests[0]
        // Token is salted and moved from query to Authorization header.
@@ -210,39 +216,58 @@ func (s *FederationSuite) TestRemoteWithTokenInQuery(c *check.C) {
 }
 
 func (s *FederationSuite) TestLocalTokenSalted(c *check.C) {
-       req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1), nil)
-       req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       s.testRequest(req)
-       c.Assert(s.remoteMockRequests, check.HasLen, 1)
-       pr := s.remoteMockRequests[0]
-       // The salted token here has a "zzzzz-" UUID instead of a
-       // "ztest-" UUID because ztest's local database has the
-       // "zzzzz-" test fixtures. The "secret" part is HMAC(sha1,
-       // arvadostest.ActiveToken, "zmock") = "7fd3...".
-       c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/7fd31b61f39c0e82a4155592163218272cedacdc")
+       defer s.localServiceReturns404(c).Close()
+       for _, path := range []string{
+               // During the transition to the strongly typed
+               // controller implementation (#14287), workflows and
+               // collections test different code paths.
+               "/arvados/v1/workflows/" + strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1),
+               "/arvados/v1/collections/" + strings.Replace(arvadostest.UserAgreementCollection, "zzzzz-", "zmock-", 1),
+       } {
+               c.Log("testing path ", path)
+               s.remoteMockRequests = nil
+               req := httptest.NewRequest("GET", path, nil)
+               req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+               s.testRequest(req).Result()
+               c.Assert(s.remoteMockRequests, check.HasLen, 1)
+               pr := s.remoteMockRequests[0]
+               // The salted token here has a "zzzzz-" UUID instead of a
+               // "ztest-" UUID because ztest's local database has the
+               // "zzzzz-" test fixtures. The "secret" part is HMAC(sha1,
+               // arvadostest.ActiveToken, "zmock") = "7fd3...".
+               c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/7fd31b61f39c0e82a4155592163218272cedacdc")
+       }
 }
 
 func (s *FederationSuite) TestRemoteTokenNotSalted(c *check.C) {
+       defer s.localServiceReturns404(c).Close()
        // remoteToken can be any v1 token that doesn't appear in
        // ztest's local db.
        remoteToken := "abcdef00000000000000000000000000000000000000000000"
-       req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1), nil)
-       req.Header.Set("Authorization", "Bearer "+remoteToken)
-       s.testRequest(req)
-       c.Assert(s.remoteMockRequests, check.HasLen, 1)
-       pr := s.remoteMockRequests[0]
-       c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer "+remoteToken)
+
+       for _, path := range []string{
+               // During the transition to the strongly typed
+               // controller implementation (#14287), workflows and
+               // collections test different code paths.
+               "/arvados/v1/workflows/" + strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1),
+               "/arvados/v1/collections/" + strings.Replace(arvadostest.UserAgreementCollection, "zzzzz-", "zmock-", 1),
+       } {
+               c.Log("testing path ", path)
+               s.remoteMockRequests = nil
+               req := httptest.NewRequest("GET", path, nil)
+               req.Header.Set("Authorization", "Bearer "+remoteToken)
+               s.testRequest(req).Result()
+               c.Assert(s.remoteMockRequests, check.HasLen, 1)
+               pr := s.remoteMockRequests[0]
+               c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer "+remoteToken)
+       }
 }
 
 func (s *FederationSuite) TestWorkflowCRUD(c *check.C) {
-       wf := arvados.Workflow{
-               Description: "TestCRUD",
-       }
+       var wf arvados.Workflow
        {
-               body := &strings.Builder{}
-               json.NewEncoder(body).Encode(&wf)
                req := httptest.NewRequest("POST", "/arvados/v1/workflows", strings.NewReader(url.Values{
-                       "workflow": {body.String()},
+                       "workflow": {`{"description": "TestCRUD"}`},
                }.Encode()))
                req.Header.Set("Content-type", "application/x-www-form-urlencoded")
                req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
@@ -273,7 +298,7 @@ func (s *FederationSuite) TestWorkflowCRUD(c *check.C) {
                req := httptest.NewRequest(method, "/arvados/v1/workflows/"+wf.UUID, strings.NewReader(form.Encode()))
                req.Header.Set("Content-type", "application/x-www-form-urlencoded")
                req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-               resp := s.testRequest(req)
+               resp := s.testRequest(req).Result()
                s.checkResponseOK(c, resp)
                err := json.NewDecoder(resp.Body).Decode(&wf)
                c.Check(err, check.IsNil)
@@ -283,7 +308,7 @@ func (s *FederationSuite) TestWorkflowCRUD(c *check.C) {
        {
                req := httptest.NewRequest("DELETE", "/arvados/v1/workflows/"+wf.UUID, nil)
                req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-               resp := s.testRequest(req)
+               resp := s.testRequest(req).Result()
                s.checkResponseOK(c, resp)
                err := json.NewDecoder(resp.Body).Decode(&wf)
                c.Check(err, check.IsNil)
@@ -291,7 +316,7 @@ func (s *FederationSuite) TestWorkflowCRUD(c *check.C) {
        {
                req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+wf.UUID, nil)
                req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-               resp := s.testRequest(req)
+               resp := s.testRequest(req).Result()
                c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
        }
 }
@@ -318,39 +343,34 @@ func (s *FederationSuite) localServiceHandler(c *check.C, h http.Handler) *https
                        Handler: h,
                },
        }
-
        c.Assert(srv.Start(), check.IsNil)
-
-       np := arvados.NodeProfile{
-               Controller: arvados.SystemServiceInstance{Listen: ":"},
-               RailsAPI: arvados.SystemServiceInstance{Listen: srv.Addr,
-                       TLS: false, Insecure: true}}
-       s.testHandler.Cluster.NodeProfiles["*"] = np
-       s.testHandler.NodeProfile = &np
-
+       arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "http://"+srv.Addr)
        return srv
 }
 
 func (s *FederationSuite) localServiceReturns404(c *check.C) *httpserver.Server {
        return s.localServiceHandler(c, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
-               w.WriteHeader(404)
+               if req.URL.Path == "/arvados/v1/api_client_authorizations/current" {
+                       if req.Header.Get("Authorization") == "Bearer "+arvadostest.ActiveToken {
+                               json.NewEncoder(w).Encode(arvados.APIClientAuthorization{UUID: arvadostest.ActiveTokenUUID, APIToken: arvadostest.ActiveToken})
+                       } else {
+                               w.WriteHeader(http.StatusUnauthorized)
+                       }
+               } else {
+                       w.WriteHeader(404)
+               }
        }))
 }
 
 func (s *FederationSuite) TestGetLocalCollection(c *check.C) {
-       np := arvados.NodeProfile{
-               Controller: arvados.SystemServiceInstance{Listen: ":"},
-               RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"),
-                       TLS: true, Insecure: true}}
        s.testHandler.Cluster.ClusterID = "zzzzz"
-       s.testHandler.Cluster.NodeProfiles["*"] = np
-       s.testHandler.NodeProfile = &np
+       arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
 
        // HTTP GET
 
        req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementCollection, nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
 
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        var col arvados.Collection
@@ -367,7 +387,7 @@ func (s *FederationSuite) TestGetLocalCollection(c *check.C) {
        }).Encode()))
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
        req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
-       resp = s.testRequest(req)
+       resp = s.testRequest(req).Result()
 
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        col = arvados.Collection{}
@@ -383,7 +403,7 @@ func (s *FederationSuite) TestGetRemoteCollection(c *check.C) {
 
        req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementCollection, nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        var col arvados.Collection
        c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
@@ -398,7 +418,7 @@ func (s *FederationSuite) TestGetRemoteCollectionError(c *check.C) {
 
        req := httptest.NewRequest("GET", "/arvados/v1/collections/zzzzz-4zz18-fakefakefakefak", nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 }
 
@@ -416,16 +436,11 @@ func (s *FederationSuite) TestSignedLocatorPattern(c *check.C) {
 }
 
 func (s *FederationSuite) TestGetLocalCollectionByPDH(c *check.C) {
-       np := arvados.NodeProfile{
-               Controller: arvados.SystemServiceInstance{Listen: ":"},
-               RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"),
-                       TLS: true, Insecure: true}}
-       s.testHandler.Cluster.NodeProfiles["*"] = np
-       s.testHandler.NodeProfile = &np
+       arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
 
        req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
 
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        var col arvados.Collection
@@ -441,7 +456,7 @@ func (s *FederationSuite) TestGetRemoteCollectionByPDH(c *check.C) {
 
        req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
 
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 
@@ -456,10 +471,14 @@ func (s *FederationSuite) TestGetRemoteCollectionByPDH(c *check.C) {
 func (s *FederationSuite) TestGetCollectionByPDHError(c *check.C) {
        defer s.localServiceReturns404(c).Close()
 
+       // zmock's normal response (200 with an empty body) would
+       // change the outcome from 404 to 502
+       delete(s.testHandler.Cluster.RemoteClusters, "zmock")
+
        req := httptest.NewRequest("GET", "/arvados/v1/collections/99999999999999999999999999999999+99", nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        defer resp.Body.Close()
 
        c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
@@ -468,6 +487,10 @@ func (s *FederationSuite) TestGetCollectionByPDHError(c *check.C) {
 func (s *FederationSuite) TestGetCollectionByPDHErrorBadHash(c *check.C) {
        defer s.localServiceReturns404(c).Close()
 
+       // zmock's normal response (200 with an empty body) would
+       // change the outcome
+       delete(s.testHandler.Cluster.RemoteClusters, "zmock")
+
        srv2 := &httpserver.Server{
                Server: http.Server{
                        Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
@@ -498,23 +521,18 @@ func (s *FederationSuite) TestGetCollectionByPDHErrorBadHash(c *check.C) {
        req := httptest.NewRequest("GET", "/arvados/v1/collections/99999999999999999999999999999999+99", nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        defer resp.Body.Close()
 
-       c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
+       c.Check(resp.StatusCode, check.Equals, http.StatusBadGateway)
 }
 
 func (s *FederationSuite) TestSaltedTokenGetCollectionByPDH(c *check.C) {
-       np := arvados.NodeProfile{
-               Controller: arvados.SystemServiceInstance{Listen: ":"},
-               RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"),
-                       TLS: true, Insecure: true}}
-       s.testHandler.Cluster.NodeProfiles["*"] = np
-       s.testHandler.NodeProfile = &np
+       arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
 
        req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
        req.Header.Set("Authorization", "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/282d7d172b6cfdce364c5ed12ddf7417b2d00065")
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
 
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        var col arvados.Collection
@@ -526,16 +544,15 @@ func (s *FederationSuite) TestSaltedTokenGetCollectionByPDH(c *check.C) {
 }
 
 func (s *FederationSuite) TestSaltedTokenGetCollectionByPDHError(c *check.C) {
-       np := arvados.NodeProfile{
-               Controller: arvados.SystemServiceInstance{Listen: ":"},
-               RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"),
-                       TLS: true, Insecure: true}}
-       s.testHandler.Cluster.NodeProfiles["*"] = np
-       s.testHandler.NodeProfile = &np
+       arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
+
+       // zmock's normal response (200 with an empty body) would
+       // change the outcome
+       delete(s.testHandler.Cluster.RemoteClusters, "zmock")
 
        req := httptest.NewRequest("GET", "/arvados/v1/collections/99999999999999999999999999999999+99", nil)
        req.Header.Set("Authorization", "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/282d7d172b6cfdce364c5ed12ddf7417b2d00065")
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
 
        c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 }
@@ -544,7 +561,7 @@ func (s *FederationSuite) TestGetRemoteContainerRequest(c *check.C) {
        defer s.localServiceReturns404(c).Close()
        req := httptest.NewRequest("GET", "/arvados/v1/container_requests/"+arvadostest.QueuedContainerRequestUUID, nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        var cr arvados.ContainerRequest
        c.Check(json.NewDecoder(resp.Body).Decode(&cr), check.IsNil)
@@ -559,7 +576,7 @@ func (s *FederationSuite) TestUpdateRemoteContainerRequest(c *check.C) {
                        strings.NewReader(fmt.Sprintf(`{"container_request": {"priority": %d}}`, pri)))
                req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
                req.Header.Set("Content-type", "application/json")
-               resp := s.testRequest(req)
+               resp := s.testRequest(req).Result()
                c.Check(resp.StatusCode, check.Equals, http.StatusOK)
                var cr arvados.ContainerRequest
                c.Check(json.NewDecoder(resp.Body).Decode(&cr), check.IsNil)
@@ -570,6 +587,21 @@ func (s *FederationSuite) TestUpdateRemoteContainerRequest(c *check.C) {
        setPri(1) // Reset fixture so side effect doesn't break other tests.
 }
 
+func (s *FederationSuite) TestCreateContainerRequestBadToken(c *check.C) {
+       defer s.localServiceReturns404(c).Close()
+       // pass cluster_id via query parameter, this allows arvados-controller
+       // to avoid parsing the body
+       req := httptest.NewRequest("POST", "/arvados/v1/container_requests?cluster_id=zzzzz",
+               strings.NewReader(`{"container_request":{}}`))
+       req.Header.Set("Authorization", "Bearer abcdefg")
+       req.Header.Set("Content-type", "application/json")
+       resp := s.testRequest(req).Result()
+       c.Check(resp.StatusCode, check.Equals, http.StatusForbidden)
+       var e map[string][]string
+       c.Check(json.NewDecoder(resp.Body).Decode(&e), check.IsNil)
+       c.Check(e["errors"], check.DeepEquals, []string{"invalid API token"})
+}
+
 func (s *FederationSuite) TestCreateRemoteContainerRequest(c *check.C) {
        defer s.localServiceReturns404(c).Close()
        // pass cluster_id via query parameter, this allows arvados-controller
@@ -587,7 +619,7 @@ func (s *FederationSuite) TestCreateRemoteContainerRequest(c *check.C) {
 `))
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
        req.Header.Set("Content-type", "application/json")
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        var cr arvados.ContainerRequest
        c.Check(json.NewDecoder(resp.Body).Decode(&cr), check.IsNil)
@@ -616,15 +648,10 @@ func (s *FederationSuite) TestCreateRemoteContainerRequestCheckRuntimeToken(c *c
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
        req.Header.Set("Content-type", "application/json")
 
-       np := arvados.NodeProfile{
-               Controller: arvados.SystemServiceInstance{Listen: ":"},
-               RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"),
-                       TLS: true, Insecure: true}}
+       arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
        s.testHandler.Cluster.ClusterID = "zzzzz"
-       s.testHandler.Cluster.NodeProfiles["*"] = np
-       s.testHandler.NodeProfile = &np
 
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        var cr struct {
                arvados.ContainerRequest `json:"container_request"`
@@ -655,7 +682,7 @@ func (s *FederationSuite) TestCreateRemoteContainerRequestCheckSetRuntimeToken(c
 `))
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
        req.Header.Set("Content-type", "application/json")
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        var cr struct {
                arvados.ContainerRequest `json:"container_request"`
@@ -684,7 +711,7 @@ func (s *FederationSuite) TestCreateRemoteContainerRequestRuntimeTokenFromAuth(c
 `))
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2+"/zzzzz-dz642-parentcontainer")
        req.Header.Set("Content-type", "application/json")
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        var cr struct {
                arvados.ContainerRequest `json:"container_request"`
@@ -710,7 +737,7 @@ func (s *FederationSuite) TestCreateRemoteContainerRequestError(c *check.C) {
 `))
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
        req.Header.Set("Content-type", "application/json")
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 }
 
@@ -719,7 +746,7 @@ func (s *FederationSuite) TestGetRemoteContainer(c *check.C) {
        req := httptest.NewRequest("GET", "/arvados/v1/containers/"+arvadostest.QueuedContainerUUID, nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
        resp := s.testRequest(req)
-       c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+       c.Check(resp.Code, check.Equals, http.StatusOK)
        var cn arvados.Container
        c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
        c.Check(cn.UUID, check.Equals, arvadostest.QueuedContainerUUID)
@@ -730,10 +757,11 @@ func (s *FederationSuite) TestListRemoteContainer(c *check.C) {
        req := httptest.NewRequest("GET", "/arvados/v1/containers?count=none&filters="+
                url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v"]]]`, arvadostest.QueuedContainerUUID)), nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        var cn arvados.ContainerList
        c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
+       c.Assert(cn.Items, check.HasLen, 1)
        c.Check(cn.Items[0].UUID, check.Equals, arvadostest.QueuedContainerUUID)
 }
 
@@ -750,7 +778,7 @@ func (s *FederationSuite) TestListMultiRemoteContainers(c *check.C) {
                url.QueryEscape(`["uuid", "command"]`)),
                nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        var cn arvados.ContainerList
        c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
@@ -773,7 +801,7 @@ func (s *FederationSuite) TestListMultiRemoteContainerError(c *check.C) {
                url.QueryEscape(`["uuid", "command"]`)),
                nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusBadGateway)
        s.checkJSONErrorMatches(c, resp, `error fetching from zhome \(404 Not Found\): EOF`)
 }
@@ -799,7 +827,7 @@ func (s *FederationSuite) TestListMultiRemoteContainersPaged(c *check.C) {
                        arvadostest.QueuedContainerUUID))),
                nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        c.Check(callCount, check.Equals, 2)
        var cn arvados.ContainerList
@@ -835,7 +863,7 @@ func (s *FederationSuite) TestListMultiRemoteContainersMissing(c *check.C) {
                        arvadostest.QueuedContainerUUID))),
                nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        c.Check(callCount, check.Equals, 2)
        var cn arvados.ContainerList
@@ -850,13 +878,13 @@ func (s *FederationSuite) TestListMultiRemoteContainersMissing(c *check.C) {
 }
 
 func (s *FederationSuite) TestListMultiRemoteContainerPageSizeError(c *check.C) {
-       s.testHandler.Cluster.RequestLimits.MaxItemsPerResponse = 1
+       s.testHandler.Cluster.API.MaxItemsPerResponse = 1
        req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s",
                url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
                        arvadostest.QueuedContainerUUID))),
                nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
        s.checkJSONErrorMatches(c, resp, `Federated multi-object request for 2 objects which is more than max page size 1.`)
 }
@@ -867,7 +895,7 @@ func (s *FederationSuite) TestListMultiRemoteContainerLimitError(c *check.C) {
                        arvadostest.QueuedContainerUUID))),
                nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
        s.checkJSONErrorMatches(c, resp, `Federated multi-object may not provide 'limit', 'offset' or 'order'.`)
 }
@@ -878,7 +906,7 @@ func (s *FederationSuite) TestListMultiRemoteContainerOffsetError(c *check.C) {
                        arvadostest.QueuedContainerUUID))),
                nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
        s.checkJSONErrorMatches(c, resp, `Federated multi-object may not provide 'limit', 'offset' or 'order'.`)
 }
@@ -889,7 +917,7 @@ func (s *FederationSuite) TestListMultiRemoteContainerOrderError(c *check.C) {
                        arvadostest.QueuedContainerUUID))),
                nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
        s.checkJSONErrorMatches(c, resp, `Federated multi-object may not provide 'limit', 'offset' or 'order'.`)
 }
@@ -901,7 +929,7 @@ func (s *FederationSuite) TestListMultiRemoteContainerSelectError(c *check.C) {
                url.QueryEscape(`["command"]`)),
                nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
-       resp := s.testRequest(req)
+       resp := s.testRequest(req).Result()
        c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
        s.checkJSONErrorMatches(c, resp, `Federated multi-object request must include 'uuid' in 'select'`)
 }