13619: Code cleanups
[arvados.git] / lib / controller / federation_test.go
index 52906ead326fd4dad2cd9eda3bf1f7e80a2e7c95..1b099be5e002f55e2f1be9ce18fe6a5e9535d1fa 100644 (file)
@@ -6,6 +6,7 @@ package controller
 
 import (
        "encoding/json"
+       "fmt"
        "io/ioutil"
        "net/http"
        "net/http/httptest"
@@ -62,6 +63,8 @@ func (s *FederationSuite) SetUpTest(c *check.C) {
                NodeProfiles: map[string]arvados.NodeProfile{
                        "*": nodeProfile,
                },
+               MaxItemsPerResponse:         1000,
+               FederatedRequestConcurrency: 4,
        }, NodeProfile: &nodeProfile}
        s.testServer = newServerFromIntegrationTestEnv(c)
        s.testServer.Server.Handler = httpserver.AddRequestIDs(httpserver.LogRequests(s.log, s.testHandler))
@@ -192,7 +195,7 @@ func (s *FederationSuite) TestOptionsMethod(c *check.C) {
 func (s *FederationSuite) TestRemoteWithTokenInQuery(c *check.C) {
        req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1)+"?api_token="+arvadostest.ActiveToken, nil)
        s.testRequest(req)
-       c.Assert(len(s.remoteMockRequests), check.Equals, 1)
+       c.Assert(s.remoteMockRequests, check.HasLen, 1)
        pr := s.remoteMockRequests[0]
        // Token is salted and moved from query to Authorization header.
        c.Check(pr.URL.String(), check.Not(check.Matches), `.*api_token=.*`)
@@ -203,7 +206,7 @@ func (s *FederationSuite) TestLocalTokenSalted(c *check.C) {
        req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1), nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
        s.testRequest(req)
-       c.Assert(len(s.remoteMockRequests), check.Equals, 1)
+       c.Assert(s.remoteMockRequests, check.HasLen, 1)
        pr := s.remoteMockRequests[0]
        // The salted token here has a "zzzzz-" UUID instead of a
        // "ztest-" UUID because ztest's local database has the
@@ -219,7 +222,7 @@ func (s *FederationSuite) TestRemoteTokenNotSalted(c *check.C) {
        req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1), nil)
        req.Header.Set("Authorization", "Bearer "+remoteToken)
        s.testRequest(req)
-       c.Assert(len(s.remoteMockRequests), check.Equals, 1)
+       c.Assert(s.remoteMockRequests, check.HasLen, 1)
        pr := s.remoteMockRequests[0]
        c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer "+remoteToken)
 }
@@ -298,10 +301,35 @@ func (s *FederationSuite) checkJSONErrorMatches(c *check.C, resp *http.Response,
        var jresp httpserver.ErrorResponse
        err := json.NewDecoder(resp.Body).Decode(&jresp)
        c.Check(err, check.IsNil)
-       c.Assert(len(jresp.Errors), check.Equals, 1)
+       c.Assert(jresp.Errors, check.HasLen, 1)
        c.Check(jresp.Errors[0], check.Matches, re)
 }
 
+func (s *FederationSuite) localServiceHandler(c *check.C, h http.Handler) *httpserver.Server {
+       srv := &httpserver.Server{
+               Server: http.Server{
+                       Handler: h,
+               },
+       }
+
+       c.Assert(srv.Start(), check.IsNil)
+
+       np := arvados.NodeProfile{
+               Controller: arvados.SystemServiceInstance{Listen: ":"},
+               RailsAPI: arvados.SystemServiceInstance{Listen: srv.Addr,
+                       TLS: false, Insecure: true}}
+       s.testHandler.Cluster.NodeProfiles["*"] = np
+       s.testHandler.NodeProfile = &np
+
+       return srv
+}
+
+func (s *FederationSuite) localServiceReturns404(c *check.C) *httpserver.Server {
+       return s.localServiceHandler(c, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+               w.WriteHeader(404)
+       }))
+}
+
 func (s *FederationSuite) TestGetLocalCollection(c *check.C) {
        np := arvados.NodeProfile{
                Controller: arvados.SystemServiceInstance{Listen: ":"},
@@ -325,6 +353,8 @@ func (s *FederationSuite) TestGetLocalCollection(c *check.C) {
 }
 
 func (s *FederationSuite) TestGetRemoteCollection(c *check.C) {
+       defer s.localServiceReturns404(c).Close()
+
        req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementCollection, nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
        resp := s.testRequest(req)
@@ -338,6 +368,8 @@ func (s *FederationSuite) TestGetRemoteCollection(c *check.C) {
 }
 
 func (s *FederationSuite) TestGetRemoteCollectionError(c *check.C) {
+       defer s.localServiceReturns404(c).Close()
+
        req := httptest.NewRequest("GET", "/arvados/v1/collections/zzzzz-4zz18-fakefakefakefak", nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
        resp := s.testRequest(req)
@@ -379,29 +411,14 @@ func (s *FederationSuite) TestGetLocalCollectionByPDH(c *check.C) {
 }
 
 func (s *FederationSuite) TestGetRemoteCollectionByPDH(c *check.C) {
-       srv := &httpserver.Server{
-               Server: http.Server{
-                       Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
-                               w.WriteHeader(404)
-                       }),
-               },
-       }
-
-       c.Assert(srv.Start(), check.IsNil)
-       defer srv.Close()
-
-       np := arvados.NodeProfile{
-               Controller: arvados.SystemServiceInstance{Listen: ":"},
-               RailsAPI: arvados.SystemServiceInstance{Listen: srv.Addr,
-                       TLS: false, Insecure: true}}
-       s.testHandler.Cluster.NodeProfiles["*"] = np
-       s.testHandler.NodeProfile = &np
+       defer s.localServiceReturns404(c).Close()
 
        req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
        resp := s.testRequest(req)
 
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+
        var col arvados.Collection
        c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
        c.Check(col.PortableDataHash, check.Equals, arvadostest.UserAgreementPDH)
@@ -411,24 +428,7 @@ func (s *FederationSuite) TestGetRemoteCollectionByPDH(c *check.C) {
 }
 
 func (s *FederationSuite) TestGetCollectionByPDHError(c *check.C) {
-       srv := &httpserver.Server{
-               Server: http.Server{
-                       Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
-                               w.WriteHeader(404)
-                       }),
-               },
-       }
-
-       c.Assert(srv.Start(), check.IsNil)
-       defer srv.Close()
-
-       // Make the "local" cluster to a service that always returns 404
-       np := arvados.NodeProfile{
-               Controller: arvados.SystemServiceInstance{Listen: ":"},
-               RailsAPI: arvados.SystemServiceInstance{Listen: srv.Addr,
-                       TLS: false, Insecure: true}}
-       s.testHandler.Cluster.NodeProfiles["*"] = np
-       s.testHandler.NodeProfile = &np
+       defer s.localServiceReturns404(c).Close()
 
        req := httptest.NewRequest("GET", "/arvados/v1/collections/99999999999999999999999999999999+99", nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
@@ -440,24 +440,7 @@ func (s *FederationSuite) TestGetCollectionByPDHError(c *check.C) {
 }
 
 func (s *FederationSuite) TestGetCollectionByPDHErrorBadHash(c *check.C) {
-       srv := &httpserver.Server{
-               Server: http.Server{
-                       Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
-                               w.WriteHeader(404)
-                       }),
-               },
-       }
-
-       c.Assert(srv.Start(), check.IsNil)
-       defer srv.Close()
-
-       // Make the "local" cluster to a service that always returns 404
-       np := arvados.NodeProfile{
-               Controller: arvados.SystemServiceInstance{Listen: ":"},
-               RailsAPI: arvados.SystemServiceInstance{Listen: srv.Addr,
-                       TLS: false, Insecure: true}}
-       s.testHandler.Cluster.NodeProfiles["*"] = np
-       s.testHandler.NodeProfile = &np
+       defer s.localServiceReturns404(c).Close()
 
        srv2 := &httpserver.Server{
                Server: http.Server{
@@ -530,3 +513,267 @@ func (s *FederationSuite) TestSaltedTokenGetCollectionByPDHError(c *check.C) {
 
        c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 }
+
+func (s *FederationSuite) TestGetRemoteContainerRequest(c *check.C) {
+       defer s.localServiceReturns404(c).Close()
+       req := httptest.NewRequest("GET", "/arvados/v1/container_requests/"+arvadostest.QueuedContainerRequestUUID, nil)
+       req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+       resp := s.testRequest(req)
+       c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+       var cr arvados.ContainerRequest
+       c.Check(json.NewDecoder(resp.Body).Decode(&cr), check.IsNil)
+       c.Check(cr.UUID, check.Equals, arvadostest.QueuedContainerRequestUUID)
+       c.Check(cr.Priority, check.Equals, 1)
+}
+
+func (s *FederationSuite) TestUpdateRemoteContainerRequest(c *check.C) {
+       defer s.localServiceReturns404(c).Close()
+       req := httptest.NewRequest("PATCH", "/arvados/v1/container_requests/"+arvadostest.QueuedContainerRequestUUID,
+               strings.NewReader(`{"container_request": {"priority": 696}}`))
+       req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+       req.Header.Set("Content-type", "application/json")
+       resp := s.testRequest(req)
+       c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+       var cr arvados.ContainerRequest
+       c.Check(json.NewDecoder(resp.Body).Decode(&cr), check.IsNil)
+       c.Check(cr.UUID, check.Equals, arvadostest.QueuedContainerRequestUUID)
+       c.Check(cr.Priority, check.Equals, 696)
+}
+
+func (s *FederationSuite) TestCreateRemoteContainerRequest(c *check.C) {
+       defer s.localServiceReturns404(c).Close()
+       // pass cluster_id via query parameter, this allows arvados-controller
+       // to avoid parsing the body
+       req := httptest.NewRequest("POST", "/arvados/v1/container_requests?cluster_id=zzzzz",
+               strings.NewReader(`{
+  "container_request": {
+    "name": "hello world",
+    "state": "Uncommitted",
+    "output_path": "/",
+    "container_image": "123",
+    "command": ["abc"]
+  }
+}
+`))
+       req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+       req.Header.Set("Content-type", "application/json")
+       resp := s.testRequest(req)
+       c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+       var cr arvados.ContainerRequest
+       c.Check(json.NewDecoder(resp.Body).Decode(&cr), check.IsNil)
+       c.Check(cr.Name, check.Equals, "hello world")
+       c.Check(strings.HasPrefix(cr.UUID, "zzzzz-"), check.Equals, true)
+}
+
+func (s *FederationSuite) TestCreateRemoteContainerRequestError(c *check.C) {
+       defer s.localServiceReturns404(c).Close()
+       // pass cluster_id via query parameter, this allows arvados-controller
+       // to avoid parsing the body
+       req := httptest.NewRequest("POST", "/arvados/v1/container_requests?cluster_id=zz404",
+               strings.NewReader(`{
+  "container_request": {
+    "name": "hello world",
+    "state": "Uncommitted",
+    "output_path": "/",
+    "container_image": "123",
+    "command": ["abc"]
+  }
+}
+`))
+       req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+       req.Header.Set("Content-type", "application/json")
+       resp := s.testRequest(req)
+       c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
+}
+
+func (s *FederationSuite) TestGetRemoteContainer(c *check.C) {
+       defer s.localServiceReturns404(c).Close()
+       req := httptest.NewRequest("GET", "/arvados/v1/containers/"+arvadostest.QueuedContainerUUID, nil)
+       req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+       resp := s.testRequest(req)
+       c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+       var cn arvados.Container
+       c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
+       c.Check(cn.UUID, check.Equals, arvadostest.QueuedContainerUUID)
+}
+
+func (s *FederationSuite) TestListRemoteContainer(c *check.C) {
+       defer s.localServiceReturns404(c).Close()
+       req := httptest.NewRequest("GET", "/arvados/v1/containers?count=none&filters="+
+               url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v"]]]`, arvadostest.QueuedContainerUUID)), nil)
+       req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+       resp := s.testRequest(req)
+       c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+       var cn arvados.ContainerList
+       c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
+       c.Check(cn.Items[0].UUID, check.Equals, arvadostest.QueuedContainerUUID)
+}
+
+func (s *FederationSuite) TestListMultiRemoteContainers(c *check.C) {
+       defer s.localServiceHandler(c, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+               bd, _ := ioutil.ReadAll(req.Body)
+               c.Check(string(bd), check.Equals, `_method=GET&count=none&filters=%5B%5B%22uuid%22%2C+%22in%22%2C+%5B%22zhome-xvhdp-cr5queuedcontnr%22%5D%5D%5D&select=%5B%22uuid%22%2C+%22command%22%5D`)
+               w.WriteHeader(200)
+               w.Write([]byte(`{"kind": "arvados#containerList", "items": [{"uuid": "zhome-xvhdp-cr5queuedcontnr", "command": ["abc"]}]}`))
+       })).Close()
+       req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&select=%s",
+               url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
+                       arvadostest.QueuedContainerUUID)),
+               url.QueryEscape(`["uuid", "command"]`)),
+               nil)
+       req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+       resp := s.testRequest(req)
+       c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+       var cn arvados.ContainerList
+       c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
+       c.Check(cn.Items, check.HasLen, 2)
+       mp := make(map[string]arvados.Container)
+       for _, cr := range cn.Items {
+               mp[cr.UUID] = cr
+       }
+       c.Check(mp[arvadostest.QueuedContainerUUID].Command, check.DeepEquals, []string{"echo", "hello"})
+       c.Check(mp[arvadostest.QueuedContainerUUID].ContainerImage, check.Equals, "")
+       c.Check(mp["zhome-xvhdp-cr5queuedcontnr"].Command, check.DeepEquals, []string{"abc"})
+       c.Check(mp["zhome-xvhdp-cr5queuedcontnr"].ContainerImage, check.Equals, "")
+}
+
+func (s *FederationSuite) TestListMultiRemoteContainerError(c *check.C) {
+       defer s.localServiceReturns404(c).Close()
+       req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&select=%s",
+               url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
+                       arvadostest.QueuedContainerUUID)),
+               url.QueryEscape(`["uuid", "command"]`)),
+               nil)
+       req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+       resp := s.testRequest(req)
+       c.Check(resp.StatusCode, check.Equals, http.StatusBadGateway)
+       s.checkJSONErrorMatches(c, resp, `error fetching from zhome \(404 Not Found\): EOF`)
+}
+
+func (s *FederationSuite) TestListMultiRemoteContainersPaged(c *check.C) {
+
+       callCount := 0
+       defer s.localServiceHandler(c, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+               bd, _ := ioutil.ReadAll(req.Body)
+               if callCount == 0 {
+                       c.Check(string(bd), check.Equals, `_method=GET&count=none&filters=%5B%5B%22uuid%22%2C+%22in%22%2C+%5B%22zhome-xvhdp-cr5queuedcontnr%22%2C%22zhome-xvhdp-cr6queuedcontnr%22%5D%5D%5D`)
+                       w.WriteHeader(200)
+                       w.Write([]byte(`{"kind": "arvados#containerList", "items": [{"uuid": "zhome-xvhdp-cr5queuedcontnr", "command": ["abc"]}]}`))
+               } else if callCount == 1 {
+                       c.Check(string(bd), check.Equals, `_method=GET&count=none&filters=%5B%5B%22uuid%22%2C+%22in%22%2C+%5B%22zhome-xvhdp-cr6queuedcontnr%22%5D%5D%5D`)
+                       w.WriteHeader(200)
+                       w.Write([]byte(`{"kind": "arvados#containerList", "items": [{"uuid": "zhome-xvhdp-cr6queuedcontnr", "command": ["efg"]}]}`))
+               }
+               callCount += 1
+       })).Close()
+       req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s",
+               url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr", "zhome-xvhdp-cr6queuedcontnr"]]]`,
+                       arvadostest.QueuedContainerUUID))),
+               nil)
+       req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+       resp := s.testRequest(req)
+       c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+       c.Check(callCount, check.Equals, 2)
+       var cn arvados.ContainerList
+       c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
+       c.Check(cn.Items, check.HasLen, 3)
+       mp := make(map[string]arvados.Container)
+       for _, cr := range cn.Items {
+               mp[cr.UUID] = cr
+       }
+       c.Check(mp[arvadostest.QueuedContainerUUID].Command, check.DeepEquals, []string{"echo", "hello"})
+       c.Check(mp["zhome-xvhdp-cr5queuedcontnr"].Command, check.DeepEquals, []string{"abc"})
+       c.Check(mp["zhome-xvhdp-cr6queuedcontnr"].Command, check.DeepEquals, []string{"efg"})
+}
+
+func (s *FederationSuite) TestListMultiRemoteContainersMissing(c *check.C) {
+
+       callCount := 0
+       defer s.localServiceHandler(c, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+               bd, _ := ioutil.ReadAll(req.Body)
+               if callCount == 0 {
+                       c.Check(string(bd), check.Equals, `_method=GET&count=none&filters=%5B%5B%22uuid%22%2C+%22in%22%2C+%5B%22zhome-xvhdp-cr5queuedcontnr%22%2C%22zhome-xvhdp-cr6queuedcontnr%22%5D%5D%5D`)
+                       w.WriteHeader(200)
+                       w.Write([]byte(`{"kind": "arvados#containerList", "items": [{"uuid": "zhome-xvhdp-cr6queuedcontnr", "command": ["efg"]}]}`))
+               } else if callCount == 1 {
+                       c.Check(string(bd), check.Equals, `_method=GET&count=none&filters=%5B%5B%22uuid%22%2C+%22in%22%2C+%5B%22zhome-xvhdp-cr5queuedcontnr%22%5D%5D%5D`)
+                       w.WriteHeader(200)
+                       w.Write([]byte(`{"kind": "arvados#containerList", "items": []}`))
+               }
+               callCount += 1
+       })).Close()
+       req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s",
+               url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr", "zhome-xvhdp-cr6queuedcontnr"]]]`,
+                       arvadostest.QueuedContainerUUID))),
+               nil)
+       req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+       resp := s.testRequest(req)
+       c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+       c.Check(callCount, check.Equals, 2)
+       var cn arvados.ContainerList
+       c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
+       c.Check(cn.Items, check.HasLen, 2)
+       mp := make(map[string]arvados.Container)
+       for _, cr := range cn.Items {
+               mp[cr.UUID] = cr
+       }
+       c.Check(mp[arvadostest.QueuedContainerUUID].Command, check.DeepEquals, []string{"echo", "hello"})
+       c.Check(mp["zhome-xvhdp-cr6queuedcontnr"].Command, check.DeepEquals, []string{"efg"})
+}
+
+func (s *FederationSuite) TestListMultiRemoteContainerPageSizeError(c *check.C) {
+       s.testHandler.Cluster.MaxItemsPerResponse = 1
+       req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s",
+               url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
+                       arvadostest.QueuedContainerUUID))),
+               nil)
+       req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+       resp := s.testRequest(req)
+       c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
+       s.checkJSONErrorMatches(c, resp, `Federated multi-object request for 2 objects which is more than max page size 1.`)
+}
+
+func (s *FederationSuite) TestListMultiRemoteContainerLimitError(c *check.C) {
+       req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&limit=1",
+               url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
+                       arvadostest.QueuedContainerUUID))),
+               nil)
+       req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+       resp := s.testRequest(req)
+       c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
+       s.checkJSONErrorMatches(c, resp, `Federated multi-object may not provide 'limit', 'offset' or 'order'.`)
+}
+
+func (s *FederationSuite) TestListMultiRemoteContainerOffsetError(c *check.C) {
+       req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&offset=1",
+               url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
+                       arvadostest.QueuedContainerUUID))),
+               nil)
+       req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+       resp := s.testRequest(req)
+       c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
+       s.checkJSONErrorMatches(c, resp, `Federated multi-object may not provide 'limit', 'offset' or 'order'.`)
+}
+
+func (s *FederationSuite) TestListMultiRemoteContainerOrderError(c *check.C) {
+       req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&order=uuid",
+               url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
+                       arvadostest.QueuedContainerUUID))),
+               nil)
+       req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+       resp := s.testRequest(req)
+       c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
+       s.checkJSONErrorMatches(c, resp, `Federated multi-object may not provide 'limit', 'offset' or 'order'.`)
+}
+
+func (s *FederationSuite) TestListMultiRemoteContainerSelectError(c *check.C) {
+       req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&select=%s",
+               url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
+                       arvadostest.QueuedContainerUUID)),
+               url.QueryEscape(`["command"]`)),
+               nil)
+       req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+       resp := s.testRequest(req)
+       c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
+       s.checkJSONErrorMatches(c, resp, `Federated multi-object request must include 'uuid' in 'select'`)
+}