14807: Merge branch 'master'
[arvados.git] / lib / controller / federation_test.go
index d1128b3d37ec3c94f51e5b58f9cda37a27cc0e51..62916acd2ac10be14d90d4e02e2703e77949e32b 100644 (file)
@@ -5,7 +5,10 @@
 package controller
 
 import (
+       "bytes"
        "encoding/json"
+       "fmt"
+       "io"
        "io/ioutil"
        "net/http"
        "net/http/httptest"
@@ -16,9 +19,10 @@ import (
 
        "git.curoverse.com/arvados.git/sdk/go/arvados"
        "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+       "git.curoverse.com/arvados.git/sdk/go/ctxlog"
        "git.curoverse.com/arvados.git/sdk/go/httpserver"
        "git.curoverse.com/arvados.git/sdk/go/keepclient"
-       "github.com/Sirupsen/logrus"
+       "github.com/sirupsen/logrus"
        check "gopkg.in/check.v1"
 )
 
@@ -26,7 +30,7 @@ import (
 var _ = check.Suite(&FederationSuite{})
 
 type FederationSuite struct {
-       log *logrus.Logger
+       log logrus.FieldLogger
        // testServer and testHandler are the controller being tested,
        // "zhome".
        testServer  *httpserver.Server
@@ -41,9 +45,7 @@ type FederationSuite struct {
 }
 
 func (s *FederationSuite) SetUpTest(c *check.C) {
-       s.log = logrus.New()
-       s.log.Formatter = &logrus.JSONFormatter{}
-       s.log.Out = &logWriter{c.Log}
+       s.log = ctxlog.TestLogger(c)
 
        s.remoteServer = newServerFromIntegrationTestEnv(c)
        c.Assert(s.remoteServer.Start(), check.IsNil)
@@ -62,6 +64,10 @@ func (s *FederationSuite) SetUpTest(c *check.C) {
                NodeProfiles: map[string]arvados.NodeProfile{
                        "*": nodeProfile,
                },
+               RequestLimits: arvados.RequestLimits{
+                       MaxItemsPerResponse:            1000,
+                       MultiClusterRequestConcurrency: 4,
+               },
        }, NodeProfile: &nodeProfile}
        s.testServer = newServerFromIntegrationTestEnv(c)
        s.testServer.Server.Handler = httpserver.AddRequestIDs(httpserver.LogRequests(s.log, s.testHandler))
@@ -85,6 +91,10 @@ func (s *FederationSuite) SetUpTest(c *check.C) {
 }
 
 func (s *FederationSuite) remoteMockHandler(w http.ResponseWriter, req *http.Request) {
+       b := &bytes.Buffer{}
+       io.Copy(b, req.Body)
+       req.Body.Close()
+       req.Body = ioutil.NopCloser(b)
        s.remoteMockRequests = append(s.remoteMockRequests, *req)
 }
 
@@ -192,7 +202,7 @@ func (s *FederationSuite) TestOptionsMethod(c *check.C) {
 func (s *FederationSuite) TestRemoteWithTokenInQuery(c *check.C) {
        req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1)+"?api_token="+arvadostest.ActiveToken, nil)
        s.testRequest(req)
-       c.Assert(len(s.remoteMockRequests), check.Equals, 1)
+       c.Assert(s.remoteMockRequests, check.HasLen, 1)
        pr := s.remoteMockRequests[0]
        // Token is salted and moved from query to Authorization header.
        c.Check(pr.URL.String(), check.Not(check.Matches), `.*api_token=.*`)
@@ -203,7 +213,7 @@ func (s *FederationSuite) TestLocalTokenSalted(c *check.C) {
        req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1), nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
        s.testRequest(req)
-       c.Assert(len(s.remoteMockRequests), check.Equals, 1)
+       c.Assert(s.remoteMockRequests, check.HasLen, 1)
        pr := s.remoteMockRequests[0]
        // The salted token here has a "zzzzz-" UUID instead of a
        // "ztest-" UUID because ztest's local database has the
@@ -219,7 +229,7 @@ func (s *FederationSuite) TestRemoteTokenNotSalted(c *check.C) {
        req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1), nil)
        req.Header.Set("Authorization", "Bearer "+remoteToken)
        s.testRequest(req)
-       c.Assert(len(s.remoteMockRequests), check.Equals, 1)
+       c.Assert(s.remoteMockRequests, check.HasLen, 1)
        pr := s.remoteMockRequests[0]
        c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer "+remoteToken)
 }
@@ -298,16 +308,14 @@ func (s *FederationSuite) checkJSONErrorMatches(c *check.C, resp *http.Response,
        var jresp httpserver.ErrorResponse
        err := json.NewDecoder(resp.Body).Decode(&jresp)
        c.Check(err, check.IsNil)
-       c.Assert(len(jresp.Errors), check.Equals, 1)
+       c.Assert(jresp.Errors, check.HasLen, 1)
        c.Check(jresp.Errors[0], check.Matches, re)
 }
 
-func (s *FederationSuite) localServiceReturns404(c *check.C) *httpserver.Server {
+func (s *FederationSuite) localServiceHandler(c *check.C, h http.Handler) *httpserver.Server {
        srv := &httpserver.Server{
                Server: http.Server{
-                       Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
-                               w.WriteHeader(404)
-                       }),
+                       Handler: h,
                },
        }
 
@@ -323,6 +331,12 @@ func (s *FederationSuite) localServiceReturns404(c *check.C) *httpserver.Server
        return srv
 }
 
+func (s *FederationSuite) localServiceReturns404(c *check.C) *httpserver.Server {
+       return s.localServiceHandler(c, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+               w.WriteHeader(404)
+       }))
+}
+
 func (s *FederationSuite) TestGetLocalCollection(c *check.C) {
        np := arvados.NodeProfile{
                Controller: arvados.SystemServiceInstance{Listen: ":"},
@@ -332,6 +346,8 @@ func (s *FederationSuite) TestGetLocalCollection(c *check.C) {
        s.testHandler.Cluster.NodeProfiles["*"] = np
        s.testHandler.NodeProfile = &np
 
+       // HTTP GET
+
        req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementCollection, nil)
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
        resp := s.testRequest(req)
@@ -343,6 +359,23 @@ func (s *FederationSuite) TestGetLocalCollection(c *check.C) {
        c.Check(col.ManifestText, check.Matches,
                `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+A[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
 `)
+
+       // HTTP POST with _method=GET as a form parameter
+
+       req = httptest.NewRequest("POST", "/arvados/v1/collections/"+arvadostest.UserAgreementCollection, bytes.NewBufferString((url.Values{
+               "_method": {"GET"},
+       }).Encode()))
+       req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+       req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
+       resp = s.testRequest(req)
+
+       c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+       col = arvados.Collection{}
+       c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
+       c.Check(col.UUID, check.Equals, arvadostest.UserAgreementCollection)
+       c.Check(col.ManifestText, check.Matches,
+               `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+A[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
+`)
 }
 
 func (s *FederationSuite) TestGetRemoteCollection(c *check.C) {
@@ -521,23 +554,56 @@ func (s *FederationSuite) TestGetRemoteContainerRequest(c *check.C) {
 
 func (s *FederationSuite) TestUpdateRemoteContainerRequest(c *check.C) {
        defer s.localServiceReturns404(c).Close()
-       req := httptest.NewRequest("PATCH", "/arvados/v1/container_requests/"+arvadostest.QueuedContainerRequestUUID,
-               strings.NewReader(`{"container_request": {"priority": 696}}`))
+       setPri := func(pri int) {
+               req := httptest.NewRequest("PATCH", "/arvados/v1/container_requests/"+arvadostest.QueuedContainerRequestUUID,
+                       strings.NewReader(fmt.Sprintf(`{"container_request": {"priority": %d}}`, pri)))
+               req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+               req.Header.Set("Content-type", "application/json")
+               resp := s.testRequest(req)
+               c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+               var cr arvados.ContainerRequest
+               c.Check(json.NewDecoder(resp.Body).Decode(&cr), check.IsNil)
+               c.Check(cr.UUID, check.Equals, arvadostest.QueuedContainerRequestUUID)
+               c.Check(cr.Priority, check.Equals, pri)
+       }
+       setPri(696)
+       setPri(1) // Reset fixture so side effect doesn't break other tests.
+}
+
+func (s *FederationSuite) TestCreateRemoteContainerRequest(c *check.C) {
+       defer s.localServiceReturns404(c).Close()
+       // pass cluster_id via query parameter, this allows arvados-controller
+       // to avoid parsing the body
+       req := httptest.NewRequest("POST", "/arvados/v1/container_requests?cluster_id=zzzzz",
+               strings.NewReader(`{
+  "container_request": {
+    "name": "hello world",
+    "state": "Uncommitted",
+    "output_path": "/",
+    "container_image": "123",
+    "command": ["abc"]
+  }
+}
+`))
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
        req.Header.Set("Content-type", "application/json")
        resp := s.testRequest(req)
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
        var cr arvados.ContainerRequest
        c.Check(json.NewDecoder(resp.Body).Decode(&cr), check.IsNil)
-       c.Check(cr.UUID, check.Equals, arvadostest.QueuedContainerRequestUUID)
-       c.Check(cr.Priority, check.Equals, 696)
+       c.Check(cr.Name, check.Equals, "hello world")
+       c.Check(strings.HasPrefix(cr.UUID, "zzzzz-"), check.Equals, true)
 }
 
-func (s *FederationSuite) TestCreateRemoteContainerRequest1(c *check.C) {
+func (s *FederationSuite) TestCreateRemoteContainerRequestCheckRuntimeToken(c *check.C) {
+       // Send request to zmock and check that outgoing request has
+       // runtime_token set with a new random v2 token.
+
        defer s.localServiceReturns404(c).Close()
-       req := httptest.NewRequest("POST", "/arvados/v1/container_requests",
+       // pass cluster_id via query parameter, this allows arvados-controller
+       // to avoid parsing the body
+       req := httptest.NewRequest("POST", "/arvados/v1/container_requests?cluster_id=zmock",
                strings.NewReader(`{
-  "cluster_id": "zzzzz",
   "container_request": {
     "name": "hello world",
     "state": "Uncommitted",
@@ -546,21 +612,66 @@ func (s *FederationSuite) TestCreateRemoteContainerRequest1(c *check.C) {
     "command": ["abc"]
   }
 }
+`))
+       req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
+       req.Header.Set("Content-type", "application/json")
+
+       np := arvados.NodeProfile{
+               Controller: arvados.SystemServiceInstance{Listen: ":"},
+               RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"),
+                       TLS: true, Insecure: true}}
+       s.testHandler.Cluster.ClusterID = "zzzzz"
+       s.testHandler.Cluster.NodeProfiles["*"] = np
+       s.testHandler.NodeProfile = &np
+
+       resp := s.testRequest(req)
+       c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+       var cr struct {
+               arvados.ContainerRequest `json:"container_request"`
+       }
+       c.Check(json.NewDecoder(s.remoteMockRequests[0].Body).Decode(&cr), check.IsNil)
+       c.Check(strings.HasPrefix(cr.ContainerRequest.RuntimeToken, "v2/zzzzz-gj3su-"), check.Equals, true)
+       c.Check(cr.ContainerRequest.RuntimeToken, check.Not(check.Equals), arvadostest.ActiveTokenV2)
+}
+
+func (s *FederationSuite) TestCreateRemoteContainerRequestCheckSetRuntimeToken(c *check.C) {
+       // Send request to zmock and check that outgoing request has
+       // runtime_token set with the explicitly provided token.
+
+       defer s.localServiceReturns404(c).Close()
+       // pass cluster_id via query parameter, this allows arvados-controller
+       // to avoid parsing the body
+       req := httptest.NewRequest("POST", "/arvados/v1/container_requests?cluster_id=zmock",
+               strings.NewReader(`{
+  "container_request": {
+    "name": "hello world",
+    "state": "Uncommitted",
+    "output_path": "/",
+    "container_image": "123",
+    "command": ["abc"],
+    "runtime_token": "xyz"
+  }
+}
 `))
        req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
        req.Header.Set("Content-type", "application/json")
        resp := s.testRequest(req)
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
-       var cr arvados.ContainerRequest
-       c.Check(json.NewDecoder(resp.Body).Decode(&cr), check.IsNil)
-       c.Check(cr.Name, check.Equals, "hello world")
+       var cr struct {
+               arvados.ContainerRequest `json:"container_request"`
+       }
+       c.Check(json.NewDecoder(s.remoteMockRequests[0].Body).Decode(&cr), check.IsNil)
+       c.Check(cr.ContainerRequest.RuntimeToken, check.Equals, "xyz")
 }
 
-func (s *FederationSuite) TestCreateRemoteContainerRequest2(c *check.C) {
+func (s *FederationSuite) TestCreateRemoteContainerRequestRuntimeTokenFromAuth(c *check.C) {
+       // Send request to zmock and check that outgoing request has
+       // runtime_token set using the Auth token because the user is remote.
+
        defer s.localServiceReturns404(c).Close()
        // pass cluster_id via query parameter, this allows arvados-controller
        // to avoid parsing the body
-       req := httptest.NewRequest("POST", "/arvados/v1/container_requests?cluster_id=zzzzz",
+       req := httptest.NewRequest("POST", "/arvados/v1/container_requests?cluster_id=zmock",
                strings.NewReader(`{
   "container_request": {
     "name": "hello world",
@@ -571,14 +682,15 @@ func (s *FederationSuite) TestCreateRemoteContainerRequest2(c *check.C) {
   }
 }
 `))
-       req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+       req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2+"/zzzzz-dz642-parentcontainer")
        req.Header.Set("Content-type", "application/json")
        resp := s.testRequest(req)
        c.Check(resp.StatusCode, check.Equals, http.StatusOK)
-       var cr arvados.ContainerRequest
-       c.Check(json.NewDecoder(resp.Body).Decode(&cr), check.IsNil)
-       c.Check(cr.Name, check.Equals, "hello world")
-       c.Check(strings.HasPrefix(cr.UUID, "zzzzz-"), check.Equals, true)
+       var cr struct {
+               arvados.ContainerRequest `json:"container_request"`
+       }
+       c.Check(json.NewDecoder(s.remoteMockRequests[0].Body).Decode(&cr), check.IsNil)
+       c.Check(cr.ContainerRequest.RuntimeToken, check.Equals, arvadostest.ActiveTokenV2)
 }
 
 func (s *FederationSuite) TestCreateRemoteContainerRequestError(c *check.C) {
@@ -612,3 +724,184 @@ func (s *FederationSuite) TestGetRemoteContainer(c *check.C) {
        c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
        c.Check(cn.UUID, check.Equals, arvadostest.QueuedContainerUUID)
 }
+
+func (s *FederationSuite) TestListRemoteContainer(c *check.C) {
+       defer s.localServiceReturns404(c).Close()
+       req := httptest.NewRequest("GET", "/arvados/v1/containers?count=none&filters="+
+               url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v"]]]`, arvadostest.QueuedContainerUUID)), nil)
+       req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+       resp := s.testRequest(req)
+       c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+       var cn arvados.ContainerList
+       c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
+       c.Check(cn.Items[0].UUID, check.Equals, arvadostest.QueuedContainerUUID)
+}
+
+func (s *FederationSuite) TestListMultiRemoteContainers(c *check.C) {
+       defer s.localServiceHandler(c, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+               bd, _ := ioutil.ReadAll(req.Body)
+               c.Check(string(bd), check.Equals, `_method=GET&count=none&filters=%5B%5B%22uuid%22%2C+%22in%22%2C+%5B%22zhome-xvhdp-cr5queuedcontnr%22%5D%5D%5D&select=%5B%22uuid%22%2C+%22command%22%5D`)
+               w.WriteHeader(200)
+               w.Write([]byte(`{"kind": "arvados#containerList", "items": [{"uuid": "zhome-xvhdp-cr5queuedcontnr", "command": ["abc"]}]}`))
+       })).Close()
+       req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&select=%s",
+               url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
+                       arvadostest.QueuedContainerUUID)),
+               url.QueryEscape(`["uuid", "command"]`)),
+               nil)
+       req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+       resp := s.testRequest(req)
+       c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+       var cn arvados.ContainerList
+       c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
+       c.Check(cn.Items, check.HasLen, 2)
+       mp := make(map[string]arvados.Container)
+       for _, cr := range cn.Items {
+               mp[cr.UUID] = cr
+       }
+       c.Check(mp[arvadostest.QueuedContainerUUID].Command, check.DeepEquals, []string{"echo", "hello"})
+       c.Check(mp[arvadostest.QueuedContainerUUID].ContainerImage, check.Equals, "")
+       c.Check(mp["zhome-xvhdp-cr5queuedcontnr"].Command, check.DeepEquals, []string{"abc"})
+       c.Check(mp["zhome-xvhdp-cr5queuedcontnr"].ContainerImage, check.Equals, "")
+}
+
+func (s *FederationSuite) TestListMultiRemoteContainerError(c *check.C) {
+       defer s.localServiceReturns404(c).Close()
+       req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&select=%s",
+               url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
+                       arvadostest.QueuedContainerUUID)),
+               url.QueryEscape(`["uuid", "command"]`)),
+               nil)
+       req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+       resp := s.testRequest(req)
+       c.Check(resp.StatusCode, check.Equals, http.StatusBadGateway)
+       s.checkJSONErrorMatches(c, resp, `error fetching from zhome \(404 Not Found\): EOF`)
+}
+
+func (s *FederationSuite) TestListMultiRemoteContainersPaged(c *check.C) {
+
+       callCount := 0
+       defer s.localServiceHandler(c, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+               bd, _ := ioutil.ReadAll(req.Body)
+               if callCount == 0 {
+                       c.Check(string(bd), check.Equals, `_method=GET&count=none&filters=%5B%5B%22uuid%22%2C+%22in%22%2C+%5B%22zhome-xvhdp-cr5queuedcontnr%22%2C%22zhome-xvhdp-cr6queuedcontnr%22%5D%5D%5D`)
+                       w.WriteHeader(200)
+                       w.Write([]byte(`{"kind": "arvados#containerList", "items": [{"uuid": "zhome-xvhdp-cr5queuedcontnr", "command": ["abc"]}]}`))
+               } else if callCount == 1 {
+                       c.Check(string(bd), check.Equals, `_method=GET&count=none&filters=%5B%5B%22uuid%22%2C+%22in%22%2C+%5B%22zhome-xvhdp-cr6queuedcontnr%22%5D%5D%5D`)
+                       w.WriteHeader(200)
+                       w.Write([]byte(`{"kind": "arvados#containerList", "items": [{"uuid": "zhome-xvhdp-cr6queuedcontnr", "command": ["efg"]}]}`))
+               }
+               callCount += 1
+       })).Close()
+       req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s",
+               url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr", "zhome-xvhdp-cr6queuedcontnr"]]]`,
+                       arvadostest.QueuedContainerUUID))),
+               nil)
+       req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+       resp := s.testRequest(req)
+       c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+       c.Check(callCount, check.Equals, 2)
+       var cn arvados.ContainerList
+       c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
+       c.Check(cn.Items, check.HasLen, 3)
+       mp := make(map[string]arvados.Container)
+       for _, cr := range cn.Items {
+               mp[cr.UUID] = cr
+       }
+       c.Check(mp[arvadostest.QueuedContainerUUID].Command, check.DeepEquals, []string{"echo", "hello"})
+       c.Check(mp["zhome-xvhdp-cr5queuedcontnr"].Command, check.DeepEquals, []string{"abc"})
+       c.Check(mp["zhome-xvhdp-cr6queuedcontnr"].Command, check.DeepEquals, []string{"efg"})
+}
+
+func (s *FederationSuite) TestListMultiRemoteContainersMissing(c *check.C) {
+
+       callCount := 0
+       defer s.localServiceHandler(c, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+               bd, _ := ioutil.ReadAll(req.Body)
+               if callCount == 0 {
+                       c.Check(string(bd), check.Equals, `_method=GET&count=none&filters=%5B%5B%22uuid%22%2C+%22in%22%2C+%5B%22zhome-xvhdp-cr5queuedcontnr%22%2C%22zhome-xvhdp-cr6queuedcontnr%22%5D%5D%5D`)
+                       w.WriteHeader(200)
+                       w.Write([]byte(`{"kind": "arvados#containerList", "items": [{"uuid": "zhome-xvhdp-cr6queuedcontnr", "command": ["efg"]}]}`))
+               } else if callCount == 1 {
+                       c.Check(string(bd), check.Equals, `_method=GET&count=none&filters=%5B%5B%22uuid%22%2C+%22in%22%2C+%5B%22zhome-xvhdp-cr5queuedcontnr%22%5D%5D%5D`)
+                       w.WriteHeader(200)
+                       w.Write([]byte(`{"kind": "arvados#containerList", "items": []}`))
+               }
+               callCount += 1
+       })).Close()
+       req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s",
+               url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr", "zhome-xvhdp-cr6queuedcontnr"]]]`,
+                       arvadostest.QueuedContainerUUID))),
+               nil)
+       req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+       resp := s.testRequest(req)
+       c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+       c.Check(callCount, check.Equals, 2)
+       var cn arvados.ContainerList
+       c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
+       c.Check(cn.Items, check.HasLen, 2)
+       mp := make(map[string]arvados.Container)
+       for _, cr := range cn.Items {
+               mp[cr.UUID] = cr
+       }
+       c.Check(mp[arvadostest.QueuedContainerUUID].Command, check.DeepEquals, []string{"echo", "hello"})
+       c.Check(mp["zhome-xvhdp-cr6queuedcontnr"].Command, check.DeepEquals, []string{"efg"})
+}
+
+func (s *FederationSuite) TestListMultiRemoteContainerPageSizeError(c *check.C) {
+       s.testHandler.Cluster.RequestLimits.MaxItemsPerResponse = 1
+       req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s",
+               url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
+                       arvadostest.QueuedContainerUUID))),
+               nil)
+       req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+       resp := s.testRequest(req)
+       c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
+       s.checkJSONErrorMatches(c, resp, `Federated multi-object request for 2 objects which is more than max page size 1.`)
+}
+
+func (s *FederationSuite) TestListMultiRemoteContainerLimitError(c *check.C) {
+       req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&limit=1",
+               url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
+                       arvadostest.QueuedContainerUUID))),
+               nil)
+       req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+       resp := s.testRequest(req)
+       c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
+       s.checkJSONErrorMatches(c, resp, `Federated multi-object may not provide 'limit', 'offset' or 'order'.`)
+}
+
+func (s *FederationSuite) TestListMultiRemoteContainerOffsetError(c *check.C) {
+       req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&offset=1",
+               url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
+                       arvadostest.QueuedContainerUUID))),
+               nil)
+       req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+       resp := s.testRequest(req)
+       c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
+       s.checkJSONErrorMatches(c, resp, `Federated multi-object may not provide 'limit', 'offset' or 'order'.`)
+}
+
+func (s *FederationSuite) TestListMultiRemoteContainerOrderError(c *check.C) {
+       req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&order=uuid",
+               url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
+                       arvadostest.QueuedContainerUUID))),
+               nil)
+       req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+       resp := s.testRequest(req)
+       c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
+       s.checkJSONErrorMatches(c, resp, `Federated multi-object may not provide 'limit', 'offset' or 'order'.`)
+}
+
+func (s *FederationSuite) TestListMultiRemoteContainerSelectError(c *check.C) {
+       req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&select=%s",
+               url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
+                       arvadostest.QueuedContainerUUID)),
+               url.QueryEscape(`["command"]`)),
+               nil)
+       req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+       resp := s.testRequest(req)
+       c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
+       s.checkJSONErrorMatches(c, resp, `Federated multi-object request must include 'uuid' in 'select'`)
+}