lib/controller/federation_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package controller
   6
   7 import (
   8         "bytes"
   9         "encoding/json"
  10         "fmt"
  11         "io"
  12         "io/ioutil"
  13         "net/http"
  14         "net/http/httptest"
  15         "net/url"
  16         "os"
  17         "strings"
  18         "time"
  19
  20         "git.curoverse.com/arvados.git/sdk/go/arvados"
  21         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  22         "git.curoverse.com/arvados.git/sdk/go/ctxlog"
  23         "git.curoverse.com/arvados.git/sdk/go/httpserver"
  24         "git.curoverse.com/arvados.git/sdk/go/keepclient"
  25         "github.com/sirupsen/logrus"
  26         check "gopkg.in/check.v1"
  27 )
  28
  29 // Gocheck boilerplate
  30 var _ = check.Suite(&FederationSuite{})
  31
  32 type FederationSuite struct {
  33         log logrus.FieldLogger
  34         // testServer and testHandler are the controller being tested,
  35         // "zhome".
  36         testServer  *httpserver.Server
  37         testHandler *Handler
  38         // remoteServer ("zzzzz") forwards requests to the Rails API
  39         // provided by the integration test environment.
  40         remoteServer *httpserver.Server
  41         // remoteMock ("zmock") appends each incoming request to
  42         // remoteMockRequests, and returns 200 with an empty JSON
  43         // object.
  44         remoteMock         *httpserver.Server
  45         remoteMockRequests []http.Request
  46 }
  47
  48 func (s *FederationSuite) SetUpTest(c *check.C) {
  49         s.log = ctxlog.TestLogger(c)
  50
  51         s.remoteServer = newServerFromIntegrationTestEnv(c)
  52         c.Assert(s.remoteServer.Start(), check.IsNil)
  53
  54         s.remoteMock = newServerFromIntegrationTestEnv(c)
  55         s.remoteMock.Server.Handler = http.HandlerFunc(s.remoteMockHandler)
  56         c.Assert(s.remoteMock.Start(), check.IsNil)
  57
  58         cluster := &arvados.Cluster{
  59                 ClusterID:                 "zhome",
  60                 PostgreSQL:                integrationTestCluster().PostgreSQL,
  61                 EnableBetaController14287: enableBetaController14287,
  62         }
  63         cluster.TLS.Insecure = true
  64         cluster.API.MaxItemsPerResponse = 1000
  65         cluster.API.MaxRequestAmplification = 4
  66         arvadostest.SetServiceURL(&cluster.Services.RailsAPI, "http://localhost:1/")
  67         arvadostest.SetServiceURL(&cluster.Services.Controller, "http://localhost:/")
  68         s.testHandler = &Handler{Cluster: cluster}
  69         s.testServer = newServerFromIntegrationTestEnv(c)
  70         s.testServer.Server.Handler = httpserver.AddRequestIDs(httpserver.LogRequests(s.log, s.testHandler))
  71
  72         cluster.RemoteClusters = map[string]arvados.RemoteCluster{
  73                 "zzzzz": {
  74                         Host:   s.remoteServer.Addr,
  75                         Proxy:  true,
  76                         Scheme: "http",
  77                 },
  78                 "zmock": {
  79                         Host:   s.remoteMock.Addr,
  80                         Proxy:  true,
  81                         Scheme: "http",
  82                 },
  83         }
  84
  85         c.Assert(s.testServer.Start(), check.IsNil)
  86
  87         s.remoteMockRequests = nil
  88 }
  89
  90 func (s *FederationSuite) remoteMockHandler(w http.ResponseWriter, req *http.Request) {
  91         b := &bytes.Buffer{}
  92         io.Copy(b, req.Body)
  93         req.Body.Close()
  94         req.Body = ioutil.NopCloser(b)
  95         s.remoteMockRequests = append(s.remoteMockRequests, *req)
  96         // Repond 200 with a valid JSON object
  97         fmt.Fprint(w, "{}")
  98 }
  99
 100 func (s *FederationSuite) TearDownTest(c *check.C) {
 101         if s.remoteServer != nil {
 102                 s.remoteServer.Close()
 103         }
 104         if s.testServer != nil {
 105                 s.testServer.Close()
 106         }
 107 }
 108
 109 func (s *FederationSuite) testRequest(req *http.Request) *httptest.ResponseRecorder {
 110         resp := httptest.NewRecorder()
 111         s.testServer.Server.Handler.ServeHTTP(resp, req)
 112         return resp
 113 }
 114
 115 func (s *FederationSuite) TestLocalRequest(c *check.C) {
 116         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zhome-", 1), nil)
 117         resp := s.testRequest(req).Result()
 118         s.checkHandledLocally(c, resp)
 119 }
 120
 121 func (s *FederationSuite) checkHandledLocally(c *check.C, resp *http.Response) {
 122         // Our "home" controller can't handle local requests because
 123         // it doesn't have its own stub/test Rails API, so we rely on
 124         // "connection refused" to indicate the controller tried to
 125         // proxy the request to its local Rails API.
 126         c.Check(resp.StatusCode, check.Equals, http.StatusBadGateway)
 127         s.checkJSONErrorMatches(c, resp, `.*connection refused`)
 128 }
 129
 130 func (s *FederationSuite) TestNoAuth(c *check.C) {
 131         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
 132         resp := s.testRequest(req).Result()
 133         c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
 134         s.checkJSONErrorMatches(c, resp, `Not logged in`)
 135 }
 136
 137 func (s *FederationSuite) TestBadAuth(c *check.C) {
 138         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
 139         req.Header.Set("Authorization", "Bearer aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
 140         resp := s.testRequest(req).Result()
 141         c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
 142         s.checkJSONErrorMatches(c, resp, `Not logged in`)
 143 }
 144
 145 func (s *FederationSuite) TestNoAccess(c *check.C) {
 146         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
 147         req.Header.Set("Authorization", "Bearer "+arvadostest.SpectatorToken)
 148         resp := s.testRequest(req).Result()
 149         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 150         s.checkJSONErrorMatches(c, resp, `.*not found`)
 151 }
 152
 153 func (s *FederationSuite) TestGetUnknownRemote(c *check.C) {
 154         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zz404-", 1), nil)
 155         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 156         resp := s.testRequest(req).Result()
 157         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 158         s.checkJSONErrorMatches(c, resp, `.*no proxy available for cluster zz404`)
 159 }
 160
 161 func (s *FederationSuite) TestRemoteError(c *check.C) {
 162         rc := s.testHandler.Cluster.RemoteClusters["zzzzz"]
 163         rc.Scheme = "https"
 164         s.testHandler.Cluster.RemoteClusters["zzzzz"] = rc
 165
 166         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
 167         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 168         resp := s.testRequest(req).Result()
 169         c.Check(resp.StatusCode, check.Equals, http.StatusBadGateway)
 170         s.checkJSONErrorMatches(c, resp, `.*HTTP response to HTTPS client`)
 171 }
 172
 173 func (s *FederationSuite) TestGetRemoteWorkflow(c *check.C) {
 174         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
 175         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 176         resp := s.testRequest(req).Result()
 177         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 178         var wf arvados.Workflow
 179         c.Check(json.NewDecoder(resp.Body).Decode(&wf), check.IsNil)
 180         c.Check(wf.UUID, check.Equals, arvadostest.WorkflowWithDefinitionYAMLUUID)
 181         c.Check(wf.OwnerUUID, check.Equals, arvadostest.ActiveUserUUID)
 182 }
 183
 184 func (s *FederationSuite) TestOptionsMethod(c *check.C) {
 185         req := httptest.NewRequest("OPTIONS", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
 186         req.Header.Set("Origin", "https://example.com")
 187         resp := s.testRequest(req).Result()
 188         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 189         body, err := ioutil.ReadAll(resp.Body)
 190         c.Check(err, check.IsNil)
 191         c.Check(string(body), check.Equals, "")
 192         c.Check(resp.Header.Get("Access-Control-Allow-Origin"), check.Equals, "*")
 193         for _, hdr := range []string{"Authorization", "Content-Type"} {
 194                 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), check.Matches, ".*"+hdr+".*")
 195         }
 196         for _, method := range []string{"GET", "HEAD", "PUT", "POST", "DELETE"} {
 197                 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), check.Matches, ".*"+method+".*")
 198         }
 199 }
 200
 201 func (s *FederationSuite) TestRemoteWithTokenInQuery(c *check.C) {
 202         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1)+"?api_token="+arvadostest.ActiveToken, nil)
 203         s.testRequest(req).Result()
 204         c.Assert(s.remoteMockRequests, check.HasLen, 1)
 205         pr := s.remoteMockRequests[0]
 206         // Token is salted and moved from query to Authorization header.
 207         c.Check(pr.URL.String(), check.Not(check.Matches), `.*api_token=.*`)
 208         c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/7fd31b61f39c0e82a4155592163218272cedacdc")
 209 }
 210
 211 func (s *FederationSuite) TestLocalTokenSalted(c *check.C) {
 212         defer s.localServiceReturns404(c).Close()
 213         for _, path := range []string{
 214                 // During the transition to the strongly typed
 215                 // controller implementation (#14287), workflows and
 216                 // collections test different code paths.
 217                 "/arvados/v1/workflows/" + strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1),
 218                 "/arvados/v1/collections/" + strings.Replace(arvadostest.UserAgreementCollection, "zzzzz-", "zmock-", 1),
 219         } {
 220                 c.Log("testing path ", path)
 221                 s.remoteMockRequests = nil
 222                 req := httptest.NewRequest("GET", path, nil)
 223                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 224                 s.testRequest(req).Result()
 225                 c.Assert(s.remoteMockRequests, check.HasLen, 1)
 226                 pr := s.remoteMockRequests[0]
 227                 // The salted token here has a "zzzzz-" UUID instead of a
 228                 // "ztest-" UUID because ztest's local database has the
 229                 // "zzzzz-" test fixtures. The "secret" part is HMAC(sha1,
 230                 // arvadostest.ActiveToken, "zmock") = "7fd3...".
 231                 c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/7fd31b61f39c0e82a4155592163218272cedacdc")
 232         }
 233 }
 234
 235 func (s *FederationSuite) TestRemoteTokenNotSalted(c *check.C) {
 236         defer s.localServiceReturns404(c).Close()
 237         // remoteToken can be any v1 token that doesn't appear in
 238         // ztest's local db.
 239         remoteToken := "abcdef00000000000000000000000000000000000000000000"
 240
 241         for _, path := range []string{
 242                 // During the transition to the strongly typed
 243                 // controller implementation (#14287), workflows and
 244                 // collections test different code paths.
 245                 "/arvados/v1/workflows/" + strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1),
 246                 "/arvados/v1/collections/" + strings.Replace(arvadostest.UserAgreementCollection, "zzzzz-", "zmock-", 1),
 247         } {
 248                 c.Log("testing path ", path)
 249                 s.remoteMockRequests = nil
 250                 req := httptest.NewRequest("GET", path, nil)
 251                 req.Header.Set("Authorization", "Bearer "+remoteToken)
 252                 s.testRequest(req).Result()
 253                 c.Assert(s.remoteMockRequests, check.HasLen, 1)
 254                 pr := s.remoteMockRequests[0]
 255                 c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer "+remoteToken)
 256         }
 257 }
 258
 259 func (s *FederationSuite) TestWorkflowCRUD(c *check.C) {
 260         wf := arvados.Workflow{
 261                 Description: "TestCRUD",
 262         }
 263         {
 264                 body := &strings.Builder{}
 265                 json.NewEncoder(body).Encode(&wf)
 266                 req := httptest.NewRequest("POST", "/arvados/v1/workflows", strings.NewReader(url.Values{
 267                         "workflow": {body.String()},
 268                 }.Encode()))
 269                 req.Header.Set("Content-type", "application/x-www-form-urlencoded")
 270                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 271                 rec := httptest.NewRecorder()
 272                 s.remoteServer.Server.Handler.ServeHTTP(rec, req) // direct to remote -- can't proxy a create req because no uuid
 273                 resp := rec.Result()
 274                 s.checkResponseOK(c, resp)
 275                 json.NewDecoder(resp.Body).Decode(&wf)
 276
 277                 defer func() {
 278                         req := httptest.NewRequest("DELETE", "/arvados/v1/workflows/"+wf.UUID, nil)
 279                         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 280                         s.remoteServer.Server.Handler.ServeHTTP(httptest.NewRecorder(), req)
 281                 }()
 282                 c.Check(wf.UUID, check.Not(check.Equals), "")
 283
 284                 c.Assert(wf.ModifiedAt, check.NotNil)
 285                 c.Logf("wf.ModifiedAt: %v", wf.ModifiedAt)
 286                 c.Check(time.Since(*wf.ModifiedAt) < time.Minute, check.Equals, true)
 287         }
 288         for _, method := range []string{"PATCH", "PUT", "POST"} {
 289                 form := url.Values{
 290                         "workflow": {`{"description": "Updated with ` + method + `"}`},
 291                 }
 292                 if method == "POST" {
 293                         form["_method"] = []string{"PATCH"}
 294                 }
 295                 req := httptest.NewRequest(method, "/arvados/v1/workflows/"+wf.UUID, strings.NewReader(form.Encode()))
 296                 req.Header.Set("Content-type", "application/x-www-form-urlencoded")
 297                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 298                 resp := s.testRequest(req).Result()
 299                 s.checkResponseOK(c, resp)
 300                 err := json.NewDecoder(resp.Body).Decode(&wf)
 301                 c.Check(err, check.IsNil)
 302
 303                 c.Check(wf.Description, check.Equals, "Updated with "+method)
 304         }
 305         {
 306                 req := httptest.NewRequest("DELETE", "/arvados/v1/workflows/"+wf.UUID, nil)
 307                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 308                 resp := s.testRequest(req).Result()
 309                 s.checkResponseOK(c, resp)
 310                 err := json.NewDecoder(resp.Body).Decode(&wf)
 311                 c.Check(err, check.IsNil)
 312         }
 313         {
 314                 req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+wf.UUID, nil)
 315                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 316                 resp := s.testRequest(req).Result()
 317                 c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 318         }
 319 }
 320
 321 func (s *FederationSuite) checkResponseOK(c *check.C, resp *http.Response) {
 322         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 323         if resp.StatusCode != http.StatusOK {
 324                 body, err := ioutil.ReadAll(resp.Body)
 325                 c.Logf("... response body = %q, %v\n", body, err)
 326         }
 327 }
 328
 329 func (s *FederationSuite) checkJSONErrorMatches(c *check.C, resp *http.Response, re string) {
 330         var jresp httpserver.ErrorResponse
 331         err := json.NewDecoder(resp.Body).Decode(&jresp)
 332         c.Check(err, check.IsNil)
 333         c.Assert(jresp.Errors, check.HasLen, 1)
 334         c.Check(jresp.Errors[0], check.Matches, re)
 335 }
 336
 337 func (s *FederationSuite) localServiceHandler(c *check.C, h http.Handler) *httpserver.Server {
 338         srv := &httpserver.Server{
 339                 Server: http.Server{
 340                         Handler: h,
 341                 },
 342         }
 343         c.Assert(srv.Start(), check.IsNil)
 344         arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "http://"+srv.Addr)
 345         return srv
 346 }
 347
 348 func (s *FederationSuite) localServiceReturns404(c *check.C) *httpserver.Server {
 349         return s.localServiceHandler(c, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 350                 if req.URL.Path == "/arvados/v1/api_client_authorizations/current" {
 351                         if req.Header.Get("Authorization") == "Bearer "+arvadostest.ActiveToken {
 352                                 json.NewEncoder(w).Encode(arvados.APIClientAuthorization{UUID: arvadostest.ActiveTokenUUID, APIToken: arvadostest.ActiveToken})
 353                         } else {
 354                                 w.WriteHeader(http.StatusUnauthorized)
 355                         }
 356                 } else {
 357                         w.WriteHeader(404)
 358                 }
 359         }))
 360 }
 361
 362 func (s *FederationSuite) TestGetLocalCollection(c *check.C) {
 363         s.testHandler.Cluster.ClusterID = "zzzzz"
 364         arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
 365
 366         // HTTP GET
 367
 368         req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementCollection, nil)
 369         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 370         resp := s.testRequest(req).Result()
 371
 372         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 373         var col arvados.Collection
 374         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
 375         c.Check(col.UUID, check.Equals, arvadostest.UserAgreementCollection)
 376         c.Check(col.ManifestText, check.Matches,
 377                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+A[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
 378 `)
 379
 380         // HTTP POST with _method=GET as a form parameter
 381
 382         req = httptest.NewRequest("POST", "/arvados/v1/collections/"+arvadostest.UserAgreementCollection, bytes.NewBufferString((url.Values{
 383                 "_method": {"GET"},
 384         }).Encode()))
 385         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 386         req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
 387         resp = s.testRequest(req).Result()
 388
 389         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 390         col = arvados.Collection{}
 391         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
 392         c.Check(col.UUID, check.Equals, arvadostest.UserAgreementCollection)
 393         c.Check(col.ManifestText, check.Matches,
 394                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+A[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
 395 `)
 396 }
 397
 398 func (s *FederationSuite) TestGetRemoteCollection(c *check.C) {
 399         defer s.localServiceReturns404(c).Close()
 400
 401         req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementCollection, nil)
 402         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 403         resp := s.testRequest(req).Result()
 404         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 405         var col arvados.Collection
 406         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
 407         c.Check(col.UUID, check.Equals, arvadostest.UserAgreementCollection)
 408         c.Check(col.ManifestText, check.Matches,
 409                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+Rzzzzz-[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
 410 `)
 411 }
 412
 413 func (s *FederationSuite) TestGetRemoteCollectionError(c *check.C) {
 414         defer s.localServiceReturns404(c).Close()
 415
 416         req := httptest.NewRequest("GET", "/arvados/v1/collections/zzzzz-4zz18-fakefakefakefak", nil)
 417         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 418         resp := s.testRequest(req).Result()
 419         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 420 }
 421
 422 func (s *FederationSuite) TestSignedLocatorPattern(c *check.C) {
 423         // Confirm the regular expression identifies other groups of hints correctly
 424         c.Check(keepclient.SignedLocatorRe.FindStringSubmatch(`6a4ff0499484c6c79c95cd8c566bd25f+249025+B1+C2+A05227438989d04712ea9ca1c91b556cef01d5cc7@5ba5405b+D3+E4`),
 425                 check.DeepEquals,
 426                 []string{"6a4ff0499484c6c79c95cd8c566bd25f+249025+B1+C2+A05227438989d04712ea9ca1c91b556cef01d5cc7@5ba5405b+D3+E4",
 427                         "6a4ff0499484c6c79c95cd8c566bd25f",
 428                         "+249025",
 429                         "+B1+C2", "+C2",
 430                         "+A05227438989d04712ea9ca1c91b556cef01d5cc7@5ba5405b",
 431                         "05227438989d04712ea9ca1c91b556cef01d5cc7", "5ba5405b",
 432                         "+D3+E4", "+E4"})
 433 }
 434
 435 func (s *FederationSuite) TestGetLocalCollectionByPDH(c *check.C) {
 436         arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
 437
 438         req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
 439         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 440         resp := s.testRequest(req).Result()
 441
 442         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 443         var col arvados.Collection
 444         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
 445         c.Check(col.PortableDataHash, check.Equals, arvadostest.UserAgreementPDH)
 446         c.Check(col.ManifestText, check.Matches,
 447                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+A[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
 448 `)
 449 }
 450
 451 func (s *FederationSuite) TestGetRemoteCollectionByPDH(c *check.C) {
 452         defer s.localServiceReturns404(c).Close()
 453
 454         req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
 455         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 456         resp := s.testRequest(req).Result()
 457
 458         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 459
 460         var col arvados.Collection
 461         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
 462         c.Check(col.PortableDataHash, check.Equals, arvadostest.UserAgreementPDH)
 463         c.Check(col.ManifestText, check.Matches,
 464                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+Rzzzzz-[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
 465 `)
 466 }
 467
 468 func (s *FederationSuite) TestGetCollectionByPDHError(c *check.C) {
 469         defer s.localServiceReturns404(c).Close()
 470
 471         req := httptest.NewRequest("GET", "/arvados/v1/collections/99999999999999999999999999999999+99", nil)
 472         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 473
 474         resp := s.testRequest(req).Result()
 475         defer resp.Body.Close()
 476
 477         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 478 }
 479
 480 func (s *FederationSuite) TestGetCollectionByPDHErrorBadHash(c *check.C) {
 481         defer s.localServiceReturns404(c).Close()
 482
 483         srv2 := &httpserver.Server{
 484                 Server: http.Server{
 485                         Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 486                                 w.WriteHeader(200)
 487                                 // Return a collection where the hash
 488                                 // of the manifest text doesn't match
 489                                 // PDH that was requested.
 490                                 var col arvados.Collection
 491                                 col.PortableDataHash = "99999999999999999999999999999999+99"
 492                                 col.ManifestText = `. 6a4ff0499484c6c79c95cd8c566bd25f\+249025 0:249025:GNU_General_Public_License,_version_3.pdf
 493 `
 494                                 enc := json.NewEncoder(w)
 495                                 enc.Encode(col)
 496                         }),
 497                 },
 498         }
 499
 500         c.Assert(srv2.Start(), check.IsNil)
 501         defer srv2.Close()
 502
 503         // Direct zzzzz to service that returns a 200 result with a bogus manifest_text
 504         s.testHandler.Cluster.RemoteClusters["zzzzz"] = arvados.RemoteCluster{
 505                 Host:   srv2.Addr,
 506                 Proxy:  true,
 507                 Scheme: "http",
 508         }
 509
 510         req := httptest.NewRequest("GET", "/arvados/v1/collections/99999999999999999999999999999999+99", nil)
 511         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 512
 513         resp := s.testRequest(req).Result()
 514         defer resp.Body.Close()
 515
 516         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 517 }
 518
 519 func (s *FederationSuite) TestSaltedTokenGetCollectionByPDH(c *check.C) {
 520         arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
 521
 522         req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
 523         req.Header.Set("Authorization", "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/282d7d172b6cfdce364c5ed12ddf7417b2d00065")
 524         resp := s.testRequest(req).Result()
 525
 526         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 527         var col arvados.Collection
 528         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
 529         c.Check(col.PortableDataHash, check.Equals, arvadostest.UserAgreementPDH)
 530         c.Check(col.ManifestText, check.Matches,
 531                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+A[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
 532 `)
 533 }
 534
 535 func (s *FederationSuite) TestSaltedTokenGetCollectionByPDHError(c *check.C) {
 536         arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
 537
 538         req := httptest.NewRequest("GET", "/arvados/v1/collections/99999999999999999999999999999999+99", nil)
 539         req.Header.Set("Authorization", "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/282d7d172b6cfdce364c5ed12ddf7417b2d00065")
 540         resp := s.testRequest(req).Result()
 541
 542         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 543 }
 544
 545 func (s *FederationSuite) TestGetRemoteContainerRequest(c *check.C) {
 546         defer s.localServiceReturns404(c).Close()
 547         req := httptest.NewRequest("GET", "/arvados/v1/container_requests/"+arvadostest.QueuedContainerRequestUUID, nil)
 548         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 549         resp := s.testRequest(req).Result()
 550         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 551         var cr arvados.ContainerRequest
 552         c.Check(json.NewDecoder(resp.Body).Decode(&cr), check.IsNil)
 553         c.Check(cr.UUID, check.Equals, arvadostest.QueuedContainerRequestUUID)
 554         c.Check(cr.Priority, check.Equals, 1)
 555 }
 556
 557 func (s *FederationSuite) TestUpdateRemoteContainerRequest(c *check.C) {
 558         defer s.localServiceReturns404(c).Close()
 559         setPri := func(pri int) {
 560                 req := httptest.NewRequest("PATCH", "/arvados/v1/container_requests/"+arvadostest.QueuedContainerRequestUUID,
 561                         strings.NewReader(fmt.Sprintf(`{"container_request": {"priority": %d}}`, pri)))
 562                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 563                 req.Header.Set("Content-type", "application/json")
 564                 resp := s.testRequest(req).Result()
 565                 c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 566                 var cr arvados.ContainerRequest
 567                 c.Check(json.NewDecoder(resp.Body).Decode(&cr), check.IsNil)
 568                 c.Check(cr.UUID, check.Equals, arvadostest.QueuedContainerRequestUUID)
 569                 c.Check(cr.Priority, check.Equals, pri)
 570         }
 571         setPri(696)
 572         setPri(1) // Reset fixture so side effect doesn't break other tests.
 573 }
 574
 575 func (s *FederationSuite) TestCreateRemoteContainerRequest(c *check.C) {
 576         defer s.localServiceReturns404(c).Close()
 577         // pass cluster_id via query parameter, this allows arvados-controller
 578         // to avoid parsing the body
 579         req := httptest.NewRequest("POST", "/arvados/v1/container_requests?cluster_id=zzzzz",
 580                 strings.NewReader(`{
 581   "container_request": {
 582     "name": "hello world",
 583     "state": "Uncommitted",
 584     "output_path": "/",
 585     "container_image": "123",
 586     "command": ["abc"]
 587   }
 588 }
 589 `))
 590         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 591         req.Header.Set("Content-type", "application/json")
 592         resp := s.testRequest(req).Result()
 593         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 594         var cr arvados.ContainerRequest
 595         c.Check(json.NewDecoder(resp.Body).Decode(&cr), check.IsNil)
 596         c.Check(cr.Name, check.Equals, "hello world")
 597         c.Check(strings.HasPrefix(cr.UUID, "zzzzz-"), check.Equals, true)
 598 }
 599
 600 func (s *FederationSuite) TestCreateRemoteContainerRequestCheckRuntimeToken(c *check.C) {
 601         // Send request to zmock and check that outgoing request has
 602         // runtime_token set with a new random v2 token.
 603
 604         defer s.localServiceReturns404(c).Close()
 605         // pass cluster_id via query parameter, this allows arvados-controller
 606         // to avoid parsing the body
 607         req := httptest.NewRequest("POST", "/arvados/v1/container_requests?cluster_id=zmock",
 608                 strings.NewReader(`{
 609   "container_request": {
 610     "name": "hello world",
 611     "state": "Uncommitted",
 612     "output_path": "/",
 613     "container_image": "123",
 614     "command": ["abc"]
 615   }
 616 }
 617 `))
 618         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
 619         req.Header.Set("Content-type", "application/json")
 620
 621         arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
 622         s.testHandler.Cluster.ClusterID = "zzzzz"
 623
 624         resp := s.testRequest(req).Result()
 625         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 626         var cr struct {
 627                 arvados.ContainerRequest `json:"container_request"`
 628         }
 629         c.Check(json.NewDecoder(s.remoteMockRequests[0].Body).Decode(&cr), check.IsNil)
 630         c.Check(strings.HasPrefix(cr.ContainerRequest.RuntimeToken, "v2/zzzzz-gj3su-"), check.Equals, true)
 631         c.Check(cr.ContainerRequest.RuntimeToken, check.Not(check.Equals), arvadostest.ActiveTokenV2)
 632 }
 633
 634 func (s *FederationSuite) TestCreateRemoteContainerRequestCheckSetRuntimeToken(c *check.C) {
 635         // Send request to zmock and check that outgoing request has
 636         // runtime_token set with the explicitly provided token.
 637
 638         defer s.localServiceReturns404(c).Close()
 639         // pass cluster_id via query parameter, this allows arvados-controller
 640         // to avoid parsing the body
 641         req := httptest.NewRequest("POST", "/arvados/v1/container_requests?cluster_id=zmock",
 642                 strings.NewReader(`{
 643   "container_request": {
 644     "name": "hello world",
 645     "state": "Uncommitted",
 646     "output_path": "/",
 647     "container_image": "123",
 648     "command": ["abc"],
 649     "runtime_token": "xyz"
 650   }
 651 }
 652 `))
 653         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 654         req.Header.Set("Content-type", "application/json")
 655         resp := s.testRequest(req).Result()
 656         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 657         var cr struct {
 658                 arvados.ContainerRequest `json:"container_request"`
 659         }
 660         c.Check(json.NewDecoder(s.remoteMockRequests[0].Body).Decode(&cr), check.IsNil)
 661         c.Check(cr.ContainerRequest.RuntimeToken, check.Equals, "xyz")
 662 }
 663
 664 func (s *FederationSuite) TestCreateRemoteContainerRequestRuntimeTokenFromAuth(c *check.C) {
 665         // Send request to zmock and check that outgoing request has
 666         // runtime_token set using the Auth token because the user is remote.
 667
 668         defer s.localServiceReturns404(c).Close()
 669         // pass cluster_id via query parameter, this allows arvados-controller
 670         // to avoid parsing the body
 671         req := httptest.NewRequest("POST", "/arvados/v1/container_requests?cluster_id=zmock",
 672                 strings.NewReader(`{
 673   "container_request": {
 674     "name": "hello world",
 675     "state": "Uncommitted",
 676     "output_path": "/",
 677     "container_image": "123",
 678     "command": ["abc"]
 679   }
 680 }
 681 `))
 682         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2+"/zzzzz-dz642-parentcontainer")
 683         req.Header.Set("Content-type", "application/json")
 684         resp := s.testRequest(req).Result()
 685         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 686         var cr struct {
 687                 arvados.ContainerRequest `json:"container_request"`
 688         }
 689         c.Check(json.NewDecoder(s.remoteMockRequests[0].Body).Decode(&cr), check.IsNil)
 690         c.Check(cr.ContainerRequest.RuntimeToken, check.Equals, arvadostest.ActiveTokenV2)
 691 }
 692
 693 func (s *FederationSuite) TestCreateRemoteContainerRequestError(c *check.C) {
 694         defer s.localServiceReturns404(c).Close()
 695         // pass cluster_id via query parameter, this allows arvados-controller
 696         // to avoid parsing the body
 697         req := httptest.NewRequest("POST", "/arvados/v1/container_requests?cluster_id=zz404",
 698                 strings.NewReader(`{
 699   "container_request": {
 700     "name": "hello world",
 701     "state": "Uncommitted",
 702     "output_path": "/",
 703     "container_image": "123",
 704     "command": ["abc"]
 705   }
 706 }
 707 `))
 708         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 709         req.Header.Set("Content-type", "application/json")
 710         resp := s.testRequest(req).Result()
 711         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 712 }
 713
 714 func (s *FederationSuite) TestGetRemoteContainer(c *check.C) {
 715         defer s.localServiceReturns404(c).Close()
 716         req := httptest.NewRequest("GET", "/arvados/v1/containers/"+arvadostest.QueuedContainerUUID, nil)
 717         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 718         resp := s.testRequest(req)
 719         c.Check(resp.Code, check.Equals, http.StatusOK)
 720         var cn arvados.Container
 721         c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
 722         c.Check(cn.UUID, check.Equals, arvadostest.QueuedContainerUUID)
 723 }
 724
 725 func (s *FederationSuite) TestListRemoteContainer(c *check.C) {
 726         defer s.localServiceReturns404(c).Close()
 727         req := httptest.NewRequest("GET", "/arvados/v1/containers?count=none&filters="+
 728                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v"]]]`, arvadostest.QueuedContainerUUID)), nil)
 729         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 730         resp := s.testRequest(req).Result()
 731         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 732         var cn arvados.ContainerList
 733         c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
 734         c.Assert(cn.Items, check.HasLen, 1)
 735         c.Check(cn.Items[0].UUID, check.Equals, arvadostest.QueuedContainerUUID)
 736 }
 737
 738 func (s *FederationSuite) TestListMultiRemoteContainers(c *check.C) {
 739         defer s.localServiceHandler(c, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 740                 bd, _ := ioutil.ReadAll(req.Body)
 741                 c.Check(string(bd), check.Equals, `_method=GET&count=none&filters=%5B%5B%22uuid%22%2C+%22in%22%2C+%5B%22zhome-xvhdp-cr5queuedcontnr%22%5D%5D%5D&select=%5B%22uuid%22%2C+%22command%22%5D`)
 742                 w.WriteHeader(200)
 743                 w.Write([]byte(`{"kind": "arvados#containerList", "items": [{"uuid": "zhome-xvhdp-cr5queuedcontnr", "command": ["abc"]}]}`))
 744         })).Close()
 745         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&select=%s",
 746                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
 747                         arvadostest.QueuedContainerUUID)),
 748                 url.QueryEscape(`["uuid", "command"]`)),
 749                 nil)
 750         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 751         resp := s.testRequest(req).Result()
 752         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 753         var cn arvados.ContainerList
 754         c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
 755         c.Check(cn.Items, check.HasLen, 2)
 756         mp := make(map[string]arvados.Container)
 757         for _, cr := range cn.Items {
 758                 mp[cr.UUID] = cr
 759         }
 760         c.Check(mp[arvadostest.QueuedContainerUUID].Command, check.DeepEquals, []string{"echo", "hello"})
 761         c.Check(mp[arvadostest.QueuedContainerUUID].ContainerImage, check.Equals, "")
 762         c.Check(mp["zhome-xvhdp-cr5queuedcontnr"].Command, check.DeepEquals, []string{"abc"})
 763         c.Check(mp["zhome-xvhdp-cr5queuedcontnr"].ContainerImage, check.Equals, "")
 764 }
 765
 766 func (s *FederationSuite) TestListMultiRemoteContainerError(c *check.C) {
 767         defer s.localServiceReturns404(c).Close()
 768         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&select=%s",
 769                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
 770                         arvadostest.QueuedContainerUUID)),
 771                 url.QueryEscape(`["uuid", "command"]`)),
 772                 nil)
 773         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 774         resp := s.testRequest(req).Result()
 775         c.Check(resp.StatusCode, check.Equals, http.StatusBadGateway)
 776         s.checkJSONErrorMatches(c, resp, `error fetching from zhome \(404 Not Found\): EOF`)
 777 }
 778
 779 func (s *FederationSuite) TestListMultiRemoteContainersPaged(c *check.C) {
 780
 781         callCount := 0
 782         defer s.localServiceHandler(c, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 783                 bd, _ := ioutil.ReadAll(req.Body)
 784                 if callCount == 0 {
 785                         c.Check(string(bd), check.Equals, `_method=GET&count=none&filters=%5B%5B%22uuid%22%2C+%22in%22%2C+%5B%22zhome-xvhdp-cr5queuedcontnr%22%2C%22zhome-xvhdp-cr6queuedcontnr%22%5D%5D%5D`)
 786                         w.WriteHeader(200)
 787                         w.Write([]byte(`{"kind": "arvados#containerList", "items": [{"uuid": "zhome-xvhdp-cr5queuedcontnr", "command": ["abc"]}]}`))
 788                 } else if callCount == 1 {
 789                         c.Check(string(bd), check.Equals, `_method=GET&count=none&filters=%5B%5B%22uuid%22%2C+%22in%22%2C+%5B%22zhome-xvhdp-cr6queuedcontnr%22%5D%5D%5D`)
 790                         w.WriteHeader(200)
 791                         w.Write([]byte(`{"kind": "arvados#containerList", "items": [{"uuid": "zhome-xvhdp-cr6queuedcontnr", "command": ["efg"]}]}`))
 792                 }
 793                 callCount += 1
 794         })).Close()
 795         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s",
 796                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr", "zhome-xvhdp-cr6queuedcontnr"]]]`,
 797                         arvadostest.QueuedContainerUUID))),
 798                 nil)
 799         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 800         resp := s.testRequest(req).Result()
 801         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 802         c.Check(callCount, check.Equals, 2)
 803         var cn arvados.ContainerList
 804         c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
 805         c.Check(cn.Items, check.HasLen, 3)
 806         mp := make(map[string]arvados.Container)
 807         for _, cr := range cn.Items {
 808                 mp[cr.UUID] = cr
 809         }
 810         c.Check(mp[arvadostest.QueuedContainerUUID].Command, check.DeepEquals, []string{"echo", "hello"})
 811         c.Check(mp["zhome-xvhdp-cr5queuedcontnr"].Command, check.DeepEquals, []string{"abc"})
 812         c.Check(mp["zhome-xvhdp-cr6queuedcontnr"].Command, check.DeepEquals, []string{"efg"})
 813 }
 814
 815 func (s *FederationSuite) TestListMultiRemoteContainersMissing(c *check.C) {
 816
 817         callCount := 0
 818         defer s.localServiceHandler(c, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 819                 bd, _ := ioutil.ReadAll(req.Body)
 820                 if callCount == 0 {
 821                         c.Check(string(bd), check.Equals, `_method=GET&count=none&filters=%5B%5B%22uuid%22%2C+%22in%22%2C+%5B%22zhome-xvhdp-cr5queuedcontnr%22%2C%22zhome-xvhdp-cr6queuedcontnr%22%5D%5D%5D`)
 822                         w.WriteHeader(200)
 823                         w.Write([]byte(`{"kind": "arvados#containerList", "items": [{"uuid": "zhome-xvhdp-cr6queuedcontnr", "command": ["efg"]}]}`))
 824                 } else if callCount == 1 {
 825                         c.Check(string(bd), check.Equals, `_method=GET&count=none&filters=%5B%5B%22uuid%22%2C+%22in%22%2C+%5B%22zhome-xvhdp-cr5queuedcontnr%22%5D%5D%5D`)
 826                         w.WriteHeader(200)
 827                         w.Write([]byte(`{"kind": "arvados#containerList", "items": []}`))
 828                 }
 829                 callCount += 1
 830         })).Close()
 831         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s",
 832                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr", "zhome-xvhdp-cr6queuedcontnr"]]]`,
 833                         arvadostest.QueuedContainerUUID))),
 834                 nil)
 835         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 836         resp := s.testRequest(req).Result()
 837         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 838         c.Check(callCount, check.Equals, 2)
 839         var cn arvados.ContainerList
 840         c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
 841         c.Check(cn.Items, check.HasLen, 2)
 842         mp := make(map[string]arvados.Container)
 843         for _, cr := range cn.Items {
 844                 mp[cr.UUID] = cr
 845         }
 846         c.Check(mp[arvadostest.QueuedContainerUUID].Command, check.DeepEquals, []string{"echo", "hello"})
 847         c.Check(mp["zhome-xvhdp-cr6queuedcontnr"].Command, check.DeepEquals, []string{"efg"})
 848 }
 849
 850 func (s *FederationSuite) TestListMultiRemoteContainerPageSizeError(c *check.C) {
 851         s.testHandler.Cluster.API.MaxItemsPerResponse = 1
 852         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s",
 853                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
 854                         arvadostest.QueuedContainerUUID))),
 855                 nil)
 856         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 857         resp := s.testRequest(req).Result()
 858         c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
 859         s.checkJSONErrorMatches(c, resp, `Federated multi-object request for 2 objects which is more than max page size 1.`)
 860 }
 861
 862 func (s *FederationSuite) TestListMultiRemoteContainerLimitError(c *check.C) {
 863         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&limit=1",
 864                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
 865                         arvadostest.QueuedContainerUUID))),
 866                 nil)
 867         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 868         resp := s.testRequest(req).Result()
 869         c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
 870         s.checkJSONErrorMatches(c, resp, `Federated multi-object may not provide 'limit', 'offset' or 'order'.`)
 871 }
 872
 873 func (s *FederationSuite) TestListMultiRemoteContainerOffsetError(c *check.C) {
 874         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&offset=1",
 875                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
 876                         arvadostest.QueuedContainerUUID))),
 877                 nil)
 878         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 879         resp := s.testRequest(req).Result()
 880         c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
 881         s.checkJSONErrorMatches(c, resp, `Federated multi-object may not provide 'limit', 'offset' or 'order'.`)
 882 }
 883
 884 func (s *FederationSuite) TestListMultiRemoteContainerOrderError(c *check.C) {
 885         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&order=uuid",
 886                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
 887                         arvadostest.QueuedContainerUUID))),
 888                 nil)
 889         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 890         resp := s.testRequest(req).Result()
 891         c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
 892         s.checkJSONErrorMatches(c, resp, `Federated multi-object may not provide 'limit', 'offset' or 'order'.`)
 893 }
 894
 895 func (s *FederationSuite) TestListMultiRemoteContainerSelectError(c *check.C) {
 896         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&select=%s",
 897                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
 898                         arvadostest.QueuedContainerUUID)),
 899                 url.QueryEscape(`["command"]`)),
 900                 nil)
 901         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 902         resp := s.testRequest(req).Result()
 903         c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
 904         s.checkJSONErrorMatches(c, resp, `Federated multi-object request must include 'uuid' in 'select'`)
 905 }