lib/controller/federation_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package controller
   6
   7 import (
   8         "bytes"
   9         "encoding/json"
  10         "fmt"
  11         "io"
  12         "io/ioutil"
  13         "net/http"
  14         "net/http/httptest"
  15         "net/url"
  16         "os"
  17         "strings"
  18         "time"
  19
  20         "git.curoverse.com/arvados.git/sdk/go/arvados"
  21         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  22         "git.curoverse.com/arvados.git/sdk/go/ctxlog"
  23         "git.curoverse.com/arvados.git/sdk/go/httpserver"
  24         "git.curoverse.com/arvados.git/sdk/go/keepclient"
  25         "github.com/sirupsen/logrus"
  26         check "gopkg.in/check.v1"
  27 )
  28
  29 // Gocheck boilerplate
  30 var _ = check.Suite(&FederationSuite{})
  31
  32 type FederationSuite struct {
  33         log logrus.FieldLogger
  34         // testServer and testHandler are the controller being tested,
  35         // "zhome".
  36         testServer  *httpserver.Server
  37         testHandler *Handler
  38         // remoteServer ("zzzzz") forwards requests to the Rails API
  39         // provided by the integration test environment.
  40         remoteServer *httpserver.Server
  41         // remoteMock ("zmock") appends each incoming request to
  42         // remoteMockRequests, and returns an empty 200 response.
  43         remoteMock         *httpserver.Server
  44         remoteMockRequests []http.Request
  45 }
  46
  47 func (s *FederationSuite) SetUpTest(c *check.C) {
  48         s.log = ctxlog.TestLogger(c)
  49
  50         s.remoteServer = newServerFromIntegrationTestEnv(c)
  51         c.Assert(s.remoteServer.Start(), check.IsNil)
  52
  53         s.remoteMock = newServerFromIntegrationTestEnv(c)
  54         s.remoteMock.Server.Handler = http.HandlerFunc(s.remoteMockHandler)
  55         c.Assert(s.remoteMock.Start(), check.IsNil)
  56
  57         nodeProfile := arvados.NodeProfile{
  58                 Controller: arvados.SystemServiceInstance{Listen: ":"},
  59                 RailsAPI:   arvados.SystemServiceInstance{Listen: ":1"}, // local reqs will error "connection refused"
  60         }
  61         s.testHandler = &Handler{Cluster: &arvados.Cluster{
  62                 ClusterID:  "zhome",
  63                 PostgreSQL: integrationTestCluster().PostgreSQL,
  64                 NodeProfiles: map[string]arvados.NodeProfile{
  65                         "*": nodeProfile,
  66                 },
  67                 RequestLimits: arvados.RequestLimits{
  68                         MaxItemsPerResponse:            1000,
  69                         MultiClusterRequestConcurrency: 4,
  70                 },
  71         }, NodeProfile: &nodeProfile}
  72         s.testServer = newServerFromIntegrationTestEnv(c)
  73         s.testServer.Server.Handler = httpserver.AddRequestIDs(httpserver.LogRequests(s.log, s.testHandler))
  74
  75         s.testHandler.Cluster.RemoteClusters = map[string]arvados.RemoteCluster{
  76                 "zzzzz": {
  77                         Host:   s.remoteServer.Addr,
  78                         Proxy:  true,
  79                         Scheme: "http",
  80                 },
  81                 "zmock": {
  82                         Host:   s.remoteMock.Addr,
  83                         Proxy:  true,
  84                         Scheme: "http",
  85                 },
  86         }
  87
  88         c.Assert(s.testServer.Start(), check.IsNil)
  89
  90         s.remoteMockRequests = nil
  91 }
  92
  93 func (s *FederationSuite) remoteMockHandler(w http.ResponseWriter, req *http.Request) {
  94         b := &bytes.Buffer{}
  95         io.Copy(b, req.Body)
  96         req.Body.Close()
  97         req.Body = ioutil.NopCloser(b)
  98         s.remoteMockRequests = append(s.remoteMockRequests, *req)
  99 }
 100
 101 func (s *FederationSuite) TearDownTest(c *check.C) {
 102         if s.remoteServer != nil {
 103                 s.remoteServer.Close()
 104         }
 105         if s.testServer != nil {
 106                 s.testServer.Close()
 107         }
 108 }
 109
 110 func (s *FederationSuite) testRequest(req *http.Request) *http.Response {
 111         resp := httptest.NewRecorder()
 112         s.testServer.Server.Handler.ServeHTTP(resp, req)
 113         return resp.Result()
 114 }
 115
 116 func (s *FederationSuite) TestLocalRequest(c *check.C) {
 117         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zhome-", 1), nil)
 118         resp := s.testRequest(req)
 119         s.checkHandledLocally(c, resp)
 120 }
 121
 122 func (s *FederationSuite) checkHandledLocally(c *check.C, resp *http.Response) {
 123         // Our "home" controller can't handle local requests because
 124         // it doesn't have its own stub/test Rails API, so we rely on
 125         // "connection refused" to indicate the controller tried to
 126         // proxy the request to its local Rails API.
 127         c.Check(resp.StatusCode, check.Equals, http.StatusBadGateway)
 128         s.checkJSONErrorMatches(c, resp, `.*connection refused`)
 129 }
 130
 131 func (s *FederationSuite) TestNoAuth(c *check.C) {
 132         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
 133         resp := s.testRequest(req)
 134         c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
 135         s.checkJSONErrorMatches(c, resp, `Not logged in`)
 136 }
 137
 138 func (s *FederationSuite) TestBadAuth(c *check.C) {
 139         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
 140         req.Header.Set("Authorization", "Bearer aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
 141         resp := s.testRequest(req)
 142         c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
 143         s.checkJSONErrorMatches(c, resp, `Not logged in`)
 144 }
 145
 146 func (s *FederationSuite) TestNoAccess(c *check.C) {
 147         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
 148         req.Header.Set("Authorization", "Bearer "+arvadostest.SpectatorToken)
 149         resp := s.testRequest(req)
 150         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 151         s.checkJSONErrorMatches(c, resp, `.*not found`)
 152 }
 153
 154 func (s *FederationSuite) TestGetUnknownRemote(c *check.C) {
 155         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zz404-", 1), nil)
 156         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 157         resp := s.testRequest(req)
 158         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 159         s.checkJSONErrorMatches(c, resp, `.*no proxy available for cluster zz404`)
 160 }
 161
 162 func (s *FederationSuite) TestRemoteError(c *check.C) {
 163         rc := s.testHandler.Cluster.RemoteClusters["zzzzz"]
 164         rc.Scheme = "https"
 165         s.testHandler.Cluster.RemoteClusters["zzzzz"] = rc
 166
 167         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
 168         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 169         resp := s.testRequest(req)
 170         c.Check(resp.StatusCode, check.Equals, http.StatusBadGateway)
 171         s.checkJSONErrorMatches(c, resp, `.*HTTP response to HTTPS client`)
 172 }
 173
 174 func (s *FederationSuite) TestGetRemoteWorkflow(c *check.C) {
 175         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
 176         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 177         resp := s.testRequest(req)
 178         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 179         var wf arvados.Workflow
 180         c.Check(json.NewDecoder(resp.Body).Decode(&wf), check.IsNil)
 181         c.Check(wf.UUID, check.Equals, arvadostest.WorkflowWithDefinitionYAMLUUID)
 182         c.Check(wf.OwnerUUID, check.Equals, arvadostest.ActiveUserUUID)
 183 }
 184
 185 func (s *FederationSuite) TestOptionsMethod(c *check.C) {
 186         req := httptest.NewRequest("OPTIONS", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
 187         req.Header.Set("Origin", "https://example.com")
 188         resp := s.testRequest(req)
 189         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 190         body, err := ioutil.ReadAll(resp.Body)
 191         c.Check(err, check.IsNil)
 192         c.Check(string(body), check.Equals, "")
 193         c.Check(resp.Header.Get("Access-Control-Allow-Origin"), check.Equals, "*")
 194         for _, hdr := range []string{"Authorization", "Content-Type"} {
 195                 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), check.Matches, ".*"+hdr+".*")
 196         }
 197         for _, method := range []string{"GET", "HEAD", "PUT", "POST", "DELETE"} {
 198                 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), check.Matches, ".*"+method+".*")
 199         }
 200 }
 201
 202 func (s *FederationSuite) TestRemoteWithTokenInQuery(c *check.C) {
 203         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1)+"?api_token="+arvadostest.ActiveToken, nil)
 204         s.testRequest(req)
 205         c.Assert(s.remoteMockRequests, check.HasLen, 1)
 206         pr := s.remoteMockRequests[0]
 207         // Token is salted and moved from query to Authorization header.
 208         c.Check(pr.URL.String(), check.Not(check.Matches), `.*api_token=.*`)
 209         c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/7fd31b61f39c0e82a4155592163218272cedacdc")
 210 }
 211
 212 func (s *FederationSuite) TestLocalTokenSalted(c *check.C) {
 213         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1), nil)
 214         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 215         s.testRequest(req)
 216         c.Assert(s.remoteMockRequests, check.HasLen, 1)
 217         pr := s.remoteMockRequests[0]
 218         // The salted token here has a "zzzzz-" UUID instead of a
 219         // "ztest-" UUID because ztest's local database has the
 220         // "zzzzz-" test fixtures. The "secret" part is HMAC(sha1,
 221         // arvadostest.ActiveToken, "zmock") = "7fd3...".
 222         c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/7fd31b61f39c0e82a4155592163218272cedacdc")
 223 }
 224
 225 func (s *FederationSuite) TestRemoteTokenNotSalted(c *check.C) {
 226         // remoteToken can be any v1 token that doesn't appear in
 227         // ztest's local db.
 228         remoteToken := "abcdef00000000000000000000000000000000000000000000"
 229         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1), nil)
 230         req.Header.Set("Authorization", "Bearer "+remoteToken)
 231         s.testRequest(req)
 232         c.Assert(s.remoteMockRequests, check.HasLen, 1)
 233         pr := s.remoteMockRequests[0]
 234         c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer "+remoteToken)
 235 }
 236
 237 func (s *FederationSuite) TestWorkflowCRUD(c *check.C) {
 238         wf := arvados.Workflow{
 239                 Description: "TestCRUD",
 240         }
 241         {
 242                 body := &strings.Builder{}
 243                 json.NewEncoder(body).Encode(&wf)
 244                 req := httptest.NewRequest("POST", "/arvados/v1/workflows", strings.NewReader(url.Values{
 245                         "workflow": {body.String()},
 246                 }.Encode()))
 247                 req.Header.Set("Content-type", "application/x-www-form-urlencoded")
 248                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 249                 rec := httptest.NewRecorder()
 250                 s.remoteServer.Server.Handler.ServeHTTP(rec, req) // direct to remote -- can't proxy a create req because no uuid
 251                 resp := rec.Result()
 252                 s.checkResponseOK(c, resp)
 253                 json.NewDecoder(resp.Body).Decode(&wf)
 254
 255                 defer func() {
 256                         req := httptest.NewRequest("DELETE", "/arvados/v1/workflows/"+wf.UUID, nil)
 257                         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 258                         s.remoteServer.Server.Handler.ServeHTTP(httptest.NewRecorder(), req)
 259                 }()
 260                 c.Check(wf.UUID, check.Not(check.Equals), "")
 261
 262                 c.Assert(wf.ModifiedAt, check.NotNil)
 263                 c.Logf("wf.ModifiedAt: %v", wf.ModifiedAt)
 264                 c.Check(time.Since(*wf.ModifiedAt) < time.Minute, check.Equals, true)
 265         }
 266         for _, method := range []string{"PATCH", "PUT", "POST"} {
 267                 form := url.Values{
 268                         "workflow": {`{"description": "Updated with ` + method + `"}`},
 269                 }
 270                 if method == "POST" {
 271                         form["_method"] = []string{"PATCH"}
 272                 }
 273                 req := httptest.NewRequest(method, "/arvados/v1/workflows/"+wf.UUID, strings.NewReader(form.Encode()))
 274                 req.Header.Set("Content-type", "application/x-www-form-urlencoded")
 275                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 276                 resp := s.testRequest(req)
 277                 s.checkResponseOK(c, resp)
 278                 err := json.NewDecoder(resp.Body).Decode(&wf)
 279                 c.Check(err, check.IsNil)
 280
 281                 c.Check(wf.Description, check.Equals, "Updated with "+method)
 282         }
 283         {
 284                 req := httptest.NewRequest("DELETE", "/arvados/v1/workflows/"+wf.UUID, nil)
 285                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 286                 resp := s.testRequest(req)
 287                 s.checkResponseOK(c, resp)
 288                 err := json.NewDecoder(resp.Body).Decode(&wf)
 289                 c.Check(err, check.IsNil)
 290         }
 291         {
 292                 req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+wf.UUID, nil)
 293                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 294                 resp := s.testRequest(req)
 295                 c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 296         }
 297 }
 298
 299 func (s *FederationSuite) checkResponseOK(c *check.C, resp *http.Response) {
 300         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 301         if resp.StatusCode != http.StatusOK {
 302                 body, err := ioutil.ReadAll(resp.Body)
 303                 c.Logf("... response body = %q, %v\n", body, err)
 304         }
 305 }
 306
 307 func (s *FederationSuite) checkJSONErrorMatches(c *check.C, resp *http.Response, re string) {
 308         var jresp httpserver.ErrorResponse
 309         err := json.NewDecoder(resp.Body).Decode(&jresp)
 310         c.Check(err, check.IsNil)
 311         c.Assert(jresp.Errors, check.HasLen, 1)
 312         c.Check(jresp.Errors[0], check.Matches, re)
 313 }
 314
 315 func (s *FederationSuite) localServiceHandler(c *check.C, h http.Handler) *httpserver.Server {
 316         srv := &httpserver.Server{
 317                 Server: http.Server{
 318                         Handler: h,
 319                 },
 320         }
 321
 322         c.Assert(srv.Start(), check.IsNil)
 323
 324         np := arvados.NodeProfile{
 325                 Controller: arvados.SystemServiceInstance{Listen: ":"},
 326                 RailsAPI: arvados.SystemServiceInstance{Listen: srv.Addr,
 327                         TLS: false, Insecure: true}}
 328         s.testHandler.Cluster.NodeProfiles["*"] = np
 329         s.testHandler.NodeProfile = &np
 330
 331         return srv
 332 }
 333
 334 func (s *FederationSuite) localServiceReturns404(c *check.C) *httpserver.Server {
 335         return s.localServiceHandler(c, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 336                 w.WriteHeader(404)
 337         }))
 338 }
 339
 340 func (s *FederationSuite) TestGetLocalCollection(c *check.C) {
 341         np := arvados.NodeProfile{
 342                 Controller: arvados.SystemServiceInstance{Listen: ":"},
 343                 RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"),
 344                         TLS: true, Insecure: true}}
 345         s.testHandler.Cluster.ClusterID = "zzzzz"
 346         s.testHandler.Cluster.NodeProfiles["*"] = np
 347         s.testHandler.NodeProfile = &np
 348
 349         // HTTP GET
 350
 351         req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementCollection, nil)
 352         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 353         resp := s.testRequest(req)
 354
 355         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 356         var col arvados.Collection
 357         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
 358         c.Check(col.UUID, check.Equals, arvadostest.UserAgreementCollection)
 359         c.Check(col.ManifestText, check.Matches,
 360                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+A[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
 361 `)
 362
 363         // HTTP POST with _method=GET as a form parameter
 364
 365         req = httptest.NewRequest("POST", "/arvados/v1/collections/"+arvadostest.UserAgreementCollection, bytes.NewBufferString((url.Values{
 366                 "_method": {"GET"},
 367         }).Encode()))
 368         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 369         req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
 370         resp = s.testRequest(req)
 371
 372         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 373         col = arvados.Collection{}
 374         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
 375         c.Check(col.UUID, check.Equals, arvadostest.UserAgreementCollection)
 376         c.Check(col.ManifestText, check.Matches,
 377                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+A[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
 378 `)
 379 }
 380
 381 func (s *FederationSuite) TestGetRemoteCollection(c *check.C) {
 382         defer s.localServiceReturns404(c).Close()
 383
 384         req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementCollection, nil)
 385         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 386         resp := s.testRequest(req)
 387         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 388         var col arvados.Collection
 389         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
 390         c.Check(col.UUID, check.Equals, arvadostest.UserAgreementCollection)
 391         c.Check(col.ManifestText, check.Matches,
 392                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+Rzzzzz-[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
 393 `)
 394 }
 395
 396 func (s *FederationSuite) TestGetRemoteCollectionError(c *check.C) {
 397         defer s.localServiceReturns404(c).Close()
 398
 399         req := httptest.NewRequest("GET", "/arvados/v1/collections/zzzzz-4zz18-fakefakefakefak", nil)
 400         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 401         resp := s.testRequest(req)
 402         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 403 }
 404
 405 func (s *FederationSuite) TestSignedLocatorPattern(c *check.C) {
 406         // Confirm the regular expression identifies other groups of hints correctly
 407         c.Check(keepclient.SignedLocatorRe.FindStringSubmatch(`6a4ff0499484c6c79c95cd8c566bd25f+249025+B1+C2+A05227438989d04712ea9ca1c91b556cef01d5cc7@5ba5405b+D3+E4`),
 408                 check.DeepEquals,
 409                 []string{"6a4ff0499484c6c79c95cd8c566bd25f+249025+B1+C2+A05227438989d04712ea9ca1c91b556cef01d5cc7@5ba5405b+D3+E4",
 410                         "6a4ff0499484c6c79c95cd8c566bd25f",
 411                         "+249025",
 412                         "+B1+C2", "+C2",
 413                         "+A05227438989d04712ea9ca1c91b556cef01d5cc7@5ba5405b",
 414                         "05227438989d04712ea9ca1c91b556cef01d5cc7", "5ba5405b",
 415                         "+D3+E4", "+E4"})
 416 }
 417
 418 func (s *FederationSuite) TestGetLocalCollectionByPDH(c *check.C) {
 419         np := arvados.NodeProfile{
 420                 Controller: arvados.SystemServiceInstance{Listen: ":"},
 421                 RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"),
 422                         TLS: true, Insecure: true}}
 423         s.testHandler.Cluster.NodeProfiles["*"] = np
 424         s.testHandler.NodeProfile = &np
 425
 426         req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
 427         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 428         resp := s.testRequest(req)
 429
 430         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 431         var col arvados.Collection
 432         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
 433         c.Check(col.PortableDataHash, check.Equals, arvadostest.UserAgreementPDH)
 434         c.Check(col.ManifestText, check.Matches,
 435                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+A[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
 436 `)
 437 }
 438
 439 func (s *FederationSuite) TestGetRemoteCollectionByPDH(c *check.C) {
 440         defer s.localServiceReturns404(c).Close()
 441
 442         req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
 443         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 444         resp := s.testRequest(req)
 445
 446         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 447
 448         var col arvados.Collection
 449         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
 450         c.Check(col.PortableDataHash, check.Equals, arvadostest.UserAgreementPDH)
 451         c.Check(col.ManifestText, check.Matches,
 452                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+Rzzzzz-[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
 453 `)
 454 }
 455
 456 func (s *FederationSuite) TestGetCollectionByPDHError(c *check.C) {
 457         defer s.localServiceReturns404(c).Close()
 458
 459         req := httptest.NewRequest("GET", "/arvados/v1/collections/99999999999999999999999999999999+99", nil)
 460         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 461
 462         resp := s.testRequest(req)
 463         defer resp.Body.Close()
 464
 465         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 466 }
 467
 468 func (s *FederationSuite) TestGetCollectionByPDHErrorBadHash(c *check.C) {
 469         defer s.localServiceReturns404(c).Close()
 470
 471         srv2 := &httpserver.Server{
 472                 Server: http.Server{
 473                         Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 474                                 w.WriteHeader(200)
 475                                 // Return a collection where the hash
 476                                 // of the manifest text doesn't match
 477                                 // PDH that was requested.
 478                                 var col arvados.Collection
 479                                 col.PortableDataHash = "99999999999999999999999999999999+99"
 480                                 col.ManifestText = `. 6a4ff0499484c6c79c95cd8c566bd25f\+249025 0:249025:GNU_General_Public_License,_version_3.pdf
 481 `
 482                                 enc := json.NewEncoder(w)
 483                                 enc.Encode(col)
 484                         }),
 485                 },
 486         }
 487
 488         c.Assert(srv2.Start(), check.IsNil)
 489         defer srv2.Close()
 490
 491         // Direct zzzzz to service that returns a 200 result with a bogus manifest_text
 492         s.testHandler.Cluster.RemoteClusters["zzzzz"] = arvados.RemoteCluster{
 493                 Host:   srv2.Addr,
 494                 Proxy:  true,
 495                 Scheme: "http",
 496         }
 497
 498         req := httptest.NewRequest("GET", "/arvados/v1/collections/99999999999999999999999999999999+99", nil)
 499         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 500
 501         resp := s.testRequest(req)
 502         defer resp.Body.Close()
 503
 504         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 505 }
 506
 507 func (s *FederationSuite) TestSaltedTokenGetCollectionByPDH(c *check.C) {
 508         np := arvados.NodeProfile{
 509                 Controller: arvados.SystemServiceInstance{Listen: ":"},
 510                 RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"),
 511                         TLS: true, Insecure: true}}
 512         s.testHandler.Cluster.NodeProfiles["*"] = np
 513         s.testHandler.NodeProfile = &np
 514
 515         req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
 516         req.Header.Set("Authorization", "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/282d7d172b6cfdce364c5ed12ddf7417b2d00065")
 517         resp := s.testRequest(req)
 518
 519         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 520         var col arvados.Collection
 521         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
 522         c.Check(col.PortableDataHash, check.Equals, arvadostest.UserAgreementPDH)
 523         c.Check(col.ManifestText, check.Matches,
 524                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+A[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
 525 `)
 526 }
 527
 528 func (s *FederationSuite) TestSaltedTokenGetCollectionByPDHError(c *check.C) {
 529         np := arvados.NodeProfile{
 530                 Controller: arvados.SystemServiceInstance{Listen: ":"},
 531                 RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"),
 532                         TLS: true, Insecure: true}}
 533         s.testHandler.Cluster.NodeProfiles["*"] = np
 534         s.testHandler.NodeProfile = &np
 535
 536         req := httptest.NewRequest("GET", "/arvados/v1/collections/99999999999999999999999999999999+99", nil)
 537         req.Header.Set("Authorization", "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/282d7d172b6cfdce364c5ed12ddf7417b2d00065")
 538         resp := s.testRequest(req)
 539
 540         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 541 }
 542
 543 func (s *FederationSuite) TestGetRemoteContainerRequest(c *check.C) {
 544         defer s.localServiceReturns404(c).Close()
 545         req := httptest.NewRequest("GET", "/arvados/v1/container_requests/"+arvadostest.QueuedContainerRequestUUID, nil)
 546         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 547         resp := s.testRequest(req)
 548         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 549         var cr arvados.ContainerRequest
 550         c.Check(json.NewDecoder(resp.Body).Decode(&cr), check.IsNil)
 551         c.Check(cr.UUID, check.Equals, arvadostest.QueuedContainerRequestUUID)
 552         c.Check(cr.Priority, check.Equals, 1)
 553 }
 554
 555 func (s *FederationSuite) TestUpdateRemoteContainerRequest(c *check.C) {
 556         defer s.localServiceReturns404(c).Close()
 557         req := httptest.NewRequest("PATCH", "/arvados/v1/container_requests/"+arvadostest.QueuedContainerRequestUUID,
 558                 strings.NewReader(`{"container_request": {"priority": 696}}`))
 559         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 560         req.Header.Set("Content-type", "application/json")
 561         resp := s.testRequest(req)
 562         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 563         var cr arvados.ContainerRequest
 564         c.Check(json.NewDecoder(resp.Body).Decode(&cr), check.IsNil)
 565         c.Check(cr.UUID, check.Equals, arvadostest.QueuedContainerRequestUUID)
 566         c.Check(cr.Priority, check.Equals, 696)
 567 }
 568
 569 func (s *FederationSuite) TestCreateRemoteContainerRequest(c *check.C) {
 570         defer s.localServiceReturns404(c).Close()
 571         // pass cluster_id via query parameter, this allows arvados-controller
 572         // to avoid parsing the body
 573         req := httptest.NewRequest("POST", "/arvados/v1/container_requests?cluster_id=zzzzz",
 574                 strings.NewReader(`{
 575   "container_request": {
 576     "name": "hello world",
 577     "state": "Uncommitted",
 578     "output_path": "/",
 579     "container_image": "123",
 580     "command": ["abc"]
 581   }
 582 }
 583 `))
 584         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 585         req.Header.Set("Content-type", "application/json")
 586         resp := s.testRequest(req)
 587         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 588         var cr arvados.ContainerRequest
 589         c.Check(json.NewDecoder(resp.Body).Decode(&cr), check.IsNil)
 590         c.Check(cr.Name, check.Equals, "hello world")
 591         c.Check(strings.HasPrefix(cr.UUID, "zzzzz-"), check.Equals, true)
 592 }
 593
 594 func (s *FederationSuite) TestCreateRemoteContainerRequestCheckRuntimeToken(c *check.C) {
 595         // Send request to zmock and check that outgoing request has
 596         // runtime_token set with a new random v2 token.
 597
 598         defer s.localServiceReturns404(c).Close()
 599         // pass cluster_id via query parameter, this allows arvados-controller
 600         // to avoid parsing the body
 601         req := httptest.NewRequest("POST", "/arvados/v1/container_requests?cluster_id=zmock",
 602                 strings.NewReader(`{
 603   "container_request": {
 604     "name": "hello world",
 605     "state": "Uncommitted",
 606     "output_path": "/",
 607     "container_image": "123",
 608     "command": ["abc"]
 609   }
 610 }
 611 `))
 612         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
 613         req.Header.Set("Content-type", "application/json")
 614
 615         np := arvados.NodeProfile{
 616                 Controller: arvados.SystemServiceInstance{Listen: ":"},
 617                 RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"),
 618                         TLS: true, Insecure: true}}
 619         s.testHandler.Cluster.ClusterID = "zzzzz"
 620         s.testHandler.Cluster.NodeProfiles["*"] = np
 621         s.testHandler.NodeProfile = &np
 622
 623         resp := s.testRequest(req)
 624         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 625         var cr struct {
 626                 arvados.ContainerRequest `json:"container_request"`
 627         }
 628         c.Check(json.NewDecoder(s.remoteMockRequests[0].Body).Decode(&cr), check.IsNil)
 629         c.Check(strings.HasPrefix(cr.ContainerRequest.RuntimeToken, "v2/zzzzz-gj3su-"), check.Equals, true)
 630         c.Check(cr.ContainerRequest.RuntimeToken, check.Not(check.Equals), arvadostest.ActiveTokenV2)
 631 }
 632
 633 func (s *FederationSuite) TestCreateRemoteContainerRequestCheckSetRuntimeToken(c *check.C) {
 634         // Send request to zmock and check that outgoing request has
 635         // runtime_token set with the explicitly provided token.
 636
 637         defer s.localServiceReturns404(c).Close()
 638         // pass cluster_id via query parameter, this allows arvados-controller
 639         // to avoid parsing the body
 640         req := httptest.NewRequest("POST", "/arvados/v1/container_requests?cluster_id=zmock",
 641                 strings.NewReader(`{
 642   "container_request": {
 643     "name": "hello world",
 644     "state": "Uncommitted",
 645     "output_path": "/",
 646     "container_image": "123",
 647     "command": ["abc"],
 648     "runtime_token": "xyz"
 649   }
 650 }
 651 `))
 652         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 653         req.Header.Set("Content-type", "application/json")
 654         resp := s.testRequest(req)
 655         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 656         var cr struct {
 657                 arvados.ContainerRequest `json:"container_request"`
 658         }
 659         c.Check(json.NewDecoder(s.remoteMockRequests[0].Body).Decode(&cr), check.IsNil)
 660         c.Check(cr.ContainerRequest.RuntimeToken, check.Equals, "xyz")
 661 }
 662
 663 func (s *FederationSuite) TestCreateRemoteContainerRequestRuntimeTokenFromAuth(c *check.C) {
 664         // Send request to zmock and check that outgoing request has
 665         // runtime_token set using the Auth token because the user is remote.
 666
 667         defer s.localServiceReturns404(c).Close()
 668         // pass cluster_id via query parameter, this allows arvados-controller
 669         // to avoid parsing the body
 670         req := httptest.NewRequest("POST", "/arvados/v1/container_requests?cluster_id=zmock",
 671                 strings.NewReader(`{
 672   "container_request": {
 673     "name": "hello world",
 674     "state": "Uncommitted",
 675     "output_path": "/",
 676     "container_image": "123",
 677     "command": ["abc"]
 678   }
 679 }
 680 `))
 681         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2+"/zzzzz-dz642-parentcontainer")
 682         req.Header.Set("Content-type", "application/json")
 683         resp := s.testRequest(req)
 684         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 685         var cr struct {
 686                 arvados.ContainerRequest `json:"container_request"`
 687         }
 688         c.Check(json.NewDecoder(s.remoteMockRequests[0].Body).Decode(&cr), check.IsNil)
 689         c.Check(cr.ContainerRequest.RuntimeToken, check.Equals, arvadostest.ActiveTokenV2)
 690 }
 691
 692 func (s *FederationSuite) TestCreateRemoteContainerRequestError(c *check.C) {
 693         defer s.localServiceReturns404(c).Close()
 694         // pass cluster_id via query parameter, this allows arvados-controller
 695         // to avoid parsing the body
 696         req := httptest.NewRequest("POST", "/arvados/v1/container_requests?cluster_id=zz404",
 697                 strings.NewReader(`{
 698   "container_request": {
 699     "name": "hello world",
 700     "state": "Uncommitted",
 701     "output_path": "/",
 702     "container_image": "123",
 703     "command": ["abc"]
 704   }
 705 }
 706 `))
 707         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 708         req.Header.Set("Content-type", "application/json")
 709         resp := s.testRequest(req)
 710         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 711 }
 712
 713 func (s *FederationSuite) TestGetRemoteContainer(c *check.C) {
 714         defer s.localServiceReturns404(c).Close()
 715         req := httptest.NewRequest("GET", "/arvados/v1/containers/"+arvadostest.QueuedContainerUUID, nil)
 716         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 717         resp := s.testRequest(req)
 718         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 719         var cn arvados.Container
 720         c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
 721         c.Check(cn.UUID, check.Equals, arvadostest.QueuedContainerUUID)
 722 }
 723
 724 func (s *FederationSuite) TestListRemoteContainer(c *check.C) {
 725         defer s.localServiceReturns404(c).Close()
 726         req := httptest.NewRequest("GET", "/arvados/v1/containers?count=none&filters="+
 727                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v"]]]`, arvadostest.QueuedContainerUUID)), nil)
 728         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 729         resp := s.testRequest(req)
 730         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 731         var cn arvados.ContainerList
 732         c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
 733         c.Check(cn.Items[0].UUID, check.Equals, arvadostest.QueuedContainerUUID)
 734 }
 735
 736 func (s *FederationSuite) TestListMultiRemoteContainers(c *check.C) {
 737         defer s.localServiceHandler(c, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 738                 bd, _ := ioutil.ReadAll(req.Body)
 739                 c.Check(string(bd), check.Equals, `_method=GET&count=none&filters=%5B%5B%22uuid%22%2C+%22in%22%2C+%5B%22zhome-xvhdp-cr5queuedcontnr%22%5D%5D%5D&select=%5B%22uuid%22%2C+%22command%22%5D`)
 740                 w.WriteHeader(200)
 741                 w.Write([]byte(`{"kind": "arvados#containerList", "items": [{"uuid": "zhome-xvhdp-cr5queuedcontnr", "command": ["abc"]}]}`))
 742         })).Close()
 743         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&select=%s",
 744                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
 745                         arvadostest.QueuedContainerUUID)),
 746                 url.QueryEscape(`["uuid", "command"]`)),
 747                 nil)
 748         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 749         resp := s.testRequest(req)
 750         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 751         var cn arvados.ContainerList
 752         c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
 753         c.Check(cn.Items, check.HasLen, 2)
 754         mp := make(map[string]arvados.Container)
 755         for _, cr := range cn.Items {
 756                 mp[cr.UUID] = cr
 757         }
 758         c.Check(mp[arvadostest.QueuedContainerUUID].Command, check.DeepEquals, []string{"echo", "hello"})
 759         c.Check(mp[arvadostest.QueuedContainerUUID].ContainerImage, check.Equals, "")
 760         c.Check(mp["zhome-xvhdp-cr5queuedcontnr"].Command, check.DeepEquals, []string{"abc"})
 761         c.Check(mp["zhome-xvhdp-cr5queuedcontnr"].ContainerImage, check.Equals, "")
 762 }
 763
 764 func (s *FederationSuite) TestListMultiRemoteContainerError(c *check.C) {
 765         defer s.localServiceReturns404(c).Close()
 766         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&select=%s",
 767                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
 768                         arvadostest.QueuedContainerUUID)),
 769                 url.QueryEscape(`["uuid", "command"]`)),
 770                 nil)
 771         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 772         resp := s.testRequest(req)
 773         c.Check(resp.StatusCode, check.Equals, http.StatusBadGateway)
 774         s.checkJSONErrorMatches(c, resp, `error fetching from zhome \(404 Not Found\): EOF`)
 775 }
 776
 777 func (s *FederationSuite) TestListMultiRemoteContainersPaged(c *check.C) {
 778
 779         callCount := 0
 780         defer s.localServiceHandler(c, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 781                 bd, _ := ioutil.ReadAll(req.Body)
 782                 if callCount == 0 {
 783                         c.Check(string(bd), check.Equals, `_method=GET&count=none&filters=%5B%5B%22uuid%22%2C+%22in%22%2C+%5B%22zhome-xvhdp-cr5queuedcontnr%22%2C%22zhome-xvhdp-cr6queuedcontnr%22%5D%5D%5D`)
 784                         w.WriteHeader(200)
 785                         w.Write([]byte(`{"kind": "arvados#containerList", "items": [{"uuid": "zhome-xvhdp-cr5queuedcontnr", "command": ["abc"]}]}`))
 786                 } else if callCount == 1 {
 787                         c.Check(string(bd), check.Equals, `_method=GET&count=none&filters=%5B%5B%22uuid%22%2C+%22in%22%2C+%5B%22zhome-xvhdp-cr6queuedcontnr%22%5D%5D%5D`)
 788                         w.WriteHeader(200)
 789                         w.Write([]byte(`{"kind": "arvados#containerList", "items": [{"uuid": "zhome-xvhdp-cr6queuedcontnr", "command": ["efg"]}]}`))
 790                 }
 791                 callCount += 1
 792         })).Close()
 793         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s",
 794                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr", "zhome-xvhdp-cr6queuedcontnr"]]]`,
 795                         arvadostest.QueuedContainerUUID))),
 796                 nil)
 797         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 798         resp := s.testRequest(req)
 799         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 800         c.Check(callCount, check.Equals, 2)
 801         var cn arvados.ContainerList
 802         c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
 803         c.Check(cn.Items, check.HasLen, 3)
 804         mp := make(map[string]arvados.Container)
 805         for _, cr := range cn.Items {
 806                 mp[cr.UUID] = cr
 807         }
 808         c.Check(mp[arvadostest.QueuedContainerUUID].Command, check.DeepEquals, []string{"echo", "hello"})
 809         c.Check(mp["zhome-xvhdp-cr5queuedcontnr"].Command, check.DeepEquals, []string{"abc"})
 810         c.Check(mp["zhome-xvhdp-cr6queuedcontnr"].Command, check.DeepEquals, []string{"efg"})
 811 }
 812
 813 func (s *FederationSuite) TestListMultiRemoteContainersMissing(c *check.C) {
 814
 815         callCount := 0
 816         defer s.localServiceHandler(c, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 817                 bd, _ := ioutil.ReadAll(req.Body)
 818                 if callCount == 0 {
 819                         c.Check(string(bd), check.Equals, `_method=GET&count=none&filters=%5B%5B%22uuid%22%2C+%22in%22%2C+%5B%22zhome-xvhdp-cr5queuedcontnr%22%2C%22zhome-xvhdp-cr6queuedcontnr%22%5D%5D%5D`)
 820                         w.WriteHeader(200)
 821                         w.Write([]byte(`{"kind": "arvados#containerList", "items": [{"uuid": "zhome-xvhdp-cr6queuedcontnr", "command": ["efg"]}]}`))
 822                 } else if callCount == 1 {
 823                         c.Check(string(bd), check.Equals, `_method=GET&count=none&filters=%5B%5B%22uuid%22%2C+%22in%22%2C+%5B%22zhome-xvhdp-cr5queuedcontnr%22%5D%5D%5D`)
 824                         w.WriteHeader(200)
 825                         w.Write([]byte(`{"kind": "arvados#containerList", "items": []}`))
 826                 }
 827                 callCount += 1
 828         })).Close()
 829         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s",
 830                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr", "zhome-xvhdp-cr6queuedcontnr"]]]`,
 831                         arvadostest.QueuedContainerUUID))),
 832                 nil)
 833         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 834         resp := s.testRequest(req)
 835         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 836         c.Check(callCount, check.Equals, 2)
 837         var cn arvados.ContainerList
 838         c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
 839         c.Check(cn.Items, check.HasLen, 2)
 840         mp := make(map[string]arvados.Container)
 841         for _, cr := range cn.Items {
 842                 mp[cr.UUID] = cr
 843         }
 844         c.Check(mp[arvadostest.QueuedContainerUUID].Command, check.DeepEquals, []string{"echo", "hello"})
 845         c.Check(mp["zhome-xvhdp-cr6queuedcontnr"].Command, check.DeepEquals, []string{"efg"})
 846 }
 847
 848 func (s *FederationSuite) TestListMultiRemoteContainerPageSizeError(c *check.C) {
 849         s.testHandler.Cluster.RequestLimits.MaxItemsPerResponse = 1
 850         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s",
 851                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
 852                         arvadostest.QueuedContainerUUID))),
 853                 nil)
 854         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 855         resp := s.testRequest(req)
 856         c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
 857         s.checkJSONErrorMatches(c, resp, `Federated multi-object request for 2 objects which is more than max page size 1.`)
 858 }
 859
 860 func (s *FederationSuite) TestListMultiRemoteContainerLimitError(c *check.C) {
 861         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&limit=1",
 862                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
 863                         arvadostest.QueuedContainerUUID))),
 864                 nil)
 865         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 866         resp := s.testRequest(req)
 867         c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
 868         s.checkJSONErrorMatches(c, resp, `Federated multi-object may not provide 'limit', 'offset' or 'order'.`)
 869 }
 870
 871 func (s *FederationSuite) TestListMultiRemoteContainerOffsetError(c *check.C) {
 872         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&offset=1",
 873                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
 874                         arvadostest.QueuedContainerUUID))),
 875                 nil)
 876         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 877         resp := s.testRequest(req)
 878         c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
 879         s.checkJSONErrorMatches(c, resp, `Federated multi-object may not provide 'limit', 'offset' or 'order'.`)
 880 }
 881
 882 func (s *FederationSuite) TestListMultiRemoteContainerOrderError(c *check.C) {
 883         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&order=uuid",
 884                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
 885                         arvadostest.QueuedContainerUUID))),
 886                 nil)
 887         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 888         resp := s.testRequest(req)
 889         c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
 890         s.checkJSONErrorMatches(c, resp, `Federated multi-object may not provide 'limit', 'offset' or 'order'.`)
 891 }
 892
 893 func (s *FederationSuite) TestListMultiRemoteContainerSelectError(c *check.C) {
 894         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&select=%s",
 895                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
 896                         arvadostest.QueuedContainerUUID)),
 897                 url.QueryEscape(`["command"]`)),
 898                 nil)
 899         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 900         resp := s.testRequest(req)
 901         c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
 902         s.checkJSONErrorMatches(c, resp, `Federated multi-object request must include 'uuid' in 'select'`)
 903 }