lib/controller/federation_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package controller
   6
   7 import (
   8         "bytes"
   9         "encoding/json"
  10         "fmt"
  11         "io"
  12         "io/ioutil"
  13         "net/http"
  14         "net/http/httptest"
  15         "net/url"
  16         "os"
  17         "strings"
  18         "time"
  19
  20         "git.curoverse.com/arvados.git/sdk/go/arvados"
  21         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  22         "git.curoverse.com/arvados.git/sdk/go/ctxlog"
  23         "git.curoverse.com/arvados.git/sdk/go/httpserver"
  24         "git.curoverse.com/arvados.git/sdk/go/keepclient"
  25         "github.com/sirupsen/logrus"
  26         check "gopkg.in/check.v1"
  27 )
  28
  29 // Gocheck boilerplate
  30 var _ = check.Suite(&FederationSuite{})
  31
  32 type FederationSuite struct {
  33         log logrus.FieldLogger
  34         // testServer and testHandler are the controller being tested,
  35         // "zhome".
  36         testServer  *httpserver.Server
  37         testHandler *Handler
  38         // remoteServer ("zzzzz") forwards requests to the Rails API
  39         // provided by the integration test environment.
  40         remoteServer *httpserver.Server
  41         // remoteMock ("zmock") appends each incoming request to
  42         // remoteMockRequests, and returns an empty 200 response.
  43         remoteMock         *httpserver.Server
  44         remoteMockRequests []http.Request
  45 }
  46
  47 func (s *FederationSuite) SetUpTest(c *check.C) {
  48         s.log = ctxlog.TestLogger(c)
  49
  50         s.remoteServer = newServerFromIntegrationTestEnv(c)
  51         c.Assert(s.remoteServer.Start(), check.IsNil)
  52
  53         s.remoteMock = newServerFromIntegrationTestEnv(c)
  54         s.remoteMock.Server.Handler = http.HandlerFunc(s.remoteMockHandler)
  55         c.Assert(s.remoteMock.Start(), check.IsNil)
  56
  57         cluster := &arvados.Cluster{
  58                 ClusterID:  "zhome",
  59                 PostgreSQL: integrationTestCluster().PostgreSQL,
  60         }
  61         cluster.TLS.Insecure = true
  62         cluster.API.MaxItemsPerResponse = 1000
  63         cluster.API.MaxRequestAmplification = 4
  64         arvadostest.SetServiceURL(&cluster.Services.RailsAPI, "http://localhost:1/")
  65         arvadostest.SetServiceURL(&cluster.Services.Controller, "http://localhost:/")
  66         s.testHandler = &Handler{Cluster: cluster}
  67         s.testServer = newServerFromIntegrationTestEnv(c)
  68         s.testServer.Server.Handler = httpserver.AddRequestIDs(httpserver.LogRequests(s.log, s.testHandler))
  69
  70         cluster.RemoteClusters = map[string]arvados.RemoteCluster{
  71                 "zzzzz": {
  72                         Host:   s.remoteServer.Addr,
  73                         Proxy:  true,
  74                         Scheme: "http",
  75                 },
  76                 "zmock": {
  77                         Host:   s.remoteMock.Addr,
  78                         Proxy:  true,
  79                         Scheme: "http",
  80                 },
  81         }
  82
  83         c.Assert(s.testServer.Start(), check.IsNil)
  84
  85         s.remoteMockRequests = nil
  86 }
  87
  88 func (s *FederationSuite) remoteMockHandler(w http.ResponseWriter, req *http.Request) {
  89         b := &bytes.Buffer{}
  90         io.Copy(b, req.Body)
  91         req.Body.Close()
  92         req.Body = ioutil.NopCloser(b)
  93         s.remoteMockRequests = append(s.remoteMockRequests, *req)
  94 }
  95
  96 func (s *FederationSuite) TearDownTest(c *check.C) {
  97         if s.remoteServer != nil {
  98                 s.remoteServer.Close()
  99         }
 100         if s.testServer != nil {
 101                 s.testServer.Close()
 102         }
 103 }
 104
 105 func (s *FederationSuite) testRequest(req *http.Request) *http.Response {
 106         resp := httptest.NewRecorder()
 107         s.testServer.Server.Handler.ServeHTTP(resp, req)
 108         return resp.Result()
 109 }
 110
 111 func (s *FederationSuite) TestLocalRequest(c *check.C) {
 112         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zhome-", 1), nil)
 113         resp := s.testRequest(req)
 114         s.checkHandledLocally(c, resp)
 115 }
 116
 117 func (s *FederationSuite) checkHandledLocally(c *check.C, resp *http.Response) {
 118         // Our "home" controller can't handle local requests because
 119         // it doesn't have its own stub/test Rails API, so we rely on
 120         // "connection refused" to indicate the controller tried to
 121         // proxy the request to its local Rails API.
 122         c.Check(resp.StatusCode, check.Equals, http.StatusBadGateway)
 123         s.checkJSONErrorMatches(c, resp, `.*connection refused`)
 124 }
 125
 126 func (s *FederationSuite) TestNoAuth(c *check.C) {
 127         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
 128         resp := s.testRequest(req)
 129         c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
 130         s.checkJSONErrorMatches(c, resp, `Not logged in`)
 131 }
 132
 133 func (s *FederationSuite) TestBadAuth(c *check.C) {
 134         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
 135         req.Header.Set("Authorization", "Bearer aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
 136         resp := s.testRequest(req)
 137         c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
 138         s.checkJSONErrorMatches(c, resp, `Not logged in`)
 139 }
 140
 141 func (s *FederationSuite) TestNoAccess(c *check.C) {
 142         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
 143         req.Header.Set("Authorization", "Bearer "+arvadostest.SpectatorToken)
 144         resp := s.testRequest(req)
 145         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 146         s.checkJSONErrorMatches(c, resp, `.*not found`)
 147 }
 148
 149 func (s *FederationSuite) TestGetUnknownRemote(c *check.C) {
 150         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zz404-", 1), nil)
 151         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 152         resp := s.testRequest(req)
 153         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 154         s.checkJSONErrorMatches(c, resp, `.*no proxy available for cluster zz404`)
 155 }
 156
 157 func (s *FederationSuite) TestRemoteError(c *check.C) {
 158         rc := s.testHandler.Cluster.RemoteClusters["zzzzz"]
 159         rc.Scheme = "https"
 160         s.testHandler.Cluster.RemoteClusters["zzzzz"] = rc
 161
 162         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
 163         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 164         resp := s.testRequest(req)
 165         c.Check(resp.StatusCode, check.Equals, http.StatusBadGateway)
 166         s.checkJSONErrorMatches(c, resp, `.*HTTP response to HTTPS client`)
 167 }
 168
 169 func (s *FederationSuite) TestGetRemoteWorkflow(c *check.C) {
 170         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
 171         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 172         resp := s.testRequest(req)
 173         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 174         var wf arvados.Workflow
 175         c.Check(json.NewDecoder(resp.Body).Decode(&wf), check.IsNil)
 176         c.Check(wf.UUID, check.Equals, arvadostest.WorkflowWithDefinitionYAMLUUID)
 177         c.Check(wf.OwnerUUID, check.Equals, arvadostest.ActiveUserUUID)
 178 }
 179
 180 func (s *FederationSuite) TestOptionsMethod(c *check.C) {
 181         req := httptest.NewRequest("OPTIONS", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
 182         req.Header.Set("Origin", "https://example.com")
 183         resp := s.testRequest(req)
 184         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 185         body, err := ioutil.ReadAll(resp.Body)
 186         c.Check(err, check.IsNil)
 187         c.Check(string(body), check.Equals, "")
 188         c.Check(resp.Header.Get("Access-Control-Allow-Origin"), check.Equals, "*")
 189         for _, hdr := range []string{"Authorization", "Content-Type"} {
 190                 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), check.Matches, ".*"+hdr+".*")
 191         }
 192         for _, method := range []string{"GET", "HEAD", "PUT", "POST", "DELETE"} {
 193                 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), check.Matches, ".*"+method+".*")
 194         }
 195 }
 196
 197 func (s *FederationSuite) TestRemoteWithTokenInQuery(c *check.C) {
 198         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1)+"?api_token="+arvadostest.ActiveToken, nil)
 199         s.testRequest(req)
 200         c.Assert(s.remoteMockRequests, check.HasLen, 1)
 201         pr := s.remoteMockRequests[0]
 202         // Token is salted and moved from query to Authorization header.
 203         c.Check(pr.URL.String(), check.Not(check.Matches), `.*api_token=.*`)
 204         c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/7fd31b61f39c0e82a4155592163218272cedacdc")
 205 }
 206
 207 func (s *FederationSuite) TestLocalTokenSalted(c *check.C) {
 208         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1), nil)
 209         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 210         s.testRequest(req)
 211         c.Assert(s.remoteMockRequests, check.HasLen, 1)
 212         pr := s.remoteMockRequests[0]
 213         // The salted token here has a "zzzzz-" UUID instead of a
 214         // "ztest-" UUID because ztest's local database has the
 215         // "zzzzz-" test fixtures. The "secret" part is HMAC(sha1,
 216         // arvadostest.ActiveToken, "zmock") = "7fd3...".
 217         c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/7fd31b61f39c0e82a4155592163218272cedacdc")
 218 }
 219
 220 func (s *FederationSuite) TestRemoteTokenNotSalted(c *check.C) {
 221         // remoteToken can be any v1 token that doesn't appear in
 222         // ztest's local db.
 223         remoteToken := "abcdef00000000000000000000000000000000000000000000"
 224         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1), nil)
 225         req.Header.Set("Authorization", "Bearer "+remoteToken)
 226         s.testRequest(req)
 227         c.Assert(s.remoteMockRequests, check.HasLen, 1)
 228         pr := s.remoteMockRequests[0]
 229         c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer "+remoteToken)
 230 }
 231
 232 func (s *FederationSuite) TestWorkflowCRUD(c *check.C) {
 233         wf := arvados.Workflow{
 234                 Description: "TestCRUD",
 235         }
 236         {
 237                 body := &strings.Builder{}
 238                 json.NewEncoder(body).Encode(&wf)
 239                 req := httptest.NewRequest("POST", "/arvados/v1/workflows", strings.NewReader(url.Values{
 240                         "workflow": {body.String()},
 241                 }.Encode()))
 242                 req.Header.Set("Content-type", "application/x-www-form-urlencoded")
 243                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 244                 rec := httptest.NewRecorder()
 245                 s.remoteServer.Server.Handler.ServeHTTP(rec, req) // direct to remote -- can't proxy a create req because no uuid
 246                 resp := rec.Result()
 247                 s.checkResponseOK(c, resp)
 248                 json.NewDecoder(resp.Body).Decode(&wf)
 249
 250                 defer func() {
 251                         req := httptest.NewRequest("DELETE", "/arvados/v1/workflows/"+wf.UUID, nil)
 252                         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 253                         s.remoteServer.Server.Handler.ServeHTTP(httptest.NewRecorder(), req)
 254                 }()
 255                 c.Check(wf.UUID, check.Not(check.Equals), "")
 256
 257                 c.Assert(wf.ModifiedAt, check.NotNil)
 258                 c.Logf("wf.ModifiedAt: %v", wf.ModifiedAt)
 259                 c.Check(time.Since(*wf.ModifiedAt) < time.Minute, check.Equals, true)
 260         }
 261         for _, method := range []string{"PATCH", "PUT", "POST"} {
 262                 form := url.Values{
 263                         "workflow": {`{"description": "Updated with ` + method + `"}`},
 264                 }
 265                 if method == "POST" {
 266                         form["_method"] = []string{"PATCH"}
 267                 }
 268                 req := httptest.NewRequest(method, "/arvados/v1/workflows/"+wf.UUID, strings.NewReader(form.Encode()))
 269                 req.Header.Set("Content-type", "application/x-www-form-urlencoded")
 270                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 271                 resp := s.testRequest(req)
 272                 s.checkResponseOK(c, resp)
 273                 err := json.NewDecoder(resp.Body).Decode(&wf)
 274                 c.Check(err, check.IsNil)
 275
 276                 c.Check(wf.Description, check.Equals, "Updated with "+method)
 277         }
 278         {
 279                 req := httptest.NewRequest("DELETE", "/arvados/v1/workflows/"+wf.UUID, nil)
 280                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 281                 resp := s.testRequest(req)
 282                 s.checkResponseOK(c, resp)
 283                 err := json.NewDecoder(resp.Body).Decode(&wf)
 284                 c.Check(err, check.IsNil)
 285         }
 286         {
 287                 req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+wf.UUID, nil)
 288                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 289                 resp := s.testRequest(req)
 290                 c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 291         }
 292 }
 293
 294 func (s *FederationSuite) checkResponseOK(c *check.C, resp *http.Response) {
 295         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 296         if resp.StatusCode != http.StatusOK {
 297                 body, err := ioutil.ReadAll(resp.Body)
 298                 c.Logf("... response body = %q, %v\n", body, err)
 299         }
 300 }
 301
 302 func (s *FederationSuite) checkJSONErrorMatches(c *check.C, resp *http.Response, re string) {
 303         var jresp httpserver.ErrorResponse
 304         err := json.NewDecoder(resp.Body).Decode(&jresp)
 305         c.Check(err, check.IsNil)
 306         c.Assert(jresp.Errors, check.HasLen, 1)
 307         c.Check(jresp.Errors[0], check.Matches, re)
 308 }
 309
 310 func (s *FederationSuite) localServiceHandler(c *check.C, h http.Handler) *httpserver.Server {
 311         srv := &httpserver.Server{
 312                 Server: http.Server{
 313                         Handler: h,
 314                 },
 315         }
 316         c.Assert(srv.Start(), check.IsNil)
 317         arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "http://"+srv.Addr)
 318         return srv
 319 }
 320
 321 func (s *FederationSuite) localServiceReturns404(c *check.C) *httpserver.Server {
 322         return s.localServiceHandler(c, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 323                 w.WriteHeader(404)
 324         }))
 325 }
 326
 327 func (s *FederationSuite) TestGetLocalCollection(c *check.C) {
 328         s.testHandler.Cluster.ClusterID = "zzzzz"
 329         arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
 330
 331         // HTTP GET
 332
 333         req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementCollection, nil)
 334         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 335         resp := s.testRequest(req)
 336
 337         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 338         var col arvados.Collection
 339         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
 340         c.Check(col.UUID, check.Equals, arvadostest.UserAgreementCollection)
 341         c.Check(col.ManifestText, check.Matches,
 342                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+A[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
 343 `)
 344
 345         // HTTP POST with _method=GET as a form parameter
 346
 347         req = httptest.NewRequest("POST", "/arvados/v1/collections/"+arvadostest.UserAgreementCollection, bytes.NewBufferString((url.Values{
 348                 "_method": {"GET"},
 349         }).Encode()))
 350         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 351         req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
 352         resp = s.testRequest(req)
 353
 354         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 355         col = arvados.Collection{}
 356         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
 357         c.Check(col.UUID, check.Equals, arvadostest.UserAgreementCollection)
 358         c.Check(col.ManifestText, check.Matches,
 359                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+A[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
 360 `)
 361 }
 362
 363 func (s *FederationSuite) TestGetRemoteCollection(c *check.C) {
 364         defer s.localServiceReturns404(c).Close()
 365
 366         req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementCollection, nil)
 367         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 368         resp := s.testRequest(req)
 369         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 370         var col arvados.Collection
 371         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
 372         c.Check(col.UUID, check.Equals, arvadostest.UserAgreementCollection)
 373         c.Check(col.ManifestText, check.Matches,
 374                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+Rzzzzz-[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
 375 `)
 376 }
 377
 378 func (s *FederationSuite) TestGetRemoteCollectionError(c *check.C) {
 379         defer s.localServiceReturns404(c).Close()
 380
 381         req := httptest.NewRequest("GET", "/arvados/v1/collections/zzzzz-4zz18-fakefakefakefak", nil)
 382         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 383         resp := s.testRequest(req)
 384         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 385 }
 386
 387 func (s *FederationSuite) TestSignedLocatorPattern(c *check.C) {
 388         // Confirm the regular expression identifies other groups of hints correctly
 389         c.Check(keepclient.SignedLocatorRe.FindStringSubmatch(`6a4ff0499484c6c79c95cd8c566bd25f+249025+B1+C2+A05227438989d04712ea9ca1c91b556cef01d5cc7@5ba5405b+D3+E4`),
 390                 check.DeepEquals,
 391                 []string{"6a4ff0499484c6c79c95cd8c566bd25f+249025+B1+C2+A05227438989d04712ea9ca1c91b556cef01d5cc7@5ba5405b+D3+E4",
 392                         "6a4ff0499484c6c79c95cd8c566bd25f",
 393                         "+249025",
 394                         "+B1+C2", "+C2",
 395                         "+A05227438989d04712ea9ca1c91b556cef01d5cc7@5ba5405b",
 396                         "05227438989d04712ea9ca1c91b556cef01d5cc7", "5ba5405b",
 397                         "+D3+E4", "+E4"})
 398 }
 399
 400 func (s *FederationSuite) TestGetLocalCollectionByPDH(c *check.C) {
 401         arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
 402
 403         req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
 404         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 405         resp := s.testRequest(req)
 406
 407         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 408         var col arvados.Collection
 409         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
 410         c.Check(col.PortableDataHash, check.Equals, arvadostest.UserAgreementPDH)
 411         c.Check(col.ManifestText, check.Matches,
 412                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+A[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
 413 `)
 414 }
 415
 416 func (s *FederationSuite) TestGetRemoteCollectionByPDH(c *check.C) {
 417         defer s.localServiceReturns404(c).Close()
 418
 419         req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
 420         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 421         resp := s.testRequest(req)
 422
 423         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 424
 425         var col arvados.Collection
 426         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
 427         c.Check(col.PortableDataHash, check.Equals, arvadostest.UserAgreementPDH)
 428         c.Check(col.ManifestText, check.Matches,
 429                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+Rzzzzz-[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
 430 `)
 431 }
 432
 433 func (s *FederationSuite) TestGetCollectionByPDHError(c *check.C) {
 434         defer s.localServiceReturns404(c).Close()
 435
 436         req := httptest.NewRequest("GET", "/arvados/v1/collections/99999999999999999999999999999999+99", nil)
 437         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 438
 439         resp := s.testRequest(req)
 440         defer resp.Body.Close()
 441
 442         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 443 }
 444
 445 func (s *FederationSuite) TestGetCollectionByPDHErrorBadHash(c *check.C) {
 446         defer s.localServiceReturns404(c).Close()
 447
 448         srv2 := &httpserver.Server{
 449                 Server: http.Server{
 450                         Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 451                                 w.WriteHeader(200)
 452                                 // Return a collection where the hash
 453                                 // of the manifest text doesn't match
 454                                 // PDH that was requested.
 455                                 var col arvados.Collection
 456                                 col.PortableDataHash = "99999999999999999999999999999999+99"
 457                                 col.ManifestText = `. 6a4ff0499484c6c79c95cd8c566bd25f\+249025 0:249025:GNU_General_Public_License,_version_3.pdf
 458 `
 459                                 enc := json.NewEncoder(w)
 460                                 enc.Encode(col)
 461                         }),
 462                 },
 463         }
 464
 465         c.Assert(srv2.Start(), check.IsNil)
 466         defer srv2.Close()
 467
 468         // Direct zzzzz to service that returns a 200 result with a bogus manifest_text
 469         s.testHandler.Cluster.RemoteClusters["zzzzz"] = arvados.RemoteCluster{
 470                 Host:   srv2.Addr,
 471                 Proxy:  true,
 472                 Scheme: "http",
 473         }
 474
 475         req := httptest.NewRequest("GET", "/arvados/v1/collections/99999999999999999999999999999999+99", nil)
 476         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 477
 478         resp := s.testRequest(req)
 479         defer resp.Body.Close()
 480
 481         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 482 }
 483
 484 func (s *FederationSuite) TestSaltedTokenGetCollectionByPDH(c *check.C) {
 485         arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
 486
 487         req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
 488         req.Header.Set("Authorization", "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/282d7d172b6cfdce364c5ed12ddf7417b2d00065")
 489         resp := s.testRequest(req)
 490
 491         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 492         var col arvados.Collection
 493         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
 494         c.Check(col.PortableDataHash, check.Equals, arvadostest.UserAgreementPDH)
 495         c.Check(col.ManifestText, check.Matches,
 496                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+A[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
 497 `)
 498 }
 499
 500 func (s *FederationSuite) TestSaltedTokenGetCollectionByPDHError(c *check.C) {
 501         arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
 502
 503         req := httptest.NewRequest("GET", "/arvados/v1/collections/99999999999999999999999999999999+99", nil)
 504         req.Header.Set("Authorization", "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/282d7d172b6cfdce364c5ed12ddf7417b2d00065")
 505         resp := s.testRequest(req)
 506
 507         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 508 }
 509
 510 func (s *FederationSuite) TestGetRemoteContainerRequest(c *check.C) {
 511         defer s.localServiceReturns404(c).Close()
 512         req := httptest.NewRequest("GET", "/arvados/v1/container_requests/"+arvadostest.QueuedContainerRequestUUID, nil)
 513         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 514         resp := s.testRequest(req)
 515         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 516         var cr arvados.ContainerRequest
 517         c.Check(json.NewDecoder(resp.Body).Decode(&cr), check.IsNil)
 518         c.Check(cr.UUID, check.Equals, arvadostest.QueuedContainerRequestUUID)
 519         c.Check(cr.Priority, check.Equals, 1)
 520 }
 521
 522 func (s *FederationSuite) TestUpdateRemoteContainerRequest(c *check.C) {
 523         defer s.localServiceReturns404(c).Close()
 524         setPri := func(pri int) {
 525                 req := httptest.NewRequest("PATCH", "/arvados/v1/container_requests/"+arvadostest.QueuedContainerRequestUUID,
 526                         strings.NewReader(fmt.Sprintf(`{"container_request": {"priority": %d}}`, pri)))
 527                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 528                 req.Header.Set("Content-type", "application/json")
 529                 resp := s.testRequest(req)
 530                 c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 531                 var cr arvados.ContainerRequest
 532                 c.Check(json.NewDecoder(resp.Body).Decode(&cr), check.IsNil)
 533                 c.Check(cr.UUID, check.Equals, arvadostest.QueuedContainerRequestUUID)
 534                 c.Check(cr.Priority, check.Equals, pri)
 535         }
 536         setPri(696)
 537         setPri(1) // Reset fixture so side effect doesn't break other tests.
 538 }
 539
 540 func (s *FederationSuite) TestCreateRemoteContainerRequest(c *check.C) {
 541         defer s.localServiceReturns404(c).Close()
 542         // pass cluster_id via query parameter, this allows arvados-controller
 543         // to avoid parsing the body
 544         req := httptest.NewRequest("POST", "/arvados/v1/container_requests?cluster_id=zzzzz",
 545                 strings.NewReader(`{
 546   "container_request": {
 547     "name": "hello world",
 548     "state": "Uncommitted",
 549     "output_path": "/",
 550     "container_image": "123",
 551     "command": ["abc"]
 552   }
 553 }
 554 `))
 555         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 556         req.Header.Set("Content-type", "application/json")
 557         resp := s.testRequest(req)
 558         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 559         var cr arvados.ContainerRequest
 560         c.Check(json.NewDecoder(resp.Body).Decode(&cr), check.IsNil)
 561         c.Check(cr.Name, check.Equals, "hello world")
 562         c.Check(strings.HasPrefix(cr.UUID, "zzzzz-"), check.Equals, true)
 563 }
 564
 565 func (s *FederationSuite) TestCreateRemoteContainerRequestCheckRuntimeToken(c *check.C) {
 566         // Send request to zmock and check that outgoing request has
 567         // runtime_token set with a new random v2 token.
 568
 569         defer s.localServiceReturns404(c).Close()
 570         // pass cluster_id via query parameter, this allows arvados-controller
 571         // to avoid parsing the body
 572         req := httptest.NewRequest("POST", "/arvados/v1/container_requests?cluster_id=zmock",
 573                 strings.NewReader(`{
 574   "container_request": {
 575     "name": "hello world",
 576     "state": "Uncommitted",
 577     "output_path": "/",
 578     "container_image": "123",
 579     "command": ["abc"]
 580   }
 581 }
 582 `))
 583         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
 584         req.Header.Set("Content-type", "application/json")
 585
 586         arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
 587         s.testHandler.Cluster.ClusterID = "zzzzz"
 588
 589         resp := s.testRequest(req)
 590         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 591         var cr struct {
 592                 arvados.ContainerRequest `json:"container_request"`
 593         }
 594         c.Check(json.NewDecoder(s.remoteMockRequests[0].Body).Decode(&cr), check.IsNil)
 595         c.Check(strings.HasPrefix(cr.ContainerRequest.RuntimeToken, "v2/zzzzz-gj3su-"), check.Equals, true)
 596         c.Check(cr.ContainerRequest.RuntimeToken, check.Not(check.Equals), arvadostest.ActiveTokenV2)
 597 }
 598
 599 func (s *FederationSuite) TestCreateRemoteContainerRequestCheckSetRuntimeToken(c *check.C) {
 600         // Send request to zmock and check that outgoing request has
 601         // runtime_token set with the explicitly provided token.
 602
 603         defer s.localServiceReturns404(c).Close()
 604         // pass cluster_id via query parameter, this allows arvados-controller
 605         // to avoid parsing the body
 606         req := httptest.NewRequest("POST", "/arvados/v1/container_requests?cluster_id=zmock",
 607                 strings.NewReader(`{
 608   "container_request": {
 609     "name": "hello world",
 610     "state": "Uncommitted",
 611     "output_path": "/",
 612     "container_image": "123",
 613     "command": ["abc"],
 614     "runtime_token": "xyz"
 615   }
 616 }
 617 `))
 618         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 619         req.Header.Set("Content-type", "application/json")
 620         resp := s.testRequest(req)
 621         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 622         var cr struct {
 623                 arvados.ContainerRequest `json:"container_request"`
 624         }
 625         c.Check(json.NewDecoder(s.remoteMockRequests[0].Body).Decode(&cr), check.IsNil)
 626         c.Check(cr.ContainerRequest.RuntimeToken, check.Equals, "xyz")
 627 }
 628
 629 func (s *FederationSuite) TestCreateRemoteContainerRequestRuntimeTokenFromAuth(c *check.C) {
 630         // Send request to zmock and check that outgoing request has
 631         // runtime_token set using the Auth token because the user is remote.
 632
 633         defer s.localServiceReturns404(c).Close()
 634         // pass cluster_id via query parameter, this allows arvados-controller
 635         // to avoid parsing the body
 636         req := httptest.NewRequest("POST", "/arvados/v1/container_requests?cluster_id=zmock",
 637                 strings.NewReader(`{
 638   "container_request": {
 639     "name": "hello world",
 640     "state": "Uncommitted",
 641     "output_path": "/",
 642     "container_image": "123",
 643     "command": ["abc"]
 644   }
 645 }
 646 `))
 647         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2+"/zzzzz-dz642-parentcontainer")
 648         req.Header.Set("Content-type", "application/json")
 649         resp := s.testRequest(req)
 650         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 651         var cr struct {
 652                 arvados.ContainerRequest `json:"container_request"`
 653         }
 654         c.Check(json.NewDecoder(s.remoteMockRequests[0].Body).Decode(&cr), check.IsNil)
 655         c.Check(cr.ContainerRequest.RuntimeToken, check.Equals, arvadostest.ActiveTokenV2)
 656 }
 657
 658 func (s *FederationSuite) TestCreateRemoteContainerRequestError(c *check.C) {
 659         defer s.localServiceReturns404(c).Close()
 660         // pass cluster_id via query parameter, this allows arvados-controller
 661         // to avoid parsing the body
 662         req := httptest.NewRequest("POST", "/arvados/v1/container_requests?cluster_id=zz404",
 663                 strings.NewReader(`{
 664   "container_request": {
 665     "name": "hello world",
 666     "state": "Uncommitted",
 667     "output_path": "/",
 668     "container_image": "123",
 669     "command": ["abc"]
 670   }
 671 }
 672 `))
 673         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 674         req.Header.Set("Content-type", "application/json")
 675         resp := s.testRequest(req)
 676         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 677 }
 678
 679 func (s *FederationSuite) TestGetRemoteContainer(c *check.C) {
 680         defer s.localServiceReturns404(c).Close()
 681         req := httptest.NewRequest("GET", "/arvados/v1/containers/"+arvadostest.QueuedContainerUUID, nil)
 682         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 683         resp := s.testRequest(req)
 684         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 685         var cn arvados.Container
 686         c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
 687         c.Check(cn.UUID, check.Equals, arvadostest.QueuedContainerUUID)
 688 }
 689
 690 func (s *FederationSuite) TestListRemoteContainer(c *check.C) {
 691         defer s.localServiceReturns404(c).Close()
 692         req := httptest.NewRequest("GET", "/arvados/v1/containers?count=none&filters="+
 693                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v"]]]`, arvadostest.QueuedContainerUUID)), nil)
 694         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 695         resp := s.testRequest(req)
 696         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 697         var cn arvados.ContainerList
 698         c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
 699         c.Check(cn.Items[0].UUID, check.Equals, arvadostest.QueuedContainerUUID)
 700 }
 701
 702 func (s *FederationSuite) TestListMultiRemoteContainers(c *check.C) {
 703         defer s.localServiceHandler(c, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 704                 bd, _ := ioutil.ReadAll(req.Body)
 705                 c.Check(string(bd), check.Equals, `_method=GET&count=none&filters=%5B%5B%22uuid%22%2C+%22in%22%2C+%5B%22zhome-xvhdp-cr5queuedcontnr%22%5D%5D%5D&select=%5B%22uuid%22%2C+%22command%22%5D`)
 706                 w.WriteHeader(200)
 707                 w.Write([]byte(`{"kind": "arvados#containerList", "items": [{"uuid": "zhome-xvhdp-cr5queuedcontnr", "command": ["abc"]}]}`))
 708         })).Close()
 709         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&select=%s",
 710                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
 711                         arvadostest.QueuedContainerUUID)),
 712                 url.QueryEscape(`["uuid", "command"]`)),
 713                 nil)
 714         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 715         resp := s.testRequest(req)
 716         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 717         var cn arvados.ContainerList
 718         c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
 719         c.Check(cn.Items, check.HasLen, 2)
 720         mp := make(map[string]arvados.Container)
 721         for _, cr := range cn.Items {
 722                 mp[cr.UUID] = cr
 723         }
 724         c.Check(mp[arvadostest.QueuedContainerUUID].Command, check.DeepEquals, []string{"echo", "hello"})
 725         c.Check(mp[arvadostest.QueuedContainerUUID].ContainerImage, check.Equals, "")
 726         c.Check(mp["zhome-xvhdp-cr5queuedcontnr"].Command, check.DeepEquals, []string{"abc"})
 727         c.Check(mp["zhome-xvhdp-cr5queuedcontnr"].ContainerImage, check.Equals, "")
 728 }
 729
 730 func (s *FederationSuite) TestListMultiRemoteContainerError(c *check.C) {
 731         defer s.localServiceReturns404(c).Close()
 732         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&select=%s",
 733                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
 734                         arvadostest.QueuedContainerUUID)),
 735                 url.QueryEscape(`["uuid", "command"]`)),
 736                 nil)
 737         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 738         resp := s.testRequest(req)
 739         c.Check(resp.StatusCode, check.Equals, http.StatusBadGateway)
 740         s.checkJSONErrorMatches(c, resp, `error fetching from zhome \(404 Not Found\): EOF`)
 741 }
 742
 743 func (s *FederationSuite) TestListMultiRemoteContainersPaged(c *check.C) {
 744
 745         callCount := 0
 746         defer s.localServiceHandler(c, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 747                 bd, _ := ioutil.ReadAll(req.Body)
 748                 if callCount == 0 {
 749                         c.Check(string(bd), check.Equals, `_method=GET&count=none&filters=%5B%5B%22uuid%22%2C+%22in%22%2C+%5B%22zhome-xvhdp-cr5queuedcontnr%22%2C%22zhome-xvhdp-cr6queuedcontnr%22%5D%5D%5D`)
 750                         w.WriteHeader(200)
 751                         w.Write([]byte(`{"kind": "arvados#containerList", "items": [{"uuid": "zhome-xvhdp-cr5queuedcontnr", "command": ["abc"]}]}`))
 752                 } else if callCount == 1 {
 753                         c.Check(string(bd), check.Equals, `_method=GET&count=none&filters=%5B%5B%22uuid%22%2C+%22in%22%2C+%5B%22zhome-xvhdp-cr6queuedcontnr%22%5D%5D%5D`)
 754                         w.WriteHeader(200)
 755                         w.Write([]byte(`{"kind": "arvados#containerList", "items": [{"uuid": "zhome-xvhdp-cr6queuedcontnr", "command": ["efg"]}]}`))
 756                 }
 757                 callCount += 1
 758         })).Close()
 759         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s",
 760                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr", "zhome-xvhdp-cr6queuedcontnr"]]]`,
 761                         arvadostest.QueuedContainerUUID))),
 762                 nil)
 763         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 764         resp := s.testRequest(req)
 765         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 766         c.Check(callCount, check.Equals, 2)
 767         var cn arvados.ContainerList
 768         c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
 769         c.Check(cn.Items, check.HasLen, 3)
 770         mp := make(map[string]arvados.Container)
 771         for _, cr := range cn.Items {
 772                 mp[cr.UUID] = cr
 773         }
 774         c.Check(mp[arvadostest.QueuedContainerUUID].Command, check.DeepEquals, []string{"echo", "hello"})
 775         c.Check(mp["zhome-xvhdp-cr5queuedcontnr"].Command, check.DeepEquals, []string{"abc"})
 776         c.Check(mp["zhome-xvhdp-cr6queuedcontnr"].Command, check.DeepEquals, []string{"efg"})
 777 }
 778
 779 func (s *FederationSuite) TestListMultiRemoteContainersMissing(c *check.C) {
 780
 781         callCount := 0
 782         defer s.localServiceHandler(c, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 783                 bd, _ := ioutil.ReadAll(req.Body)
 784                 if callCount == 0 {
 785                         c.Check(string(bd), check.Equals, `_method=GET&count=none&filters=%5B%5B%22uuid%22%2C+%22in%22%2C+%5B%22zhome-xvhdp-cr5queuedcontnr%22%2C%22zhome-xvhdp-cr6queuedcontnr%22%5D%5D%5D`)
 786                         w.WriteHeader(200)
 787                         w.Write([]byte(`{"kind": "arvados#containerList", "items": [{"uuid": "zhome-xvhdp-cr6queuedcontnr", "command": ["efg"]}]}`))
 788                 } else if callCount == 1 {
 789                         c.Check(string(bd), check.Equals, `_method=GET&count=none&filters=%5B%5B%22uuid%22%2C+%22in%22%2C+%5B%22zhome-xvhdp-cr5queuedcontnr%22%5D%5D%5D`)
 790                         w.WriteHeader(200)
 791                         w.Write([]byte(`{"kind": "arvados#containerList", "items": []}`))
 792                 }
 793                 callCount += 1
 794         })).Close()
 795         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s",
 796                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr", "zhome-xvhdp-cr6queuedcontnr"]]]`,
 797                         arvadostest.QueuedContainerUUID))),
 798                 nil)
 799         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 800         resp := s.testRequest(req)
 801         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 802         c.Check(callCount, check.Equals, 2)
 803         var cn arvados.ContainerList
 804         c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
 805         c.Check(cn.Items, check.HasLen, 2)
 806         mp := make(map[string]arvados.Container)
 807         for _, cr := range cn.Items {
 808                 mp[cr.UUID] = cr
 809         }
 810         c.Check(mp[arvadostest.QueuedContainerUUID].Command, check.DeepEquals, []string{"echo", "hello"})
 811         c.Check(mp["zhome-xvhdp-cr6queuedcontnr"].Command, check.DeepEquals, []string{"efg"})
 812 }
 813
 814 func (s *FederationSuite) TestListMultiRemoteContainerPageSizeError(c *check.C) {
 815         s.testHandler.Cluster.API.MaxItemsPerResponse = 1
 816         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s",
 817                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
 818                         arvadostest.QueuedContainerUUID))),
 819                 nil)
 820         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 821         resp := s.testRequest(req)
 822         c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
 823         s.checkJSONErrorMatches(c, resp, `Federated multi-object request for 2 objects which is more than max page size 1.`)
 824 }
 825
 826 func (s *FederationSuite) TestListMultiRemoteContainerLimitError(c *check.C) {
 827         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&limit=1",
 828                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
 829                         arvadostest.QueuedContainerUUID))),
 830                 nil)
 831         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 832         resp := s.testRequest(req)
 833         c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
 834         s.checkJSONErrorMatches(c, resp, `Federated multi-object may not provide 'limit', 'offset' or 'order'.`)
 835 }
 836
 837 func (s *FederationSuite) TestListMultiRemoteContainerOffsetError(c *check.C) {
 838         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&offset=1",
 839                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
 840                         arvadostest.QueuedContainerUUID))),
 841                 nil)
 842         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 843         resp := s.testRequest(req)
 844         c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
 845         s.checkJSONErrorMatches(c, resp, `Federated multi-object may not provide 'limit', 'offset' or 'order'.`)
 846 }
 847
 848 func (s *FederationSuite) TestListMultiRemoteContainerOrderError(c *check.C) {
 849         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&order=uuid",
 850                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
 851                         arvadostest.QueuedContainerUUID))),
 852                 nil)
 853         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 854         resp := s.testRequest(req)
 855         c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
 856         s.checkJSONErrorMatches(c, resp, `Federated multi-object may not provide 'limit', 'offset' or 'order'.`)
 857 }
 858
 859 func (s *FederationSuite) TestListMultiRemoteContainerSelectError(c *check.C) {
 860         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&select=%s",
 861                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
 862                         arvadostest.QueuedContainerUUID)),
 863                 url.QueryEscape(`["command"]`)),
 864                 nil)
 865         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 866         resp := s.testRequest(req)
 867         c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
 868         s.checkJSONErrorMatches(c, resp, `Federated multi-object request must include 'uuid' in 'select'`)
 869 }