lib/controller/federation_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package controller
   6
   7 import (
   8         "bytes"
   9         "encoding/json"
  10         "fmt"
  11         "io"
  12         "io/ioutil"
  13         "net/http"
  14         "net/http/httptest"
  15         "net/url"
  16         "os"
  17         "strings"
  18         "time"
  19
  20         "git.curoverse.com/arvados.git/sdk/go/arvados"
  21         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  22         "git.curoverse.com/arvados.git/sdk/go/ctxlog"
  23         "git.curoverse.com/arvados.git/sdk/go/httpserver"
  24         "git.curoverse.com/arvados.git/sdk/go/keepclient"
  25         "github.com/sirupsen/logrus"
  26         check "gopkg.in/check.v1"
  27 )
  28
  29 // Gocheck boilerplate
  30 var _ = check.Suite(&FederationSuite{})
  31
  32 type FederationSuite struct {
  33         log logrus.FieldLogger
  34         // testServer and testHandler are the controller being tested,
  35         // "zhome".
  36         testServer  *httpserver.Server
  37         testHandler *Handler
  38         // remoteServer ("zzzzz") forwards requests to the Rails API
  39         // provided by the integration test environment.
  40         remoteServer *httpserver.Server
  41         // remoteMock ("zmock") appends each incoming request to
  42         // remoteMockRequests, and returns an empty 200 response.
  43         remoteMock         *httpserver.Server
  44         remoteMockRequests []http.Request
  45 }
  46
  47 func (s *FederationSuite) SetUpTest(c *check.C) {
  48         s.log = ctxlog.TestLogger(c)
  49
  50         s.remoteServer = newServerFromIntegrationTestEnv(c)
  51         c.Assert(s.remoteServer.Start(), check.IsNil)
  52
  53         s.remoteMock = newServerFromIntegrationTestEnv(c)
  54         s.remoteMock.Server.Handler = http.HandlerFunc(s.remoteMockHandler)
  55         c.Assert(s.remoteMock.Start(), check.IsNil)
  56
  57         cluster := &arvados.Cluster{
  58                 ClusterID:  "zhome",
  59                 PostgreSQL: integrationTestCluster().PostgreSQL,
  60                 TLS:        arvados.TLS{Insecure: true},
  61                 API: arvados.API{
  62                         MaxItemsPerResponse:     1000,
  63                         MaxRequestAmplification: 4,
  64                 },
  65         }
  66         arvadostest.SetServiceURL(&cluster.Services.RailsAPI, "http://localhost:1/")
  67         arvadostest.SetServiceURL(&cluster.Services.Controller, "http://localhost:/")
  68         s.testHandler = &Handler{Cluster: cluster}
  69         s.testServer = newServerFromIntegrationTestEnv(c)
  70         s.testServer.Server.Handler = httpserver.AddRequestIDs(httpserver.LogRequests(s.log, s.testHandler))
  71
  72         cluster.RemoteClusters = map[string]arvados.RemoteCluster{
  73                 "zzzzz": {
  74                         Host:   s.remoteServer.Addr,
  75                         Proxy:  true,
  76                         Scheme: "http",
  77                 },
  78                 "zmock": {
  79                         Host:   s.remoteMock.Addr,
  80                         Proxy:  true,
  81                         Scheme: "http",
  82                 },
  83         }
  84
  85         c.Assert(s.testServer.Start(), check.IsNil)
  86
  87         s.remoteMockRequests = nil
  88 }
  89
  90 func (s *FederationSuite) remoteMockHandler(w http.ResponseWriter, req *http.Request) {
  91         b := &bytes.Buffer{}
  92         io.Copy(b, req.Body)
  93         req.Body.Close()
  94         req.Body = ioutil.NopCloser(b)
  95         s.remoteMockRequests = append(s.remoteMockRequests, *req)
  96 }
  97
  98 func (s *FederationSuite) TearDownTest(c *check.C) {
  99         if s.remoteServer != nil {
 100                 s.remoteServer.Close()
 101         }
 102         if s.testServer != nil {
 103                 s.testServer.Close()
 104         }
 105 }
 106
 107 func (s *FederationSuite) testRequest(req *http.Request) *http.Response {
 108         resp := httptest.NewRecorder()
 109         s.testServer.Server.Handler.ServeHTTP(resp, req)
 110         return resp.Result()
 111 }
 112
 113 func (s *FederationSuite) TestLocalRequest(c *check.C) {
 114         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zhome-", 1), nil)
 115         resp := s.testRequest(req)
 116         s.checkHandledLocally(c, resp)
 117 }
 118
 119 func (s *FederationSuite) checkHandledLocally(c *check.C, resp *http.Response) {
 120         // Our "home" controller can't handle local requests because
 121         // it doesn't have its own stub/test Rails API, so we rely on
 122         // "connection refused" to indicate the controller tried to
 123         // proxy the request to its local Rails API.
 124         c.Check(resp.StatusCode, check.Equals, http.StatusBadGateway)
 125         s.checkJSONErrorMatches(c, resp, `.*connection refused`)
 126 }
 127
 128 func (s *FederationSuite) TestNoAuth(c *check.C) {
 129         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
 130         resp := s.testRequest(req)
 131         c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
 132         s.checkJSONErrorMatches(c, resp, `Not logged in`)
 133 }
 134
 135 func (s *FederationSuite) TestBadAuth(c *check.C) {
 136         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
 137         req.Header.Set("Authorization", "Bearer aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
 138         resp := s.testRequest(req)
 139         c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
 140         s.checkJSONErrorMatches(c, resp, `Not logged in`)
 141 }
 142
 143 func (s *FederationSuite) TestNoAccess(c *check.C) {
 144         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
 145         req.Header.Set("Authorization", "Bearer "+arvadostest.SpectatorToken)
 146         resp := s.testRequest(req)
 147         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 148         s.checkJSONErrorMatches(c, resp, `.*not found`)
 149 }
 150
 151 func (s *FederationSuite) TestGetUnknownRemote(c *check.C) {
 152         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zz404-", 1), nil)
 153         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 154         resp := s.testRequest(req)
 155         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 156         s.checkJSONErrorMatches(c, resp, `.*no proxy available for cluster zz404`)
 157 }
 158
 159 func (s *FederationSuite) TestRemoteError(c *check.C) {
 160         rc := s.testHandler.Cluster.RemoteClusters["zzzzz"]
 161         rc.Scheme = "https"
 162         s.testHandler.Cluster.RemoteClusters["zzzzz"] = rc
 163
 164         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
 165         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 166         resp := s.testRequest(req)
 167         c.Check(resp.StatusCode, check.Equals, http.StatusBadGateway)
 168         s.checkJSONErrorMatches(c, resp, `.*HTTP response to HTTPS client`)
 169 }
 170
 171 func (s *FederationSuite) TestGetRemoteWorkflow(c *check.C) {
 172         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
 173         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 174         resp := s.testRequest(req)
 175         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 176         var wf arvados.Workflow
 177         c.Check(json.NewDecoder(resp.Body).Decode(&wf), check.IsNil)
 178         c.Check(wf.UUID, check.Equals, arvadostest.WorkflowWithDefinitionYAMLUUID)
 179         c.Check(wf.OwnerUUID, check.Equals, arvadostest.ActiveUserUUID)
 180 }
 181
 182 func (s *FederationSuite) TestOptionsMethod(c *check.C) {
 183         req := httptest.NewRequest("OPTIONS", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
 184         req.Header.Set("Origin", "https://example.com")
 185         resp := s.testRequest(req)
 186         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 187         body, err := ioutil.ReadAll(resp.Body)
 188         c.Check(err, check.IsNil)
 189         c.Check(string(body), check.Equals, "")
 190         c.Check(resp.Header.Get("Access-Control-Allow-Origin"), check.Equals, "*")
 191         for _, hdr := range []string{"Authorization", "Content-Type"} {
 192                 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), check.Matches, ".*"+hdr+".*")
 193         }
 194         for _, method := range []string{"GET", "HEAD", "PUT", "POST", "DELETE"} {
 195                 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), check.Matches, ".*"+method+".*")
 196         }
 197 }
 198
 199 func (s *FederationSuite) TestRemoteWithTokenInQuery(c *check.C) {
 200         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1)+"?api_token="+arvadostest.ActiveToken, nil)
 201         s.testRequest(req)
 202         c.Assert(s.remoteMockRequests, check.HasLen, 1)
 203         pr := s.remoteMockRequests[0]
 204         // Token is salted and moved from query to Authorization header.
 205         c.Check(pr.URL.String(), check.Not(check.Matches), `.*api_token=.*`)
 206         c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/7fd31b61f39c0e82a4155592163218272cedacdc")
 207 }
 208
 209 func (s *FederationSuite) TestLocalTokenSalted(c *check.C) {
 210         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1), nil)
 211         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 212         s.testRequest(req)
 213         c.Assert(s.remoteMockRequests, check.HasLen, 1)
 214         pr := s.remoteMockRequests[0]
 215         // The salted token here has a "zzzzz-" UUID instead of a
 216         // "ztest-" UUID because ztest's local database has the
 217         // "zzzzz-" test fixtures. The "secret" part is HMAC(sha1,
 218         // arvadostest.ActiveToken, "zmock") = "7fd3...".
 219         c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/7fd31b61f39c0e82a4155592163218272cedacdc")
 220 }
 221
 222 func (s *FederationSuite) TestRemoteTokenNotSalted(c *check.C) {
 223         // remoteToken can be any v1 token that doesn't appear in
 224         // ztest's local db.
 225         remoteToken := "abcdef00000000000000000000000000000000000000000000"
 226         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1), nil)
 227         req.Header.Set("Authorization", "Bearer "+remoteToken)
 228         s.testRequest(req)
 229         c.Assert(s.remoteMockRequests, check.HasLen, 1)
 230         pr := s.remoteMockRequests[0]
 231         c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer "+remoteToken)
 232 }
 233
 234 func (s *FederationSuite) TestWorkflowCRUD(c *check.C) {
 235         wf := arvados.Workflow{
 236                 Description: "TestCRUD",
 237         }
 238         {
 239                 body := &strings.Builder{}
 240                 json.NewEncoder(body).Encode(&wf)
 241                 req := httptest.NewRequest("POST", "/arvados/v1/workflows", strings.NewReader(url.Values{
 242                         "workflow": {body.String()},
 243                 }.Encode()))
 244                 req.Header.Set("Content-type", "application/x-www-form-urlencoded")
 245                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 246                 rec := httptest.NewRecorder()
 247                 s.remoteServer.Server.Handler.ServeHTTP(rec, req) // direct to remote -- can't proxy a create req because no uuid
 248                 resp := rec.Result()
 249                 s.checkResponseOK(c, resp)
 250                 json.NewDecoder(resp.Body).Decode(&wf)
 251
 252                 defer func() {
 253                         req := httptest.NewRequest("DELETE", "/arvados/v1/workflows/"+wf.UUID, nil)
 254                         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 255                         s.remoteServer.Server.Handler.ServeHTTP(httptest.NewRecorder(), req)
 256                 }()
 257                 c.Check(wf.UUID, check.Not(check.Equals), "")
 258
 259                 c.Assert(wf.ModifiedAt, check.NotNil)
 260                 c.Logf("wf.ModifiedAt: %v", wf.ModifiedAt)
 261                 c.Check(time.Since(*wf.ModifiedAt) < time.Minute, check.Equals, true)
 262         }
 263         for _, method := range []string{"PATCH", "PUT", "POST"} {
 264                 form := url.Values{
 265                         "workflow": {`{"description": "Updated with ` + method + `"}`},
 266                 }
 267                 if method == "POST" {
 268                         form["_method"] = []string{"PATCH"}
 269                 }
 270                 req := httptest.NewRequest(method, "/arvados/v1/workflows/"+wf.UUID, strings.NewReader(form.Encode()))
 271                 req.Header.Set("Content-type", "application/x-www-form-urlencoded")
 272                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 273                 resp := s.testRequest(req)
 274                 s.checkResponseOK(c, resp)
 275                 err := json.NewDecoder(resp.Body).Decode(&wf)
 276                 c.Check(err, check.IsNil)
 277
 278                 c.Check(wf.Description, check.Equals, "Updated with "+method)
 279         }
 280         {
 281                 req := httptest.NewRequest("DELETE", "/arvados/v1/workflows/"+wf.UUID, nil)
 282                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 283                 resp := s.testRequest(req)
 284                 s.checkResponseOK(c, resp)
 285                 err := json.NewDecoder(resp.Body).Decode(&wf)
 286                 c.Check(err, check.IsNil)
 287         }
 288         {
 289                 req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+wf.UUID, nil)
 290                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 291                 resp := s.testRequest(req)
 292                 c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 293         }
 294 }
 295
 296 func (s *FederationSuite) checkResponseOK(c *check.C, resp *http.Response) {
 297         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 298         if resp.StatusCode != http.StatusOK {
 299                 body, err := ioutil.ReadAll(resp.Body)
 300                 c.Logf("... response body = %q, %v\n", body, err)
 301         }
 302 }
 303
 304 func (s *FederationSuite) checkJSONErrorMatches(c *check.C, resp *http.Response, re string) {
 305         var jresp httpserver.ErrorResponse
 306         err := json.NewDecoder(resp.Body).Decode(&jresp)
 307         c.Check(err, check.IsNil)
 308         c.Assert(jresp.Errors, check.HasLen, 1)
 309         c.Check(jresp.Errors[0], check.Matches, re)
 310 }
 311
 312 func (s *FederationSuite) localServiceHandler(c *check.C, h http.Handler) *httpserver.Server {
 313         srv := &httpserver.Server{
 314                 Server: http.Server{
 315                         Handler: h,
 316                 },
 317         }
 318         c.Assert(srv.Start(), check.IsNil)
 319         arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "http://"+srv.Addr)
 320         return srv
 321 }
 322
 323 func (s *FederationSuite) localServiceReturns404(c *check.C) *httpserver.Server {
 324         return s.localServiceHandler(c, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 325                 w.WriteHeader(404)
 326         }))
 327 }
 328
 329 func (s *FederationSuite) TestGetLocalCollection(c *check.C) {
 330         s.testHandler.Cluster.ClusterID = "zzzzz"
 331         arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
 332
 333         // HTTP GET
 334
 335         req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementCollection, nil)
 336         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 337         resp := s.testRequest(req)
 338
 339         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 340         var col arvados.Collection
 341         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
 342         c.Check(col.UUID, check.Equals, arvadostest.UserAgreementCollection)
 343         c.Check(col.ManifestText, check.Matches,
 344                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+A[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
 345 `)
 346
 347         // HTTP POST with _method=GET as a form parameter
 348
 349         req = httptest.NewRequest("POST", "/arvados/v1/collections/"+arvadostest.UserAgreementCollection, bytes.NewBufferString((url.Values{
 350                 "_method": {"GET"},
 351         }).Encode()))
 352         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 353         req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
 354         resp = s.testRequest(req)
 355
 356         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 357         col = arvados.Collection{}
 358         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
 359         c.Check(col.UUID, check.Equals, arvadostest.UserAgreementCollection)
 360         c.Check(col.ManifestText, check.Matches,
 361                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+A[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
 362 `)
 363 }
 364
 365 func (s *FederationSuite) TestGetRemoteCollection(c *check.C) {
 366         defer s.localServiceReturns404(c).Close()
 367
 368         req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementCollection, nil)
 369         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 370         resp := s.testRequest(req)
 371         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 372         var col arvados.Collection
 373         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
 374         c.Check(col.UUID, check.Equals, arvadostest.UserAgreementCollection)
 375         c.Check(col.ManifestText, check.Matches,
 376                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+Rzzzzz-[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
 377 `)
 378 }
 379
 380 func (s *FederationSuite) TestGetRemoteCollectionError(c *check.C) {
 381         defer s.localServiceReturns404(c).Close()
 382
 383         req := httptest.NewRequest("GET", "/arvados/v1/collections/zzzzz-4zz18-fakefakefakefak", nil)
 384         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 385         resp := s.testRequest(req)
 386         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 387 }
 388
 389 func (s *FederationSuite) TestSignedLocatorPattern(c *check.C) {
 390         // Confirm the regular expression identifies other groups of hints correctly
 391         c.Check(keepclient.SignedLocatorRe.FindStringSubmatch(`6a4ff0499484c6c79c95cd8c566bd25f+249025+B1+C2+A05227438989d04712ea9ca1c91b556cef01d5cc7@5ba5405b+D3+E4`),
 392                 check.DeepEquals,
 393                 []string{"6a4ff0499484c6c79c95cd8c566bd25f+249025+B1+C2+A05227438989d04712ea9ca1c91b556cef01d5cc7@5ba5405b+D3+E4",
 394                         "6a4ff0499484c6c79c95cd8c566bd25f",
 395                         "+249025",
 396                         "+B1+C2", "+C2",
 397                         "+A05227438989d04712ea9ca1c91b556cef01d5cc7@5ba5405b",
 398                         "05227438989d04712ea9ca1c91b556cef01d5cc7", "5ba5405b",
 399                         "+D3+E4", "+E4"})
 400 }
 401
 402 func (s *FederationSuite) TestGetLocalCollectionByPDH(c *check.C) {
 403         arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
 404
 405         req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
 406         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 407         resp := s.testRequest(req)
 408
 409         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 410         var col arvados.Collection
 411         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
 412         c.Check(col.PortableDataHash, check.Equals, arvadostest.UserAgreementPDH)
 413         c.Check(col.ManifestText, check.Matches,
 414                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+A[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
 415 `)
 416 }
 417
 418 func (s *FederationSuite) TestGetRemoteCollectionByPDH(c *check.C) {
 419         defer s.localServiceReturns404(c).Close()
 420
 421         req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
 422         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 423         resp := s.testRequest(req)
 424
 425         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 426
 427         var col arvados.Collection
 428         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
 429         c.Check(col.PortableDataHash, check.Equals, arvadostest.UserAgreementPDH)
 430         c.Check(col.ManifestText, check.Matches,
 431                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+Rzzzzz-[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
 432 `)
 433 }
 434
 435 func (s *FederationSuite) TestGetCollectionByPDHError(c *check.C) {
 436         defer s.localServiceReturns404(c).Close()
 437
 438         req := httptest.NewRequest("GET", "/arvados/v1/collections/99999999999999999999999999999999+99", nil)
 439         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 440
 441         resp := s.testRequest(req)
 442         defer resp.Body.Close()
 443
 444         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 445 }
 446
 447 func (s *FederationSuite) TestGetCollectionByPDHErrorBadHash(c *check.C) {
 448         defer s.localServiceReturns404(c).Close()
 449
 450         srv2 := &httpserver.Server{
 451                 Server: http.Server{
 452                         Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 453                                 w.WriteHeader(200)
 454                                 // Return a collection where the hash
 455                                 // of the manifest text doesn't match
 456                                 // PDH that was requested.
 457                                 var col arvados.Collection
 458                                 col.PortableDataHash = "99999999999999999999999999999999+99"
 459                                 col.ManifestText = `. 6a4ff0499484c6c79c95cd8c566bd25f\+249025 0:249025:GNU_General_Public_License,_version_3.pdf
 460 `
 461                                 enc := json.NewEncoder(w)
 462                                 enc.Encode(col)
 463                         }),
 464                 },
 465         }
 466
 467         c.Assert(srv2.Start(), check.IsNil)
 468         defer srv2.Close()
 469
 470         // Direct zzzzz to service that returns a 200 result with a bogus manifest_text
 471         s.testHandler.Cluster.RemoteClusters["zzzzz"] = arvados.RemoteCluster{
 472                 Host:   srv2.Addr,
 473                 Proxy:  true,
 474                 Scheme: "http",
 475         }
 476
 477         req := httptest.NewRequest("GET", "/arvados/v1/collections/99999999999999999999999999999999+99", nil)
 478         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 479
 480         resp := s.testRequest(req)
 481         defer resp.Body.Close()
 482
 483         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 484 }
 485
 486 func (s *FederationSuite) TestSaltedTokenGetCollectionByPDH(c *check.C) {
 487         arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
 488
 489         req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
 490         req.Header.Set("Authorization", "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/282d7d172b6cfdce364c5ed12ddf7417b2d00065")
 491         resp := s.testRequest(req)
 492
 493         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 494         var col arvados.Collection
 495         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
 496         c.Check(col.PortableDataHash, check.Equals, arvadostest.UserAgreementPDH)
 497         c.Check(col.ManifestText, check.Matches,
 498                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+A[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
 499 `)
 500 }
 501
 502 func (s *FederationSuite) TestSaltedTokenGetCollectionByPDHError(c *check.C) {
 503         arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
 504
 505         req := httptest.NewRequest("GET", "/arvados/v1/collections/99999999999999999999999999999999+99", nil)
 506         req.Header.Set("Authorization", "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/282d7d172b6cfdce364c5ed12ddf7417b2d00065")
 507         resp := s.testRequest(req)
 508
 509         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 510 }
 511
 512 func (s *FederationSuite) TestGetRemoteContainerRequest(c *check.C) {
 513         defer s.localServiceReturns404(c).Close()
 514         req := httptest.NewRequest("GET", "/arvados/v1/container_requests/"+arvadostest.QueuedContainerRequestUUID, nil)
 515         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 516         resp := s.testRequest(req)
 517         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 518         var cr arvados.ContainerRequest
 519         c.Check(json.NewDecoder(resp.Body).Decode(&cr), check.IsNil)
 520         c.Check(cr.UUID, check.Equals, arvadostest.QueuedContainerRequestUUID)
 521         c.Check(cr.Priority, check.Equals, 1)
 522 }
 523
 524 func (s *FederationSuite) TestUpdateRemoteContainerRequest(c *check.C) {
 525         defer s.localServiceReturns404(c).Close()
 526         setPri := func(pri int) {
 527                 req := httptest.NewRequest("PATCH", "/arvados/v1/container_requests/"+arvadostest.QueuedContainerRequestUUID,
 528                         strings.NewReader(fmt.Sprintf(`{"container_request": {"priority": %d}}`, pri)))
 529                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 530                 req.Header.Set("Content-type", "application/json")
 531                 resp := s.testRequest(req)
 532                 c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 533                 var cr arvados.ContainerRequest
 534                 c.Check(json.NewDecoder(resp.Body).Decode(&cr), check.IsNil)
 535                 c.Check(cr.UUID, check.Equals, arvadostest.QueuedContainerRequestUUID)
 536                 c.Check(cr.Priority, check.Equals, pri)
 537         }
 538         setPri(696)
 539         setPri(1) // Reset fixture so side effect doesn't break other tests.
 540 }
 541
 542 func (s *FederationSuite) TestCreateRemoteContainerRequest(c *check.C) {
 543         defer s.localServiceReturns404(c).Close()
 544         // pass cluster_id via query parameter, this allows arvados-controller
 545         // to avoid parsing the body
 546         req := httptest.NewRequest("POST", "/arvados/v1/container_requests?cluster_id=zzzzz",
 547                 strings.NewReader(`{
 548   "container_request": {
 549     "name": "hello world",
 550     "state": "Uncommitted",
 551     "output_path": "/",
 552     "container_image": "123",
 553     "command": ["abc"]
 554   }
 555 }
 556 `))
 557         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 558         req.Header.Set("Content-type", "application/json")
 559         resp := s.testRequest(req)
 560         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 561         var cr arvados.ContainerRequest
 562         c.Check(json.NewDecoder(resp.Body).Decode(&cr), check.IsNil)
 563         c.Check(cr.Name, check.Equals, "hello world")
 564         c.Check(strings.HasPrefix(cr.UUID, "zzzzz-"), check.Equals, true)
 565 }
 566
 567 func (s *FederationSuite) TestCreateRemoteContainerRequestCheckRuntimeToken(c *check.C) {
 568         // Send request to zmock and check that outgoing request has
 569         // runtime_token set with a new random v2 token.
 570
 571         defer s.localServiceReturns404(c).Close()
 572         // pass cluster_id via query parameter, this allows arvados-controller
 573         // to avoid parsing the body
 574         req := httptest.NewRequest("POST", "/arvados/v1/container_requests?cluster_id=zmock",
 575                 strings.NewReader(`{
 576   "container_request": {
 577     "name": "hello world",
 578     "state": "Uncommitted",
 579     "output_path": "/",
 580     "container_image": "123",
 581     "command": ["abc"]
 582   }
 583 }
 584 `))
 585         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
 586         req.Header.Set("Content-type", "application/json")
 587
 588         arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
 589         s.testHandler.Cluster.ClusterID = "zzzzz"
 590
 591         resp := s.testRequest(req)
 592         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 593         var cr struct {
 594                 arvados.ContainerRequest `json:"container_request"`
 595         }
 596         c.Check(json.NewDecoder(s.remoteMockRequests[0].Body).Decode(&cr), check.IsNil)
 597         c.Check(strings.HasPrefix(cr.ContainerRequest.RuntimeToken, "v2/zzzzz-gj3su-"), check.Equals, true)
 598         c.Check(cr.ContainerRequest.RuntimeToken, check.Not(check.Equals), arvadostest.ActiveTokenV2)
 599 }
 600
 601 func (s *FederationSuite) TestCreateRemoteContainerRequestCheckSetRuntimeToken(c *check.C) {
 602         // Send request to zmock and check that outgoing request has
 603         // runtime_token set with the explicitly provided token.
 604
 605         defer s.localServiceReturns404(c).Close()
 606         // pass cluster_id via query parameter, this allows arvados-controller
 607         // to avoid parsing the body
 608         req := httptest.NewRequest("POST", "/arvados/v1/container_requests?cluster_id=zmock",
 609                 strings.NewReader(`{
 610   "container_request": {
 611     "name": "hello world",
 612     "state": "Uncommitted",
 613     "output_path": "/",
 614     "container_image": "123",
 615     "command": ["abc"],
 616     "runtime_token": "xyz"
 617   }
 618 }
 619 `))
 620         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 621         req.Header.Set("Content-type", "application/json")
 622         resp := s.testRequest(req)
 623         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 624         var cr struct {
 625                 arvados.ContainerRequest `json:"container_request"`
 626         }
 627         c.Check(json.NewDecoder(s.remoteMockRequests[0].Body).Decode(&cr), check.IsNil)
 628         c.Check(cr.ContainerRequest.RuntimeToken, check.Equals, "xyz")
 629 }
 630
 631 func (s *FederationSuite) TestCreateRemoteContainerRequestRuntimeTokenFromAuth(c *check.C) {
 632         // Send request to zmock and check that outgoing request has
 633         // runtime_token set using the Auth token because the user is remote.
 634
 635         defer s.localServiceReturns404(c).Close()
 636         // pass cluster_id via query parameter, this allows arvados-controller
 637         // to avoid parsing the body
 638         req := httptest.NewRequest("POST", "/arvados/v1/container_requests?cluster_id=zmock",
 639                 strings.NewReader(`{
 640   "container_request": {
 641     "name": "hello world",
 642     "state": "Uncommitted",
 643     "output_path": "/",
 644     "container_image": "123",
 645     "command": ["abc"]
 646   }
 647 }
 648 `))
 649         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2+"/zzzzz-dz642-parentcontainer")
 650         req.Header.Set("Content-type", "application/json")
 651         resp := s.testRequest(req)
 652         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 653         var cr struct {
 654                 arvados.ContainerRequest `json:"container_request"`
 655         }
 656         c.Check(json.NewDecoder(s.remoteMockRequests[0].Body).Decode(&cr), check.IsNil)
 657         c.Check(cr.ContainerRequest.RuntimeToken, check.Equals, arvadostest.ActiveTokenV2)
 658 }
 659
 660 func (s *FederationSuite) TestCreateRemoteContainerRequestError(c *check.C) {
 661         defer s.localServiceReturns404(c).Close()
 662         // pass cluster_id via query parameter, this allows arvados-controller
 663         // to avoid parsing the body
 664         req := httptest.NewRequest("POST", "/arvados/v1/container_requests?cluster_id=zz404",
 665                 strings.NewReader(`{
 666   "container_request": {
 667     "name": "hello world",
 668     "state": "Uncommitted",
 669     "output_path": "/",
 670     "container_image": "123",
 671     "command": ["abc"]
 672   }
 673 }
 674 `))
 675         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 676         req.Header.Set("Content-type", "application/json")
 677         resp := s.testRequest(req)
 678         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 679 }
 680
 681 func (s *FederationSuite) TestGetRemoteContainer(c *check.C) {
 682         defer s.localServiceReturns404(c).Close()
 683         req := httptest.NewRequest("GET", "/arvados/v1/containers/"+arvadostest.QueuedContainerUUID, nil)
 684         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 685         resp := s.testRequest(req)
 686         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 687         var cn arvados.Container
 688         c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
 689         c.Check(cn.UUID, check.Equals, arvadostest.QueuedContainerUUID)
 690 }
 691
 692 func (s *FederationSuite) TestListRemoteContainer(c *check.C) {
 693         defer s.localServiceReturns404(c).Close()
 694         req := httptest.NewRequest("GET", "/arvados/v1/containers?count=none&filters="+
 695                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v"]]]`, arvadostest.QueuedContainerUUID)), nil)
 696         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 697         resp := s.testRequest(req)
 698         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 699         var cn arvados.ContainerList
 700         c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
 701         c.Check(cn.Items[0].UUID, check.Equals, arvadostest.QueuedContainerUUID)
 702 }
 703
 704 func (s *FederationSuite) TestListMultiRemoteContainers(c *check.C) {
 705         defer s.localServiceHandler(c, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 706                 bd, _ := ioutil.ReadAll(req.Body)
 707                 c.Check(string(bd), check.Equals, `_method=GET&count=none&filters=%5B%5B%22uuid%22%2C+%22in%22%2C+%5B%22zhome-xvhdp-cr5queuedcontnr%22%5D%5D%5D&select=%5B%22uuid%22%2C+%22command%22%5D`)
 708                 w.WriteHeader(200)
 709                 w.Write([]byte(`{"kind": "arvados#containerList", "items": [{"uuid": "zhome-xvhdp-cr5queuedcontnr", "command": ["abc"]}]}`))
 710         })).Close()
 711         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&select=%s",
 712                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
 713                         arvadostest.QueuedContainerUUID)),
 714                 url.QueryEscape(`["uuid", "command"]`)),
 715                 nil)
 716         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 717         resp := s.testRequest(req)
 718         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 719         var cn arvados.ContainerList
 720         c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
 721         c.Check(cn.Items, check.HasLen, 2)
 722         mp := make(map[string]arvados.Container)
 723         for _, cr := range cn.Items {
 724                 mp[cr.UUID] = cr
 725         }
 726         c.Check(mp[arvadostest.QueuedContainerUUID].Command, check.DeepEquals, []string{"echo", "hello"})
 727         c.Check(mp[arvadostest.QueuedContainerUUID].ContainerImage, check.Equals, "")
 728         c.Check(mp["zhome-xvhdp-cr5queuedcontnr"].Command, check.DeepEquals, []string{"abc"})
 729         c.Check(mp["zhome-xvhdp-cr5queuedcontnr"].ContainerImage, check.Equals, "")
 730 }
 731
 732 func (s *FederationSuite) TestListMultiRemoteContainerError(c *check.C) {
 733         defer s.localServiceReturns404(c).Close()
 734         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&select=%s",
 735                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
 736                         arvadostest.QueuedContainerUUID)),
 737                 url.QueryEscape(`["uuid", "command"]`)),
 738                 nil)
 739         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 740         resp := s.testRequest(req)
 741         c.Check(resp.StatusCode, check.Equals, http.StatusBadGateway)
 742         s.checkJSONErrorMatches(c, resp, `error fetching from zhome \(404 Not Found\): EOF`)
 743 }
 744
 745 func (s *FederationSuite) TestListMultiRemoteContainersPaged(c *check.C) {
 746
 747         callCount := 0
 748         defer s.localServiceHandler(c, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 749                 bd, _ := ioutil.ReadAll(req.Body)
 750                 if callCount == 0 {
 751                         c.Check(string(bd), check.Equals, `_method=GET&count=none&filters=%5B%5B%22uuid%22%2C+%22in%22%2C+%5B%22zhome-xvhdp-cr5queuedcontnr%22%2C%22zhome-xvhdp-cr6queuedcontnr%22%5D%5D%5D`)
 752                         w.WriteHeader(200)
 753                         w.Write([]byte(`{"kind": "arvados#containerList", "items": [{"uuid": "zhome-xvhdp-cr5queuedcontnr", "command": ["abc"]}]}`))
 754                 } else if callCount == 1 {
 755                         c.Check(string(bd), check.Equals, `_method=GET&count=none&filters=%5B%5B%22uuid%22%2C+%22in%22%2C+%5B%22zhome-xvhdp-cr6queuedcontnr%22%5D%5D%5D`)
 756                         w.WriteHeader(200)
 757                         w.Write([]byte(`{"kind": "arvados#containerList", "items": [{"uuid": "zhome-xvhdp-cr6queuedcontnr", "command": ["efg"]}]}`))
 758                 }
 759                 callCount += 1
 760         })).Close()
 761         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s",
 762                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr", "zhome-xvhdp-cr6queuedcontnr"]]]`,
 763                         arvadostest.QueuedContainerUUID))),
 764                 nil)
 765         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 766         resp := s.testRequest(req)
 767         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 768         c.Check(callCount, check.Equals, 2)
 769         var cn arvados.ContainerList
 770         c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
 771         c.Check(cn.Items, check.HasLen, 3)
 772         mp := make(map[string]arvados.Container)
 773         for _, cr := range cn.Items {
 774                 mp[cr.UUID] = cr
 775         }
 776         c.Check(mp[arvadostest.QueuedContainerUUID].Command, check.DeepEquals, []string{"echo", "hello"})
 777         c.Check(mp["zhome-xvhdp-cr5queuedcontnr"].Command, check.DeepEquals, []string{"abc"})
 778         c.Check(mp["zhome-xvhdp-cr6queuedcontnr"].Command, check.DeepEquals, []string{"efg"})
 779 }
 780
 781 func (s *FederationSuite) TestListMultiRemoteContainersMissing(c *check.C) {
 782
 783         callCount := 0
 784         defer s.localServiceHandler(c, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 785                 bd, _ := ioutil.ReadAll(req.Body)
 786                 if callCount == 0 {
 787                         c.Check(string(bd), check.Equals, `_method=GET&count=none&filters=%5B%5B%22uuid%22%2C+%22in%22%2C+%5B%22zhome-xvhdp-cr5queuedcontnr%22%2C%22zhome-xvhdp-cr6queuedcontnr%22%5D%5D%5D`)
 788                         w.WriteHeader(200)
 789                         w.Write([]byte(`{"kind": "arvados#containerList", "items": [{"uuid": "zhome-xvhdp-cr6queuedcontnr", "command": ["efg"]}]}`))
 790                 } else if callCount == 1 {
 791                         c.Check(string(bd), check.Equals, `_method=GET&count=none&filters=%5B%5B%22uuid%22%2C+%22in%22%2C+%5B%22zhome-xvhdp-cr5queuedcontnr%22%5D%5D%5D`)
 792                         w.WriteHeader(200)
 793                         w.Write([]byte(`{"kind": "arvados#containerList", "items": []}`))
 794                 }
 795                 callCount += 1
 796         })).Close()
 797         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s",
 798                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr", "zhome-xvhdp-cr6queuedcontnr"]]]`,
 799                         arvadostest.QueuedContainerUUID))),
 800                 nil)
 801         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 802         resp := s.testRequest(req)
 803         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 804         c.Check(callCount, check.Equals, 2)
 805         var cn arvados.ContainerList
 806         c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
 807         c.Check(cn.Items, check.HasLen, 2)
 808         mp := make(map[string]arvados.Container)
 809         for _, cr := range cn.Items {
 810                 mp[cr.UUID] = cr
 811         }
 812         c.Check(mp[arvadostest.QueuedContainerUUID].Command, check.DeepEquals, []string{"echo", "hello"})
 813         c.Check(mp["zhome-xvhdp-cr6queuedcontnr"].Command, check.DeepEquals, []string{"efg"})
 814 }
 815
 816 func (s *FederationSuite) TestListMultiRemoteContainerPageSizeError(c *check.C) {
 817         s.testHandler.Cluster.API.MaxItemsPerResponse = 1
 818         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s",
 819                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
 820                         arvadostest.QueuedContainerUUID))),
 821                 nil)
 822         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 823         resp := s.testRequest(req)
 824         c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
 825         s.checkJSONErrorMatches(c, resp, `Federated multi-object request for 2 objects which is more than max page size 1.`)
 826 }
 827
 828 func (s *FederationSuite) TestListMultiRemoteContainerLimitError(c *check.C) {
 829         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&limit=1",
 830                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
 831                         arvadostest.QueuedContainerUUID))),
 832                 nil)
 833         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 834         resp := s.testRequest(req)
 835         c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
 836         s.checkJSONErrorMatches(c, resp, `Federated multi-object may not provide 'limit', 'offset' or 'order'.`)
 837 }
 838
 839 func (s *FederationSuite) TestListMultiRemoteContainerOffsetError(c *check.C) {
 840         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&offset=1",
 841                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
 842                         arvadostest.QueuedContainerUUID))),
 843                 nil)
 844         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 845         resp := s.testRequest(req)
 846         c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
 847         s.checkJSONErrorMatches(c, resp, `Federated multi-object may not provide 'limit', 'offset' or 'order'.`)
 848 }
 849
 850 func (s *FederationSuite) TestListMultiRemoteContainerOrderError(c *check.C) {
 851         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&order=uuid",
 852                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
 853                         arvadostest.QueuedContainerUUID))),
 854                 nil)
 855         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 856         resp := s.testRequest(req)
 857         c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
 858         s.checkJSONErrorMatches(c, resp, `Federated multi-object may not provide 'limit', 'offset' or 'order'.`)
 859 }
 860
 861 func (s *FederationSuite) TestListMultiRemoteContainerSelectError(c *check.C) {
 862         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&select=%s",
 863                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
 864                         arvadostest.QueuedContainerUUID)),
 865                 url.QueryEscape(`["command"]`)),
 866                 nil)
 867         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 868         resp := s.testRequest(req)
 869         c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
 870         s.checkJSONErrorMatches(c, resp, `Federated multi-object request must include 'uuid' in 'select'`)
 871 }