lib/controller/federation_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package controller
   6
   7 import (
   8         "bytes"
   9         "encoding/json"
  10         "fmt"
  11         "io"
  12         "io/ioutil"
  13         "net/http"
  14         "net/http/httptest"
  15         "net/url"
  16         "os"
  17         "strings"
  18         "time"
  19
  20         "git.curoverse.com/arvados.git/sdk/go/arvados"
  21         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  22         "git.curoverse.com/arvados.git/sdk/go/ctxlog"
  23         "git.curoverse.com/arvados.git/sdk/go/httpserver"
  24         "git.curoverse.com/arvados.git/sdk/go/keepclient"
  25         "github.com/sirupsen/logrus"
  26         check "gopkg.in/check.v1"
  27 )
  28
  29 // Gocheck boilerplate
  30 var _ = check.Suite(&FederationSuite{})
  31
  32 type FederationSuite struct {
  33         log logrus.FieldLogger
  34         // testServer and testHandler are the controller being tested,
  35         // "zhome".
  36         testServer  *httpserver.Server
  37         testHandler *Handler
  38         // remoteServer ("zzzzz") forwards requests to the Rails API
  39         // provided by the integration test environment.
  40         remoteServer *httpserver.Server
  41         // remoteMock ("zmock") appends each incoming request to
  42         // remoteMockRequests, and returns an empty 200 response.
  43         remoteMock         *httpserver.Server
  44         remoteMockRequests []http.Request
  45 }
  46
  47 func (s *FederationSuite) SetUpTest(c *check.C) {
  48         s.log = ctxlog.TestLogger(c)
  49
  50         s.remoteServer = newServerFromIntegrationTestEnv(c)
  51         c.Assert(s.remoteServer.Start(), check.IsNil)
  52
  53         s.remoteMock = newServerFromIntegrationTestEnv(c)
  54         s.remoteMock.Server.Handler = http.HandlerFunc(s.remoteMockHandler)
  55         c.Assert(s.remoteMock.Start(), check.IsNil)
  56
  57         nodeProfile := arvados.NodeProfile{
  58                 Controller: arvados.SystemServiceInstance{Listen: ":"},
  59                 RailsAPI:   arvados.SystemServiceInstance{Listen: ":1"}, // local reqs will error "connection refused"
  60         }
  61         s.testHandler = &Handler{Cluster: &arvados.Cluster{
  62                 ClusterID:  "zhome",
  63                 PostgreSQL: integrationTestCluster().PostgreSQL,
  64                 NodeProfiles: map[string]arvados.NodeProfile{
  65                         "*": nodeProfile,
  66                 },
  67                 RequestLimits: arvados.RequestLimits{
  68                         MaxItemsPerResponse:            1000,
  69                         MultiClusterRequestConcurrency: 4,
  70                 },
  71         }, NodeProfile: &nodeProfile}
  72         s.testServer = newServerFromIntegrationTestEnv(c)
  73         s.testServer.Server.Handler = httpserver.AddRequestIDs(httpserver.LogRequests(s.log, s.testHandler))
  74
  75         s.testHandler.Cluster.RemoteClusters = map[string]arvados.RemoteCluster{
  76                 "zzzzz": {
  77                         Host:   s.remoteServer.Addr,
  78                         Proxy:  true,
  79                         Scheme: "http",
  80                 },
  81                 "zmock": {
  82                         Host:   s.remoteMock.Addr,
  83                         Proxy:  true,
  84                         Scheme: "http",
  85                 },
  86         }
  87
  88         c.Assert(s.testServer.Start(), check.IsNil)
  89
  90         s.remoteMockRequests = nil
  91 }
  92
  93 func (s *FederationSuite) remoteMockHandler(w http.ResponseWriter, req *http.Request) {
  94         b := &bytes.Buffer{}
  95         io.Copy(b, req.Body)
  96         req.Body.Close()
  97         req.Body = ioutil.NopCloser(b)
  98         s.remoteMockRequests = append(s.remoteMockRequests, *req)
  99 }
 100
 101 func (s *FederationSuite) TearDownTest(c *check.C) {
 102         if s.remoteServer != nil {
 103                 s.remoteServer.Close()
 104         }
 105         if s.testServer != nil {
 106                 s.testServer.Close()
 107         }
 108 }
 109
 110 func (s *FederationSuite) testRequest(req *http.Request) *http.Response {
 111         resp := httptest.NewRecorder()
 112         s.testServer.Server.Handler.ServeHTTP(resp, req)
 113         return resp.Result()
 114 }
 115
 116 func (s *FederationSuite) TestLocalRequest(c *check.C) {
 117         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zhome-", 1), nil)
 118         resp := s.testRequest(req)
 119         s.checkHandledLocally(c, resp)
 120 }
 121
 122 func (s *FederationSuite) checkHandledLocally(c *check.C, resp *http.Response) {
 123         // Our "home" controller can't handle local requests because
 124         // it doesn't have its own stub/test Rails API, so we rely on
 125         // "connection refused" to indicate the controller tried to
 126         // proxy the request to its local Rails API.
 127         c.Check(resp.StatusCode, check.Equals, http.StatusBadGateway)
 128         s.checkJSONErrorMatches(c, resp, `.*connection refused`)
 129 }
 130
 131 func (s *FederationSuite) TestNoAuth(c *check.C) {
 132         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
 133         resp := s.testRequest(req)
 134         c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
 135         s.checkJSONErrorMatches(c, resp, `Not logged in`)
 136 }
 137
 138 func (s *FederationSuite) TestBadAuth(c *check.C) {
 139         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
 140         req.Header.Set("Authorization", "Bearer aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
 141         resp := s.testRequest(req)
 142         c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
 143         s.checkJSONErrorMatches(c, resp, `Not logged in`)
 144 }
 145
 146 func (s *FederationSuite) TestNoAccess(c *check.C) {
 147         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
 148         req.Header.Set("Authorization", "Bearer "+arvadostest.SpectatorToken)
 149         resp := s.testRequest(req)
 150         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 151         s.checkJSONErrorMatches(c, resp, `.*not found`)
 152 }
 153
 154 func (s *FederationSuite) TestGetUnknownRemote(c *check.C) {
 155         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zz404-", 1), nil)
 156         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 157         resp := s.testRequest(req)
 158         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 159         s.checkJSONErrorMatches(c, resp, `.*no proxy available for cluster zz404`)
 160 }
 161
 162 func (s *FederationSuite) TestRemoteError(c *check.C) {
 163         rc := s.testHandler.Cluster.RemoteClusters["zzzzz"]
 164         rc.Scheme = "https"
 165         s.testHandler.Cluster.RemoteClusters["zzzzz"] = rc
 166
 167         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
 168         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 169         resp := s.testRequest(req)
 170         c.Check(resp.StatusCode, check.Equals, http.StatusBadGateway)
 171         s.checkJSONErrorMatches(c, resp, `.*HTTP response to HTTPS client`)
 172 }
 173
 174 func (s *FederationSuite) TestGetRemoteWorkflow(c *check.C) {
 175         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
 176         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 177         resp := s.testRequest(req)
 178         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 179         var wf arvados.Workflow
 180         c.Check(json.NewDecoder(resp.Body).Decode(&wf), check.IsNil)
 181         c.Check(wf.UUID, check.Equals, arvadostest.WorkflowWithDefinitionYAMLUUID)
 182         c.Check(wf.OwnerUUID, check.Equals, arvadostest.ActiveUserUUID)
 183 }
 184
 185 func (s *FederationSuite) TestOptionsMethod(c *check.C) {
 186         req := httptest.NewRequest("OPTIONS", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
 187         req.Header.Set("Origin", "https://example.com")
 188         resp := s.testRequest(req)
 189         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 190         body, err := ioutil.ReadAll(resp.Body)
 191         c.Check(err, check.IsNil)
 192         c.Check(string(body), check.Equals, "")
 193         c.Check(resp.Header.Get("Access-Control-Allow-Origin"), check.Equals, "*")
 194         for _, hdr := range []string{"Authorization", "Content-Type"} {
 195                 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), check.Matches, ".*"+hdr+".*")
 196         }
 197         for _, method := range []string{"GET", "HEAD", "PUT", "POST", "DELETE"} {
 198                 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), check.Matches, ".*"+method+".*")
 199         }
 200 }
 201
 202 func (s *FederationSuite) TestRemoteWithTokenInQuery(c *check.C) {
 203         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1)+"?api_token="+arvadostest.ActiveToken, nil)
 204         s.testRequest(req)
 205         c.Assert(s.remoteMockRequests, check.HasLen, 1)
 206         pr := s.remoteMockRequests[0]
 207         // Token is salted and moved from query to Authorization header.
 208         c.Check(pr.URL.String(), check.Not(check.Matches), `.*api_token=.*`)
 209         c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/7fd31b61f39c0e82a4155592163218272cedacdc")
 210 }
 211
 212 func (s *FederationSuite) TestLocalTokenSalted(c *check.C) {
 213         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1), nil)
 214         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 215         s.testRequest(req)
 216         c.Assert(s.remoteMockRequests, check.HasLen, 1)
 217         pr := s.remoteMockRequests[0]
 218         // The salted token here has a "zzzzz-" UUID instead of a
 219         // "ztest-" UUID because ztest's local database has the
 220         // "zzzzz-" test fixtures. The "secret" part is HMAC(sha1,
 221         // arvadostest.ActiveToken, "zmock") = "7fd3...".
 222         c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/7fd31b61f39c0e82a4155592163218272cedacdc")
 223 }
 224
 225 func (s *FederationSuite) TestRemoteTokenNotSalted(c *check.C) {
 226         // remoteToken can be any v1 token that doesn't appear in
 227         // ztest's local db.
 228         remoteToken := "abcdef00000000000000000000000000000000000000000000"
 229         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1), nil)
 230         req.Header.Set("Authorization", "Bearer "+remoteToken)
 231         s.testRequest(req)
 232         c.Assert(s.remoteMockRequests, check.HasLen, 1)
 233         pr := s.remoteMockRequests[0]
 234         c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer "+remoteToken)
 235 }
 236
 237 func (s *FederationSuite) TestWorkflowCRUD(c *check.C) {
 238         wf := arvados.Workflow{
 239                 Description: "TestCRUD",
 240         }
 241         {
 242                 body := &strings.Builder{}
 243                 json.NewEncoder(body).Encode(&wf)
 244                 req := httptest.NewRequest("POST", "/arvados/v1/workflows", strings.NewReader(url.Values{
 245                         "workflow": {body.String()},
 246                 }.Encode()))
 247                 req.Header.Set("Content-type", "application/x-www-form-urlencoded")
 248                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 249                 rec := httptest.NewRecorder()
 250                 s.remoteServer.Server.Handler.ServeHTTP(rec, req) // direct to remote -- can't proxy a create req because no uuid
 251                 resp := rec.Result()
 252                 s.checkResponseOK(c, resp)
 253                 json.NewDecoder(resp.Body).Decode(&wf)
 254
 255                 defer func() {
 256                         req := httptest.NewRequest("DELETE", "/arvados/v1/workflows/"+wf.UUID, nil)
 257                         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 258                         s.remoteServer.Server.Handler.ServeHTTP(httptest.NewRecorder(), req)
 259                 }()
 260                 c.Check(wf.UUID, check.Not(check.Equals), "")
 261
 262                 c.Assert(wf.ModifiedAt, check.NotNil)
 263                 c.Logf("wf.ModifiedAt: %v", wf.ModifiedAt)
 264                 c.Check(time.Since(*wf.ModifiedAt) < time.Minute, check.Equals, true)
 265         }
 266         for _, method := range []string{"PATCH", "PUT", "POST"} {
 267                 form := url.Values{
 268                         "workflow": {`{"description": "Updated with ` + method + `"}`},
 269                 }
 270                 if method == "POST" {
 271                         form["_method"] = []string{"PATCH"}
 272                 }
 273                 req := httptest.NewRequest(method, "/arvados/v1/workflows/"+wf.UUID, strings.NewReader(form.Encode()))
 274                 req.Header.Set("Content-type", "application/x-www-form-urlencoded")
 275                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 276                 resp := s.testRequest(req)
 277                 s.checkResponseOK(c, resp)
 278                 err := json.NewDecoder(resp.Body).Decode(&wf)
 279                 c.Check(err, check.IsNil)
 280
 281                 c.Check(wf.Description, check.Equals, "Updated with "+method)
 282         }
 283         {
 284                 req := httptest.NewRequest("DELETE", "/arvados/v1/workflows/"+wf.UUID, nil)
 285                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 286                 resp := s.testRequest(req)
 287                 s.checkResponseOK(c, resp)
 288                 err := json.NewDecoder(resp.Body).Decode(&wf)
 289                 c.Check(err, check.IsNil)
 290         }
 291         {
 292                 req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+wf.UUID, nil)
 293                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 294                 resp := s.testRequest(req)
 295                 c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 296         }
 297 }
 298
 299 func (s *FederationSuite) checkResponseOK(c *check.C, resp *http.Response) {
 300         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 301         if resp.StatusCode != http.StatusOK {
 302                 body, err := ioutil.ReadAll(resp.Body)
 303                 c.Logf("... response body = %q, %v\n", body, err)
 304         }
 305 }
 306
 307 func (s *FederationSuite) checkJSONErrorMatches(c *check.C, resp *http.Response, re string) {
 308         var jresp httpserver.ErrorResponse
 309         err := json.NewDecoder(resp.Body).Decode(&jresp)
 310         c.Check(err, check.IsNil)
 311         c.Assert(jresp.Errors, check.HasLen, 1)
 312         c.Check(jresp.Errors[0], check.Matches, re)
 313 }
 314
 315 func (s *FederationSuite) localServiceHandler(c *check.C, h http.Handler) *httpserver.Server {
 316         srv := &httpserver.Server{
 317                 Server: http.Server{
 318                         Handler: h,
 319                 },
 320         }
 321
 322         c.Assert(srv.Start(), check.IsNil)
 323
 324         np := arvados.NodeProfile{
 325                 Controller: arvados.SystemServiceInstance{Listen: ":"},
 326                 RailsAPI: arvados.SystemServiceInstance{Listen: srv.Addr,
 327                         TLS: false, Insecure: true}}
 328         s.testHandler.Cluster.NodeProfiles["*"] = np
 329         s.testHandler.NodeProfile = &np
 330
 331         return srv
 332 }
 333
 334 func (s *FederationSuite) localServiceReturns404(c *check.C) *httpserver.Server {
 335         return s.localServiceHandler(c, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 336                 w.WriteHeader(404)
 337         }))
 338 }
 339
 340 func (s *FederationSuite) TestGetLocalCollection(c *check.C) {
 341         np := arvados.NodeProfile{
 342                 Controller: arvados.SystemServiceInstance{Listen: ":"},
 343                 RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"),
 344                         TLS: true, Insecure: true}}
 345         s.testHandler.Cluster.ClusterID = "zzzzz"
 346         s.testHandler.Cluster.NodeProfiles["*"] = np
 347         s.testHandler.NodeProfile = &np
 348
 349         // HTTP GET
 350
 351         req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementCollection, nil)
 352         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 353         resp := s.testRequest(req)
 354
 355         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 356         var col arvados.Collection
 357         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
 358         c.Check(col.UUID, check.Equals, arvadostest.UserAgreementCollection)
 359         c.Check(col.ManifestText, check.Matches,
 360                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+A[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
 361 `)
 362
 363         // HTTP POST with _method=GET as a form parameter
 364
 365         req = httptest.NewRequest("POST", "/arvados/v1/collections/"+arvadostest.UserAgreementCollection, bytes.NewBufferString((url.Values{
 366                 "_method": {"GET"},
 367         }).Encode()))
 368         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 369         req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
 370         resp = s.testRequest(req)
 371
 372         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 373         col = arvados.Collection{}
 374         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
 375         c.Check(col.UUID, check.Equals, arvadostest.UserAgreementCollection)
 376         c.Check(col.ManifestText, check.Matches,
 377                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+A[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
 378 `)
 379 }
 380
 381 func (s *FederationSuite) TestGetRemoteCollection(c *check.C) {
 382         defer s.localServiceReturns404(c).Close()
 383
 384         req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementCollection, nil)
 385         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 386         resp := s.testRequest(req)
 387         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 388         var col arvados.Collection
 389         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
 390         c.Check(col.UUID, check.Equals, arvadostest.UserAgreementCollection)
 391         c.Check(col.ManifestText, check.Matches,
 392                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+Rzzzzz-[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
 393 `)
 394 }
 395
 396 func (s *FederationSuite) TestGetRemoteCollectionError(c *check.C) {
 397         defer s.localServiceReturns404(c).Close()
 398
 399         req := httptest.NewRequest("GET", "/arvados/v1/collections/zzzzz-4zz18-fakefakefakefak", nil)
 400         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 401         resp := s.testRequest(req)
 402         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 403 }
 404
 405 func (s *FederationSuite) TestSignedLocatorPattern(c *check.C) {
 406         // Confirm the regular expression identifies other groups of hints correctly
 407         c.Check(keepclient.SignedLocatorRe.FindStringSubmatch(`6a4ff0499484c6c79c95cd8c566bd25f+249025+B1+C2+A05227438989d04712ea9ca1c91b556cef01d5cc7@5ba5405b+D3+E4`),
 408                 check.DeepEquals,
 409                 []string{"6a4ff0499484c6c79c95cd8c566bd25f+249025+B1+C2+A05227438989d04712ea9ca1c91b556cef01d5cc7@5ba5405b+D3+E4",
 410                         "6a4ff0499484c6c79c95cd8c566bd25f",
 411                         "+249025",
 412                         "+B1+C2", "+C2",
 413                         "+A05227438989d04712ea9ca1c91b556cef01d5cc7@5ba5405b",
 414                         "05227438989d04712ea9ca1c91b556cef01d5cc7", "5ba5405b",
 415                         "+D3+E4", "+E4"})
 416 }
 417
 418 func (s *FederationSuite) TestGetLocalCollectionByPDH(c *check.C) {
 419         np := arvados.NodeProfile{
 420                 Controller: arvados.SystemServiceInstance{Listen: ":"},
 421                 RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"),
 422                         TLS: true, Insecure: true}}
 423         s.testHandler.Cluster.NodeProfiles["*"] = np
 424         s.testHandler.NodeProfile = &np
 425
 426         req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
 427         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 428         resp := s.testRequest(req)
 429
 430         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 431         var col arvados.Collection
 432         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
 433         c.Check(col.PortableDataHash, check.Equals, arvadostest.UserAgreementPDH)
 434         c.Check(col.ManifestText, check.Matches,
 435                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+A[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
 436 `)
 437 }
 438
 439 func (s *FederationSuite) TestGetRemoteCollectionByPDH(c *check.C) {
 440         defer s.localServiceReturns404(c).Close()
 441
 442         req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
 443         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 444         resp := s.testRequest(req)
 445
 446         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 447
 448         var col arvados.Collection
 449         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
 450         c.Check(col.PortableDataHash, check.Equals, arvadostest.UserAgreementPDH)
 451         c.Check(col.ManifestText, check.Matches,
 452                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+Rzzzzz-[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
 453 `)
 454 }
 455
 456 func (s *FederationSuite) TestGetCollectionByPDHError(c *check.C) {
 457         defer s.localServiceReturns404(c).Close()
 458
 459         req := httptest.NewRequest("GET", "/arvados/v1/collections/99999999999999999999999999999999+99", nil)
 460         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 461
 462         resp := s.testRequest(req)
 463         defer resp.Body.Close()
 464
 465         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 466 }
 467
 468 func (s *FederationSuite) TestGetCollectionByPDHErrorBadHash(c *check.C) {
 469         defer s.localServiceReturns404(c).Close()
 470
 471         srv2 := &httpserver.Server{
 472                 Server: http.Server{
 473                         Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 474                                 w.WriteHeader(200)
 475                                 // Return a collection where the hash
 476                                 // of the manifest text doesn't match
 477                                 // PDH that was requested.
 478                                 var col arvados.Collection
 479                                 col.PortableDataHash = "99999999999999999999999999999999+99"
 480                                 col.ManifestText = `. 6a4ff0499484c6c79c95cd8c566bd25f\+249025 0:249025:GNU_General_Public_License,_version_3.pdf
 481 `
 482                                 enc := json.NewEncoder(w)
 483                                 enc.Encode(col)
 484                         }),
 485                 },
 486         }
 487
 488         c.Assert(srv2.Start(), check.IsNil)
 489         defer srv2.Close()
 490
 491         // Direct zzzzz to service that returns a 200 result with a bogus manifest_text
 492         s.testHandler.Cluster.RemoteClusters["zzzzz"] = arvados.RemoteCluster{
 493                 Host:   srv2.Addr,
 494                 Proxy:  true,
 495                 Scheme: "http",
 496         }
 497
 498         req := httptest.NewRequest("GET", "/arvados/v1/collections/99999999999999999999999999999999+99", nil)
 499         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 500
 501         resp := s.testRequest(req)
 502         defer resp.Body.Close()
 503
 504         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 505 }
 506
 507 func (s *FederationSuite) TestSaltedTokenGetCollectionByPDH(c *check.C) {
 508         np := arvados.NodeProfile{
 509                 Controller: arvados.SystemServiceInstance{Listen: ":"},
 510                 RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"),
 511                         TLS: true, Insecure: true}}
 512         s.testHandler.Cluster.NodeProfiles["*"] = np
 513         s.testHandler.NodeProfile = &np
 514
 515         req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
 516         req.Header.Set("Authorization", "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/282d7d172b6cfdce364c5ed12ddf7417b2d00065")
 517         resp := s.testRequest(req)
 518
 519         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 520         var col arvados.Collection
 521         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
 522         c.Check(col.PortableDataHash, check.Equals, arvadostest.UserAgreementPDH)
 523         c.Check(col.ManifestText, check.Matches,
 524                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+A[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
 525 `)
 526 }
 527
 528 func (s *FederationSuite) TestSaltedTokenGetCollectionByPDHError(c *check.C) {
 529         np := arvados.NodeProfile{
 530                 Controller: arvados.SystemServiceInstance{Listen: ":"},
 531                 RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"),
 532                         TLS: true, Insecure: true}}
 533         s.testHandler.Cluster.NodeProfiles["*"] = np
 534         s.testHandler.NodeProfile = &np
 535
 536         req := httptest.NewRequest("GET", "/arvados/v1/collections/99999999999999999999999999999999+99", nil)
 537         req.Header.Set("Authorization", "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/282d7d172b6cfdce364c5ed12ddf7417b2d00065")
 538         resp := s.testRequest(req)
 539
 540         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 541 }
 542
 543 func (s *FederationSuite) TestGetRemoteContainerRequest(c *check.C) {
 544         defer s.localServiceReturns404(c).Close()
 545         req := httptest.NewRequest("GET", "/arvados/v1/container_requests/"+arvadostest.QueuedContainerRequestUUID, nil)
 546         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 547         resp := s.testRequest(req)
 548         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 549         var cr arvados.ContainerRequest
 550         c.Check(json.NewDecoder(resp.Body).Decode(&cr), check.IsNil)
 551         c.Check(cr.UUID, check.Equals, arvadostest.QueuedContainerRequestUUID)
 552         c.Check(cr.Priority, check.Equals, 1)
 553 }
 554
 555 func (s *FederationSuite) TestUpdateRemoteContainerRequest(c *check.C) {
 556         defer s.localServiceReturns404(c).Close()
 557         setPri := func(pri int) {
 558                 req := httptest.NewRequest("PATCH", "/arvados/v1/container_requests/"+arvadostest.QueuedContainerRequestUUID,
 559                         strings.NewReader(fmt.Sprintf(`{"container_request": {"priority": %d}}`, pri)))
 560                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 561                 req.Header.Set("Content-type", "application/json")
 562                 resp := s.testRequest(req)
 563                 c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 564                 var cr arvados.ContainerRequest
 565                 c.Check(json.NewDecoder(resp.Body).Decode(&cr), check.IsNil)
 566                 c.Check(cr.UUID, check.Equals, arvadostest.QueuedContainerRequestUUID)
 567                 c.Check(cr.Priority, check.Equals, pri)
 568         }
 569         setPri(696)
 570         setPri(1) // Reset fixture so side effect doesn't break other tests.
 571 }
 572
 573 func (s *FederationSuite) TestCreateRemoteContainerRequest(c *check.C) {
 574         defer s.localServiceReturns404(c).Close()
 575         // pass cluster_id via query parameter, this allows arvados-controller
 576         // to avoid parsing the body
 577         req := httptest.NewRequest("POST", "/arvados/v1/container_requests?cluster_id=zzzzz",
 578                 strings.NewReader(`{
 579   "container_request": {
 580     "name": "hello world",
 581     "state": "Uncommitted",
 582     "output_path": "/",
 583     "container_image": "123",
 584     "command": ["abc"]
 585   }
 586 }
 587 `))
 588         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 589         req.Header.Set("Content-type", "application/json")
 590         resp := s.testRequest(req)
 591         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 592         var cr arvados.ContainerRequest
 593         c.Check(json.NewDecoder(resp.Body).Decode(&cr), check.IsNil)
 594         c.Check(cr.Name, check.Equals, "hello world")
 595         c.Check(strings.HasPrefix(cr.UUID, "zzzzz-"), check.Equals, true)
 596 }
 597
 598 func (s *FederationSuite) TestCreateRemoteContainerRequestCheckRuntimeToken(c *check.C) {
 599         // Send request to zmock and check that outgoing request has
 600         // runtime_token set with a new random v2 token.
 601
 602         defer s.localServiceReturns404(c).Close()
 603         // pass cluster_id via query parameter, this allows arvados-controller
 604         // to avoid parsing the body
 605         req := httptest.NewRequest("POST", "/arvados/v1/container_requests?cluster_id=zmock",
 606                 strings.NewReader(`{
 607   "container_request": {
 608     "name": "hello world",
 609     "state": "Uncommitted",
 610     "output_path": "/",
 611     "container_image": "123",
 612     "command": ["abc"]
 613   }
 614 }
 615 `))
 616         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
 617         req.Header.Set("Content-type", "application/json")
 618
 619         np := arvados.NodeProfile{
 620                 Controller: arvados.SystemServiceInstance{Listen: ":"},
 621                 RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"),
 622                         TLS: true, Insecure: true}}
 623         s.testHandler.Cluster.ClusterID = "zzzzz"
 624         s.testHandler.Cluster.NodeProfiles["*"] = np
 625         s.testHandler.NodeProfile = &np
 626
 627         resp := s.testRequest(req)
 628         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 629         var cr struct {
 630                 arvados.ContainerRequest `json:"container_request"`
 631         }
 632         c.Check(json.NewDecoder(s.remoteMockRequests[0].Body).Decode(&cr), check.IsNil)
 633         c.Check(strings.HasPrefix(cr.ContainerRequest.RuntimeToken, "v2/zzzzz-gj3su-"), check.Equals, true)
 634         c.Check(cr.ContainerRequest.RuntimeToken, check.Not(check.Equals), arvadostest.ActiveTokenV2)
 635 }
 636
 637 func (s *FederationSuite) TestCreateRemoteContainerRequestCheckSetRuntimeToken(c *check.C) {
 638         // Send request to zmock and check that outgoing request has
 639         // runtime_token set with the explicitly provided token.
 640
 641         defer s.localServiceReturns404(c).Close()
 642         // pass cluster_id via query parameter, this allows arvados-controller
 643         // to avoid parsing the body
 644         req := httptest.NewRequest("POST", "/arvados/v1/container_requests?cluster_id=zmock",
 645                 strings.NewReader(`{
 646   "container_request": {
 647     "name": "hello world",
 648     "state": "Uncommitted",
 649     "output_path": "/",
 650     "container_image": "123",
 651     "command": ["abc"],
 652     "runtime_token": "xyz"
 653   }
 654 }
 655 `))
 656         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 657         req.Header.Set("Content-type", "application/json")
 658         resp := s.testRequest(req)
 659         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 660         var cr struct {
 661                 arvados.ContainerRequest `json:"container_request"`
 662         }
 663         c.Check(json.NewDecoder(s.remoteMockRequests[0].Body).Decode(&cr), check.IsNil)
 664         c.Check(cr.ContainerRequest.RuntimeToken, check.Equals, "xyz")
 665 }
 666
 667 func (s *FederationSuite) TestCreateRemoteContainerRequestRuntimeTokenFromAuth(c *check.C) {
 668         // Send request to zmock and check that outgoing request has
 669         // runtime_token set using the Auth token because the user is remote.
 670
 671         defer s.localServiceReturns404(c).Close()
 672         // pass cluster_id via query parameter, this allows arvados-controller
 673         // to avoid parsing the body
 674         req := httptest.NewRequest("POST", "/arvados/v1/container_requests?cluster_id=zmock",
 675                 strings.NewReader(`{
 676   "container_request": {
 677     "name": "hello world",
 678     "state": "Uncommitted",
 679     "output_path": "/",
 680     "container_image": "123",
 681     "command": ["abc"]
 682   }
 683 }
 684 `))
 685         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2+"/zzzzz-dz642-parentcontainer")
 686         req.Header.Set("Content-type", "application/json")
 687         resp := s.testRequest(req)
 688         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 689         var cr struct {
 690                 arvados.ContainerRequest `json:"container_request"`
 691         }
 692         c.Check(json.NewDecoder(s.remoteMockRequests[0].Body).Decode(&cr), check.IsNil)
 693         c.Check(cr.ContainerRequest.RuntimeToken, check.Equals, arvadostest.ActiveTokenV2)
 694 }
 695
 696 func (s *FederationSuite) TestCreateRemoteContainerRequestError(c *check.C) {
 697         defer s.localServiceReturns404(c).Close()
 698         // pass cluster_id via query parameter, this allows arvados-controller
 699         // to avoid parsing the body
 700         req := httptest.NewRequest("POST", "/arvados/v1/container_requests?cluster_id=zz404",
 701                 strings.NewReader(`{
 702   "container_request": {
 703     "name": "hello world",
 704     "state": "Uncommitted",
 705     "output_path": "/",
 706     "container_image": "123",
 707     "command": ["abc"]
 708   }
 709 }
 710 `))
 711         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 712         req.Header.Set("Content-type", "application/json")
 713         resp := s.testRequest(req)
 714         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 715 }
 716
 717 func (s *FederationSuite) TestGetRemoteContainer(c *check.C) {
 718         defer s.localServiceReturns404(c).Close()
 719         req := httptest.NewRequest("GET", "/arvados/v1/containers/"+arvadostest.QueuedContainerUUID, nil)
 720         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 721         resp := s.testRequest(req)
 722         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 723         var cn arvados.Container
 724         c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
 725         c.Check(cn.UUID, check.Equals, arvadostest.QueuedContainerUUID)
 726 }
 727
 728 func (s *FederationSuite) TestListRemoteContainer(c *check.C) {
 729         defer s.localServiceReturns404(c).Close()
 730         req := httptest.NewRequest("GET", "/arvados/v1/containers?count=none&filters="+
 731                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v"]]]`, arvadostest.QueuedContainerUUID)), nil)
 732         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 733         resp := s.testRequest(req)
 734         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 735         var cn arvados.ContainerList
 736         c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
 737         c.Check(cn.Items[0].UUID, check.Equals, arvadostest.QueuedContainerUUID)
 738 }
 739
 740 func (s *FederationSuite) TestListMultiRemoteContainers(c *check.C) {
 741         defer s.localServiceHandler(c, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 742                 bd, _ := ioutil.ReadAll(req.Body)
 743                 c.Check(string(bd), check.Equals, `_method=GET&count=none&filters=%5B%5B%22uuid%22%2C+%22in%22%2C+%5B%22zhome-xvhdp-cr5queuedcontnr%22%5D%5D%5D&select=%5B%22uuid%22%2C+%22command%22%5D`)
 744                 w.WriteHeader(200)
 745                 w.Write([]byte(`{"kind": "arvados#containerList", "items": [{"uuid": "zhome-xvhdp-cr5queuedcontnr", "command": ["abc"]}]}`))
 746         })).Close()
 747         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&select=%s",
 748                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
 749                         arvadostest.QueuedContainerUUID)),
 750                 url.QueryEscape(`["uuid", "command"]`)),
 751                 nil)
 752         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 753         resp := s.testRequest(req)
 754         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 755         var cn arvados.ContainerList
 756         c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
 757         c.Check(cn.Items, check.HasLen, 2)
 758         mp := make(map[string]arvados.Container)
 759         for _, cr := range cn.Items {
 760                 mp[cr.UUID] = cr
 761         }
 762         c.Check(mp[arvadostest.QueuedContainerUUID].Command, check.DeepEquals, []string{"echo", "hello"})
 763         c.Check(mp[arvadostest.QueuedContainerUUID].ContainerImage, check.Equals, "")
 764         c.Check(mp["zhome-xvhdp-cr5queuedcontnr"].Command, check.DeepEquals, []string{"abc"})
 765         c.Check(mp["zhome-xvhdp-cr5queuedcontnr"].ContainerImage, check.Equals, "")
 766 }
 767
 768 func (s *FederationSuite) TestListMultiRemoteContainerError(c *check.C) {
 769         defer s.localServiceReturns404(c).Close()
 770         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&select=%s",
 771                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
 772                         arvadostest.QueuedContainerUUID)),
 773                 url.QueryEscape(`["uuid", "command"]`)),
 774                 nil)
 775         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 776         resp := s.testRequest(req)
 777         c.Check(resp.StatusCode, check.Equals, http.StatusBadGateway)
 778         s.checkJSONErrorMatches(c, resp, `error fetching from zhome \(404 Not Found\): EOF`)
 779 }
 780
 781 func (s *FederationSuite) TestListMultiRemoteContainersPaged(c *check.C) {
 782
 783         callCount := 0
 784         defer s.localServiceHandler(c, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 785                 bd, _ := ioutil.ReadAll(req.Body)
 786                 if callCount == 0 {
 787                         c.Check(string(bd), check.Equals, `_method=GET&count=none&filters=%5B%5B%22uuid%22%2C+%22in%22%2C+%5B%22zhome-xvhdp-cr5queuedcontnr%22%2C%22zhome-xvhdp-cr6queuedcontnr%22%5D%5D%5D`)
 788                         w.WriteHeader(200)
 789                         w.Write([]byte(`{"kind": "arvados#containerList", "items": [{"uuid": "zhome-xvhdp-cr5queuedcontnr", "command": ["abc"]}]}`))
 790                 } else if callCount == 1 {
 791                         c.Check(string(bd), check.Equals, `_method=GET&count=none&filters=%5B%5B%22uuid%22%2C+%22in%22%2C+%5B%22zhome-xvhdp-cr6queuedcontnr%22%5D%5D%5D`)
 792                         w.WriteHeader(200)
 793                         w.Write([]byte(`{"kind": "arvados#containerList", "items": [{"uuid": "zhome-xvhdp-cr6queuedcontnr", "command": ["efg"]}]}`))
 794                 }
 795                 callCount += 1
 796         })).Close()
 797         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s",
 798                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr", "zhome-xvhdp-cr6queuedcontnr"]]]`,
 799                         arvadostest.QueuedContainerUUID))),
 800                 nil)
 801         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 802         resp := s.testRequest(req)
 803         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 804         c.Check(callCount, check.Equals, 2)
 805         var cn arvados.ContainerList
 806         c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
 807         c.Check(cn.Items, check.HasLen, 3)
 808         mp := make(map[string]arvados.Container)
 809         for _, cr := range cn.Items {
 810                 mp[cr.UUID] = cr
 811         }
 812         c.Check(mp[arvadostest.QueuedContainerUUID].Command, check.DeepEquals, []string{"echo", "hello"})
 813         c.Check(mp["zhome-xvhdp-cr5queuedcontnr"].Command, check.DeepEquals, []string{"abc"})
 814         c.Check(mp["zhome-xvhdp-cr6queuedcontnr"].Command, check.DeepEquals, []string{"efg"})
 815 }
 816
 817 func (s *FederationSuite) TestListMultiRemoteContainersMissing(c *check.C) {
 818
 819         callCount := 0
 820         defer s.localServiceHandler(c, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 821                 bd, _ := ioutil.ReadAll(req.Body)
 822                 if callCount == 0 {
 823                         c.Check(string(bd), check.Equals, `_method=GET&count=none&filters=%5B%5B%22uuid%22%2C+%22in%22%2C+%5B%22zhome-xvhdp-cr5queuedcontnr%22%2C%22zhome-xvhdp-cr6queuedcontnr%22%5D%5D%5D`)
 824                         w.WriteHeader(200)
 825                         w.Write([]byte(`{"kind": "arvados#containerList", "items": [{"uuid": "zhome-xvhdp-cr6queuedcontnr", "command": ["efg"]}]}`))
 826                 } else if callCount == 1 {
 827                         c.Check(string(bd), check.Equals, `_method=GET&count=none&filters=%5B%5B%22uuid%22%2C+%22in%22%2C+%5B%22zhome-xvhdp-cr5queuedcontnr%22%5D%5D%5D`)
 828                         w.WriteHeader(200)
 829                         w.Write([]byte(`{"kind": "arvados#containerList", "items": []}`))
 830                 }
 831                 callCount += 1
 832         })).Close()
 833         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s",
 834                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr", "zhome-xvhdp-cr6queuedcontnr"]]]`,
 835                         arvadostest.QueuedContainerUUID))),
 836                 nil)
 837         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 838         resp := s.testRequest(req)
 839         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 840         c.Check(callCount, check.Equals, 2)
 841         var cn arvados.ContainerList
 842         c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
 843         c.Check(cn.Items, check.HasLen, 2)
 844         mp := make(map[string]arvados.Container)
 845         for _, cr := range cn.Items {
 846                 mp[cr.UUID] = cr
 847         }
 848         c.Check(mp[arvadostest.QueuedContainerUUID].Command, check.DeepEquals, []string{"echo", "hello"})
 849         c.Check(mp["zhome-xvhdp-cr6queuedcontnr"].Command, check.DeepEquals, []string{"efg"})
 850 }
 851
 852 func (s *FederationSuite) TestListMultiRemoteContainerPageSizeError(c *check.C) {
 853         s.testHandler.Cluster.RequestLimits.MaxItemsPerResponse = 1
 854         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s",
 855                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
 856                         arvadostest.QueuedContainerUUID))),
 857                 nil)
 858         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 859         resp := s.testRequest(req)
 860         c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
 861         s.checkJSONErrorMatches(c, resp, `Federated multi-object request for 2 objects which is more than max page size 1.`)
 862 }
 863
 864 func (s *FederationSuite) TestListMultiRemoteContainerLimitError(c *check.C) {
 865         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&limit=1",
 866                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
 867                         arvadostest.QueuedContainerUUID))),
 868                 nil)
 869         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 870         resp := s.testRequest(req)
 871         c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
 872         s.checkJSONErrorMatches(c, resp, `Federated multi-object may not provide 'limit', 'offset' or 'order'.`)
 873 }
 874
 875 func (s *FederationSuite) TestListMultiRemoteContainerOffsetError(c *check.C) {
 876         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&offset=1",
 877                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
 878                         arvadostest.QueuedContainerUUID))),
 879                 nil)
 880         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 881         resp := s.testRequest(req)
 882         c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
 883         s.checkJSONErrorMatches(c, resp, `Federated multi-object may not provide 'limit', 'offset' or 'order'.`)
 884 }
 885
 886 func (s *FederationSuite) TestListMultiRemoteContainerOrderError(c *check.C) {
 887         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&order=uuid",
 888                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
 889                         arvadostest.QueuedContainerUUID))),
 890                 nil)
 891         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 892         resp := s.testRequest(req)
 893         c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
 894         s.checkJSONErrorMatches(c, resp, `Federated multi-object may not provide 'limit', 'offset' or 'order'.`)
 895 }
 896
 897 func (s *FederationSuite) TestListMultiRemoteContainerSelectError(c *check.C) {
 898         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&select=%s",
 899                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
 900                         arvadostest.QueuedContainerUUID)),
 901                 url.QueryEscape(`["command"]`)),
 902                 nil)
 903         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 904         resp := s.testRequest(req)
 905         c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
 906         s.checkJSONErrorMatches(c, resp, `Federated multi-object request must include 'uuid' in 'select'`)
 907 }