Merge branch '13697-database-timeout'
[arvados.git] / lib / controller / federation_test.go
1 // Copyright (C) The Arvados Authors. All rights reserved.
2 //
3 // SPDX-License-Identifier: AGPL-3.0
4
5 package controller
6
7 import (
8         "bytes"
9         "context"
10         "encoding/json"
11         "fmt"
12         "io"
13         "io/ioutil"
14         "net"
15         "net/http"
16         "net/http/httptest"
17         "net/url"
18         "os"
19         "strings"
20         "time"
21
22         "git.arvados.org/arvados.git/sdk/go/arvados"
23         "git.arvados.org/arvados.git/sdk/go/arvadostest"
24         "git.arvados.org/arvados.git/sdk/go/ctxlog"
25         "git.arvados.org/arvados.git/sdk/go/httpserver"
26         "git.arvados.org/arvados.git/sdk/go/keepclient"
27         "github.com/sirupsen/logrus"
28         check "gopkg.in/check.v1"
29 )
30
31 // Gocheck boilerplate
32 var _ = check.Suite(&FederationSuite{})
33
34 type FederationSuite struct {
35         log logrus.FieldLogger
36         // testServer and testHandler are the controller being tested,
37         // "zhome".
38         testServer  *httpserver.Server
39         testHandler *Handler
40         // remoteServer ("zzzzz") forwards requests to the Rails API
41         // provided by the integration test environment.
42         remoteServer *httpserver.Server
43         // remoteMock ("zmock") appends each incoming request to
44         // remoteMockRequests, and returns 200 with an empty JSON
45         // object.
46         remoteMock         *httpserver.Server
47         remoteMockRequests []http.Request
48 }
49
50 func (s *FederationSuite) SetUpTest(c *check.C) {
51         s.log = ctxlog.TestLogger(c)
52
53         s.remoteServer = newServerFromIntegrationTestEnv(c)
54         c.Assert(s.remoteServer.Start(), check.IsNil)
55
56         s.remoteMock = newServerFromIntegrationTestEnv(c)
57         s.remoteMock.Server.Handler = http.HandlerFunc(s.remoteMockHandler)
58         c.Assert(s.remoteMock.Start(), check.IsNil)
59
60         cluster := &arvados.Cluster{
61                 ClusterID:  "zhome",
62                 PostgreSQL: integrationTestCluster().PostgreSQL,
63         }
64         cluster.TLS.Insecure = true
65         cluster.API.MaxItemsPerResponse = 1000
66         cluster.API.MaxRequestAmplification = 4
67         cluster.API.RequestTimeout = arvados.Duration(5 * time.Minute)
68         cluster.Collections.BlobSigning = true
69         cluster.Collections.BlobSigningKey = arvadostest.BlobSigningKey
70         cluster.Collections.BlobSigningTTL = arvados.Duration(time.Hour * 24 * 14)
71         arvadostest.SetServiceURL(&cluster.Services.RailsAPI, "http://localhost:1/")
72         arvadostest.SetServiceURL(&cluster.Services.Controller, "http://localhost:/")
73         s.testHandler = &Handler{Cluster: cluster}
74         s.testServer = newServerFromIntegrationTestEnv(c)
75         s.testServer.Server.BaseContext = func(net.Listener) context.Context {
76                 return ctxlog.Context(context.Background(), s.log)
77         }
78         s.testServer.Server.Handler = httpserver.AddRequestIDs(httpserver.LogRequests(s.testHandler))
79
80         cluster.RemoteClusters = map[string]arvados.RemoteCluster{
81                 "zzzzz": {
82                         Host:   s.remoteServer.Addr,
83                         Proxy:  true,
84                         Scheme: "http",
85                 },
86                 "zmock": {
87                         Host:   s.remoteMock.Addr,
88                         Proxy:  true,
89                         Scheme: "http",
90                 },
91                 "*": {
92                         Scheme: "https",
93                 },
94         }
95
96         c.Assert(s.testServer.Start(), check.IsNil)
97
98         s.remoteMockRequests = nil
99 }
100
101 func (s *FederationSuite) remoteMockHandler(w http.ResponseWriter, req *http.Request) {
102         b := &bytes.Buffer{}
103         io.Copy(b, req.Body)
104         req.Body.Close()
105         req.Body = ioutil.NopCloser(b)
106         s.remoteMockRequests = append(s.remoteMockRequests, *req)
107         // Repond 200 with a valid JSON object
108         fmt.Fprint(w, "{}")
109 }
110
111 func (s *FederationSuite) TearDownTest(c *check.C) {
112         if s.remoteServer != nil {
113                 s.remoteServer.Close()
114         }
115         if s.testServer != nil {
116                 s.testServer.Close()
117         }
118 }
119
120 func (s *FederationSuite) testRequest(req *http.Request) *httptest.ResponseRecorder {
121         resp := httptest.NewRecorder()
122         s.testServer.Server.Handler.ServeHTTP(resp, req)
123         return resp
124 }
125
126 func (s *FederationSuite) TestLocalRequest(c *check.C) {
127         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zhome-", 1), nil)
128         resp := s.testRequest(req).Result()
129         s.checkHandledLocally(c, resp)
130 }
131
132 func (s *FederationSuite) checkHandledLocally(c *check.C, resp *http.Response) {
133         // Our "home" controller can't handle local requests because
134         // it doesn't have its own stub/test Rails API, so we rely on
135         // "connection refused" to indicate the controller tried to
136         // proxy the request to its local Rails API.
137         c.Check(resp.StatusCode, check.Equals, http.StatusBadGateway)
138         s.checkJSONErrorMatches(c, resp, `.*connection refused`)
139 }
140
141 func (s *FederationSuite) TestNoAuth(c *check.C) {
142         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
143         resp := s.testRequest(req).Result()
144         c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
145         s.checkJSONErrorMatches(c, resp, `Not logged in.*`)
146 }
147
148 func (s *FederationSuite) TestBadAuth(c *check.C) {
149         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
150         req.Header.Set("Authorization", "Bearer aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
151         resp := s.testRequest(req).Result()
152         c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
153         s.checkJSONErrorMatches(c, resp, `Not logged in.*`)
154 }
155
156 func (s *FederationSuite) TestNoAccess(c *check.C) {
157         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
158         req.Header.Set("Authorization", "Bearer "+arvadostest.SpectatorToken)
159         resp := s.testRequest(req).Result()
160         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
161         s.checkJSONErrorMatches(c, resp, `.*not found.*`)
162 }
163
164 func (s *FederationSuite) TestGetUnknownRemote(c *check.C) {
165         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zz404-", 1), nil)
166         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
167         resp := s.testRequest(req).Result()
168         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
169         s.checkJSONErrorMatches(c, resp, `.*no proxy available for cluster zz404`)
170 }
171
172 func (s *FederationSuite) TestRemoteError(c *check.C) {
173         rc := s.testHandler.Cluster.RemoteClusters["zzzzz"]
174         rc.Scheme = "https"
175         s.testHandler.Cluster.RemoteClusters["zzzzz"] = rc
176
177         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
178         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
179         resp := s.testRequest(req).Result()
180         c.Check(resp.StatusCode, check.Equals, http.StatusBadGateway)
181         s.checkJSONErrorMatches(c, resp, `.*HTTP response to HTTPS client`)
182 }
183
184 func (s *FederationSuite) TestGetRemoteWorkflow(c *check.C) {
185         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
186         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
187         resp := s.testRequest(req).Result()
188         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
189         var wf arvados.Workflow
190         c.Check(json.NewDecoder(resp.Body).Decode(&wf), check.IsNil)
191         c.Check(wf.UUID, check.Equals, arvadostest.WorkflowWithDefinitionYAMLUUID)
192         c.Check(wf.OwnerUUID, check.Equals, arvadostest.ActiveUserUUID)
193 }
194
195 func (s *FederationSuite) TestOptionsMethod(c *check.C) {
196         req := httptest.NewRequest("OPTIONS", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
197         req.Header.Set("Origin", "https://example.com")
198         resp := s.testRequest(req).Result()
199         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
200         body, err := ioutil.ReadAll(resp.Body)
201         c.Check(err, check.IsNil)
202         c.Check(string(body), check.Equals, "")
203         c.Check(resp.Header.Get("Access-Control-Allow-Origin"), check.Equals, "*")
204         for _, hdr := range []string{"Authorization", "Content-Type"} {
205                 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), check.Matches, ".*"+hdr+".*")
206         }
207         for _, method := range []string{"GET", "HEAD", "PUT", "POST", "DELETE"} {
208                 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), check.Matches, ".*"+method+".*")
209         }
210 }
211
212 func (s *FederationSuite) TestRemoteWithTokenInQuery(c *check.C) {
213         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1)+"?api_token="+arvadostest.ActiveToken, nil)
214         s.testRequest(req).Result()
215         c.Assert(s.remoteMockRequests, check.HasLen, 1)
216         pr := s.remoteMockRequests[0]
217         // Token is salted and moved from query to Authorization header.
218         c.Check(pr.URL.String(), check.Not(check.Matches), `.*api_token=.*`)
219         c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/7fd31b61f39c0e82a4155592163218272cedacdc")
220 }
221
222 func (s *FederationSuite) TestLocalTokenSalted(c *check.C) {
223         defer s.localServiceReturns404(c).Close()
224         for _, path := range []string{
225                 // During the transition to the strongly typed
226                 // controller implementation (#14287), workflows and
227                 // collections test different code paths.
228                 "/arvados/v1/workflows/" + strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1),
229                 "/arvados/v1/collections/" + strings.Replace(arvadostest.UserAgreementCollection, "zzzzz-", "zmock-", 1),
230         } {
231                 c.Log("testing path ", path)
232                 s.remoteMockRequests = nil
233                 req := httptest.NewRequest("GET", path, nil)
234                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
235                 s.testRequest(req).Result()
236                 c.Assert(s.remoteMockRequests, check.HasLen, 1)
237                 pr := s.remoteMockRequests[0]
238                 // The salted token here has a "zzzzz-" UUID instead of a
239                 // "ztest-" UUID because ztest's local database has the
240                 // "zzzzz-" test fixtures. The "secret" part is HMAC(sha1,
241                 // arvadostest.ActiveToken, "zmock") = "7fd3...".
242                 c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/7fd31b61f39c0e82a4155592163218272cedacdc")
243         }
244 }
245
246 func (s *FederationSuite) TestRemoteTokenNotSalted(c *check.C) {
247         defer s.localServiceReturns404(c).Close()
248         // remoteToken can be any v1 token that doesn't appear in
249         // ztest's local db.
250         remoteToken := "abcdef00000000000000000000000000000000000000000000"
251
252         for _, path := range []string{
253                 // During the transition to the strongly typed
254                 // controller implementation (#14287), workflows and
255                 // collections test different code paths.
256                 "/arvados/v1/workflows/" + strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1),
257                 "/arvados/v1/collections/" + strings.Replace(arvadostest.UserAgreementCollection, "zzzzz-", "zmock-", 1),
258         } {
259                 c.Log("testing path ", path)
260                 s.remoteMockRequests = nil
261                 req := httptest.NewRequest("GET", path, nil)
262                 req.Header.Set("Authorization", "Bearer "+remoteToken)
263                 s.testRequest(req).Result()
264                 c.Assert(s.remoteMockRequests, check.HasLen, 1)
265                 pr := s.remoteMockRequests[0]
266                 c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer "+remoteToken)
267         }
268 }
269
270 func (s *FederationSuite) TestWorkflowCRUD(c *check.C) {
271         var wf arvados.Workflow
272         {
273                 req := httptest.NewRequest("POST", "/arvados/v1/workflows", strings.NewReader(url.Values{
274                         "workflow": {`{"description": "TestCRUD"}`},
275                 }.Encode()))
276                 req.Header.Set("Content-type", "application/x-www-form-urlencoded")
277                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
278                 rec := httptest.NewRecorder()
279                 s.remoteServer.Server.Handler.ServeHTTP(rec, req) // direct to remote -- can't proxy a create req because no uuid
280                 resp := rec.Result()
281                 s.checkResponseOK(c, resp)
282                 json.NewDecoder(resp.Body).Decode(&wf)
283
284                 defer func() {
285                         req := httptest.NewRequest("DELETE", "/arvados/v1/workflows/"+wf.UUID, nil)
286                         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
287                         s.remoteServer.Server.Handler.ServeHTTP(httptest.NewRecorder(), req)
288                 }()
289                 c.Check(wf.UUID, check.Not(check.Equals), "")
290
291                 c.Assert(wf.ModifiedAt, check.NotNil)
292                 c.Logf("wf.ModifiedAt: %v", wf.ModifiedAt)
293                 c.Check(time.Since(*wf.ModifiedAt) < time.Minute, check.Equals, true)
294         }
295         for _, method := range []string{"PATCH", "PUT", "POST"} {
296                 form := url.Values{
297                         "workflow": {`{"description": "Updated with ` + method + `"}`},
298                 }
299                 if method == "POST" {
300                         form["_method"] = []string{"PATCH"}
301                 }
302                 req := httptest.NewRequest(method, "/arvados/v1/workflows/"+wf.UUID, strings.NewReader(form.Encode()))
303                 req.Header.Set("Content-type", "application/x-www-form-urlencoded")
304                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
305                 resp := s.testRequest(req).Result()
306                 s.checkResponseOK(c, resp)
307                 err := json.NewDecoder(resp.Body).Decode(&wf)
308                 c.Check(err, check.IsNil)
309
310                 c.Check(wf.Description, check.Equals, "Updated with "+method)
311         }
312         {
313                 req := httptest.NewRequest("DELETE", "/arvados/v1/workflows/"+wf.UUID, nil)
314                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
315                 resp := s.testRequest(req).Result()
316                 s.checkResponseOK(c, resp)
317                 err := json.NewDecoder(resp.Body).Decode(&wf)
318                 c.Check(err, check.IsNil)
319         }
320         {
321                 req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+wf.UUID, nil)
322                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
323                 resp := s.testRequest(req).Result()
324                 c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
325         }
326 }
327
328 func (s *FederationSuite) checkResponseOK(c *check.C, resp *http.Response) {
329         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
330         if resp.StatusCode != http.StatusOK {
331                 body, err := ioutil.ReadAll(resp.Body)
332                 c.Logf("... response body = %q, %v\n", body, err)
333         }
334 }
335
336 func (s *FederationSuite) checkJSONErrorMatches(c *check.C, resp *http.Response, re string) {
337         var jresp httpserver.ErrorResponse
338         err := json.NewDecoder(resp.Body).Decode(&jresp)
339         c.Check(err, check.IsNil)
340         c.Assert(jresp.Errors, check.HasLen, 1)
341         c.Check(jresp.Errors[0], check.Matches, re)
342 }
343
344 func (s *FederationSuite) localServiceHandler(c *check.C, h http.Handler) *httpserver.Server {
345         srv := &httpserver.Server{
346                 Server: http.Server{
347                         Handler: h,
348                 },
349         }
350         c.Assert(srv.Start(), check.IsNil)
351         arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "http://"+srv.Addr)
352         return srv
353 }
354
355 func (s *FederationSuite) localServiceReturns404(c *check.C) *httpserver.Server {
356         return s.localServiceHandler(c, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
357                 if req.URL.Path == "/arvados/v1/api_client_authorizations/current" {
358                         if req.Header.Get("Authorization") == "Bearer "+arvadostest.ActiveToken {
359                                 json.NewEncoder(w).Encode(arvados.APIClientAuthorization{UUID: arvadostest.ActiveTokenUUID, APIToken: arvadostest.ActiveToken, Scopes: []string{"all"}})
360                         } else {
361                                 w.WriteHeader(http.StatusUnauthorized)
362                         }
363                 } else if req.URL.Path == "/arvados/v1/users/current" {
364                         if req.Header.Get("Authorization") == "Bearer "+arvadostest.ActiveToken {
365                                 json.NewEncoder(w).Encode(arvados.User{UUID: arvadostest.ActiveUserUUID})
366                         } else {
367                                 w.WriteHeader(http.StatusUnauthorized)
368                         }
369                 } else {
370                         w.WriteHeader(404)
371                 }
372         }))
373 }
374
375 func (s *FederationSuite) TestGetLocalCollection(c *check.C) {
376         s.testHandler.Cluster.ClusterID = "zzzzz"
377         arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
378
379         // HTTP GET
380
381         req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementCollection, nil)
382         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
383         resp := s.testRequest(req).Result()
384
385         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
386         var col arvados.Collection
387         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
388         c.Check(col.UUID, check.Equals, arvadostest.UserAgreementCollection)
389         c.Check(col.ManifestText, check.Matches,
390                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+A[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
391 `)
392
393         // HTTP POST with _method=GET as a form parameter
394
395         req = httptest.NewRequest("POST", "/arvados/v1/collections/"+arvadostest.UserAgreementCollection, bytes.NewBufferString((url.Values{
396                 "_method": {"GET"},
397         }).Encode()))
398         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
399         req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
400         resp = s.testRequest(req).Result()
401
402         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
403         col = arvados.Collection{}
404         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
405         c.Check(col.UUID, check.Equals, arvadostest.UserAgreementCollection)
406         c.Check(col.ManifestText, check.Matches,
407                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+A[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
408 `)
409 }
410
411 func (s *FederationSuite) TestGetRemoteCollection(c *check.C) {
412         defer s.localServiceReturns404(c).Close()
413
414         req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementCollection, nil)
415         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
416         resp := s.testRequest(req).Result()
417         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
418         var col arvados.Collection
419         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
420         c.Check(col.UUID, check.Equals, arvadostest.UserAgreementCollection)
421         c.Check(col.ManifestText, check.Matches,
422                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+Rzzzzz-[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
423 `)
424 }
425
426 func (s *FederationSuite) TestGetRemoteCollectionError(c *check.C) {
427         defer s.localServiceReturns404(c).Close()
428
429         req := httptest.NewRequest("GET", "/arvados/v1/collections/zzzzz-4zz18-fakefakefakefak", nil)
430         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
431         resp := s.testRequest(req).Result()
432         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
433 }
434
435 func (s *FederationSuite) TestSignedLocatorPattern(c *check.C) {
436         // Confirm the regular expression identifies other groups of hints correctly
437         c.Check(keepclient.SignedLocatorRe.FindStringSubmatch(`6a4ff0499484c6c79c95cd8c566bd25f+249025+B1+C2+A05227438989d04712ea9ca1c91b556cef01d5cc7@5ba5405b+D3+E4`),
438                 check.DeepEquals,
439                 []string{"6a4ff0499484c6c79c95cd8c566bd25f+249025+B1+C2+A05227438989d04712ea9ca1c91b556cef01d5cc7@5ba5405b+D3+E4",
440                         "6a4ff0499484c6c79c95cd8c566bd25f",
441                         "+249025",
442                         "+B1+C2", "+C2",
443                         "+A05227438989d04712ea9ca1c91b556cef01d5cc7@5ba5405b",
444                         "05227438989d04712ea9ca1c91b556cef01d5cc7", "5ba5405b",
445                         "+D3+E4", "+E4"})
446 }
447
448 func (s *FederationSuite) TestGetLocalCollectionByPDH(c *check.C) {
449         arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
450
451         req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
452         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
453         resp := s.testRequest(req).Result()
454
455         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
456         var col arvados.Collection
457         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
458         c.Check(col.PortableDataHash, check.Equals, arvadostest.UserAgreementPDH)
459         c.Check(col.ManifestText, check.Matches,
460                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+A[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
461 `)
462 }
463
464 func (s *FederationSuite) TestGetRemoteCollectionByPDH(c *check.C) {
465         defer s.localServiceReturns404(c).Close()
466
467         req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
468         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
469         resp := s.testRequest(req).Result()
470
471         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
472
473         var col arvados.Collection
474         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
475         c.Check(col.PortableDataHash, check.Equals, arvadostest.UserAgreementPDH)
476         c.Check(col.ManifestText, check.Matches,
477                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+Rzzzzz-[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
478 `)
479 }
480
481 func (s *FederationSuite) TestGetCollectionByPDHError(c *check.C) {
482         defer s.localServiceReturns404(c).Close()
483
484         // zmock's normal response (200 with an empty body) would
485         // change the outcome from 404 to 502
486         delete(s.testHandler.Cluster.RemoteClusters, "zmock")
487
488         req := httptest.NewRequest("GET", "/arvados/v1/collections/99999999999999999999999999999999+99", nil)
489         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
490
491         resp := s.testRequest(req).Result()
492         defer resp.Body.Close()
493
494         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
495 }
496
497 func (s *FederationSuite) TestGetCollectionByPDHErrorBadHash(c *check.C) {
498         defer s.localServiceReturns404(c).Close()
499
500         // zmock's normal response (200 with an empty body) would
501         // change the outcome
502         delete(s.testHandler.Cluster.RemoteClusters, "zmock")
503
504         srv2 := &httpserver.Server{
505                 Server: http.Server{
506                         Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
507                                 w.WriteHeader(200)
508                                 // Return a collection where the hash
509                                 // of the manifest text doesn't match
510                                 // PDH that was requested.
511                                 var col arvados.Collection
512                                 col.PortableDataHash = "99999999999999999999999999999999+99"
513                                 col.ManifestText = `. 6a4ff0499484c6c79c95cd8c566bd25f\+249025 0:249025:GNU_General_Public_License,_version_3.pdf
514 `
515                                 enc := json.NewEncoder(w)
516                                 enc.Encode(col)
517                         }),
518                 },
519         }
520
521         c.Assert(srv2.Start(), check.IsNil)
522         defer srv2.Close()
523
524         // Direct zzzzz to service that returns a 200 result with a bogus manifest_text
525         s.testHandler.Cluster.RemoteClusters["zzzzz"] = arvados.RemoteCluster{
526                 Host:   srv2.Addr,
527                 Proxy:  true,
528                 Scheme: "http",
529         }
530
531         req := httptest.NewRequest("GET", "/arvados/v1/collections/99999999999999999999999999999999+99", nil)
532         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
533
534         resp := s.testRequest(req).Result()
535         defer resp.Body.Close()
536
537         c.Check(resp.StatusCode, check.Equals, http.StatusBadGateway)
538 }
539
540 func (s *FederationSuite) TestSaltedTokenGetCollectionByPDH(c *check.C) {
541         arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
542
543         req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
544         req.Header.Set("Authorization", "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/282d7d172b6cfdce364c5ed12ddf7417b2d00065")
545         resp := s.testRequest(req).Result()
546
547         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
548         var col arvados.Collection
549         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
550         c.Check(col.PortableDataHash, check.Equals, arvadostest.UserAgreementPDH)
551         c.Check(col.ManifestText, check.Matches,
552                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+A[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
553 `)
554 }
555
556 func (s *FederationSuite) TestSaltedTokenGetCollectionByPDHError(c *check.C) {
557         arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
558
559         // zmock's normal response (200 with an empty body) would
560         // change the outcome
561         delete(s.testHandler.Cluster.RemoteClusters, "zmock")
562
563         req := httptest.NewRequest("GET", "/arvados/v1/collections/99999999999999999999999999999999+99", nil)
564         req.Header.Set("Authorization", "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/282d7d172b6cfdce364c5ed12ddf7417b2d00065")
565         resp := s.testRequest(req).Result()
566
567         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
568 }
569
570 func (s *FederationSuite) TestGetRemoteContainerRequest(c *check.C) {
571         defer s.localServiceReturns404(c).Close()
572         req := httptest.NewRequest("GET", "/arvados/v1/container_requests/"+arvadostest.QueuedContainerRequestUUID, nil)
573         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
574         resp := s.testRequest(req).Result()
575         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
576         var cr arvados.ContainerRequest
577         c.Check(json.NewDecoder(resp.Body).Decode(&cr), check.IsNil)
578         c.Check(cr.UUID, check.Equals, arvadostest.QueuedContainerRequestUUID)
579         c.Check(cr.Priority, check.Equals, 1)
580 }
581
582 func (s *FederationSuite) TestUpdateRemoteContainerRequest(c *check.C) {
583         defer s.localServiceReturns404(c).Close()
584         setPri := func(pri int) {
585                 req := httptest.NewRequest("PATCH", "/arvados/v1/container_requests/"+arvadostest.QueuedContainerRequestUUID,
586                         strings.NewReader(fmt.Sprintf(`{"container_request": {"priority": %d}}`, pri)))
587                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
588                 req.Header.Set("Content-type", "application/json")
589                 resp := s.testRequest(req).Result()
590                 c.Check(resp.StatusCode, check.Equals, http.StatusOK)
591                 var cr arvados.ContainerRequest
592                 c.Check(json.NewDecoder(resp.Body).Decode(&cr), check.IsNil)
593                 c.Check(cr.UUID, check.Equals, arvadostest.QueuedContainerRequestUUID)
594                 c.Check(cr.Priority, check.Equals, pri)
595         }
596         setPri(696)
597         setPri(1) // Reset fixture so side effect doesn't break other tests.
598 }
599
600 func (s *FederationSuite) TestCreateContainerRequestBadToken(c *check.C) {
601         defer s.localServiceReturns404(c).Close()
602         // pass cluster_id via query parameter, this allows arvados-controller
603         // to avoid parsing the body
604         req := httptest.NewRequest("POST", "/arvados/v1/container_requests?cluster_id=zzzzz",
605                 strings.NewReader(`{"container_request":{}}`))
606         req.Header.Set("Authorization", "Bearer abcdefg")
607         req.Header.Set("Content-type", "application/json")
608         resp := s.testRequest(req).Result()
609         c.Check(resp.StatusCode, check.Equals, http.StatusForbidden)
610         var e map[string][]string
611         c.Check(json.NewDecoder(resp.Body).Decode(&e), check.IsNil)
612         c.Check(e["errors"], check.DeepEquals, []string{"invalid API token"})
613 }
614
615 func (s *FederationSuite) TestCreateRemoteContainerRequest(c *check.C) {
616         defer s.localServiceReturns404(c).Close()
617         // pass cluster_id via query parameter, this allows arvados-controller
618         // to avoid parsing the body
619         req := httptest.NewRequest("POST", "/arvados/v1/container_requests?cluster_id=zzzzz",
620                 strings.NewReader(`{
621   "container_request": {
622     "name": "hello world",
623     "state": "Uncommitted",
624     "output_path": "/",
625     "container_image": "123",
626     "command": ["abc"]
627   }
628 }
629 `))
630         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
631         req.Header.Set("Content-type", "application/json")
632         resp := s.testRequest(req).Result()
633         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
634         var cr arvados.ContainerRequest
635         c.Check(json.NewDecoder(resp.Body).Decode(&cr), check.IsNil)
636         c.Check(cr.Name, check.Equals, "hello world")
637         c.Check(strings.HasPrefix(cr.UUID, "zzzzz-"), check.Equals, true)
638 }
639
640 // getCRfromMockRequest returns a ContainerRequest with the content of the
641 // request sent to the remote mock. This function takes into account the
642 // Content-Type and acts accordingly.
643 func (s *FederationSuite) getCRfromMockRequest(c *check.C) arvados.ContainerRequest {
644
645         // Body can be a json formated or something like:
646         //  cluster_id=zmock&container_request=%7B%22command%22%3A%5B%22abc%22%5D%2C%22container_image%22%3A%22123%22%2C%22...7D
647         // or:
648         //  "{\"container_request\":{\"command\":[\"abc\"],\"container_image\":\"12...Uncommitted\"}}"
649
650         var cr arvados.ContainerRequest
651         data, err := ioutil.ReadAll(s.remoteMockRequests[0].Body)
652         c.Check(err, check.IsNil)
653
654         if s.remoteMockRequests[0].Header.Get("Content-Type") == "application/json" {
655                 // legacy code path sends a JSON request body
656                 var answerCR struct {
657                         ContainerRequest arvados.ContainerRequest `json:"container_request"`
658                 }
659                 c.Check(json.Unmarshal(data, &answerCR), check.IsNil)
660                 cr = answerCR.ContainerRequest
661         } else if s.remoteMockRequests[0].Header.Get("Content-Type") == "application/x-www-form-urlencoded" {
662                 // new code path sends a form-encoded request body with a JSON-encoded parameter value
663                 decodedValue, err := url.ParseQuery(string(data))
664                 c.Check(err, check.IsNil)
665                 decodedValueCR := decodedValue.Get("container_request")
666                 c.Check(json.Unmarshal([]byte(decodedValueCR), &cr), check.IsNil)
667         } else {
668                 // mock needs to have Content-Type that we can parse.
669                 c.Fail()
670         }
671
672         return cr
673 }
674
675 func (s *FederationSuite) TestCreateRemoteContainerRequestCheckRuntimeToken(c *check.C) {
676         // Send request to zmock and check that outgoing request has
677         // runtime_token set with a new random v2 token.
678
679         defer s.localServiceReturns404(c).Close()
680         req := httptest.NewRequest("POST", "/arvados/v1/container_requests?cluster_id=zmock",
681                 strings.NewReader(`{
682           "container_request": {
683             "name": "hello world",
684             "state": "Uncommitted",
685             "output_path": "/",
686             "container_image": "123",
687             "command": ["abc"]
688           }
689         }
690         `))
691         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
692         req.Header.Set("Content-type", "application/json")
693
694         // We replace zhome with zzzzz values (RailsAPI, ClusterID, SystemRootToken)
695         // SystemRoot token is needed because we check the
696         // https://[RailsAPI]/arvados/v1/api_client_authorizations/current
697         // https://[RailsAPI]/arvados/v1/users/current and
698         // https://[RailsAPI]/auth/controller/callback
699         arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
700         s.testHandler.Cluster.ClusterID = "zzzzz"
701         s.testHandler.Cluster.SystemRootToken = arvadostest.SystemRootToken
702         s.testHandler.Cluster.API.MaxTokenLifetime = arvados.Duration(time.Hour)
703
704         resp := s.testRequest(req).Result()
705         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
706
707         cr := s.getCRfromMockRequest(c)
708
709         // Runtime token must match zzzzz cluster
710         c.Check(cr.RuntimeToken, check.Matches, "v2/zzzzz-gj3su-.*")
711
712         // RuntimeToken must be different than the Original Token we originally did the request with.
713         c.Check(cr.RuntimeToken, check.Not(check.Equals), arvadostest.ActiveTokenV2)
714
715         // Runtime token should not have an expiration based on API.MaxTokenLifetime
716         req2 := httptest.NewRequest("GET", "/arvados/v1/api_client_authorizations/current", nil)
717         req2.Header.Set("Authorization", "Bearer "+cr.RuntimeToken)
718         req2.Header.Set("Content-type", "application/json")
719         resp = s.testRequest(req2).Result()
720         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
721         var aca arvados.APIClientAuthorization
722         c.Check(json.NewDecoder(resp.Body).Decode(&aca), check.IsNil)
723         c.Check(aca.ExpiresAt, check.NotNil) // Time.Now()+BlobSigningTTL
724         t, _ := time.Parse(time.RFC3339Nano, aca.ExpiresAt)
725         c.Check(t.After(time.Now().Add(s.testHandler.Cluster.API.MaxTokenLifetime.Duration())), check.Equals, true)
726         c.Check(t.Before(time.Now().Add(s.testHandler.Cluster.Collections.BlobSigningTTL.Duration())), check.Equals, true)
727 }
728
729 func (s *FederationSuite) TestCreateRemoteContainerRequestCheckSetRuntimeToken(c *check.C) {
730         // Send request to zmock and check that outgoing request has
731         // runtime_token set with the explicitly provided token.
732
733         defer s.localServiceReturns404(c).Close()
734         // pass cluster_id via query parameter, this allows arvados-controller
735         // to avoid parsing the body
736         req := httptest.NewRequest("POST", "/arvados/v1/container_requests?cluster_id=zmock",
737                 strings.NewReader(`{
738           "container_request": {
739             "name": "hello world",
740             "state": "Uncommitted",
741             "output_path": "/",
742             "container_image": "123",
743             "command": ["abc"],
744             "runtime_token": "xyz"
745           }
746         }
747         `))
748         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
749         req.Header.Set("Content-type", "application/json")
750         resp := s.testRequest(req).Result()
751         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
752
753         cr := s.getCRfromMockRequest(c)
754
755         // After mocking around now making sure the runtime_token we sent is still there.
756         c.Check(cr.RuntimeToken, check.Equals, "xyz")
757 }
758
759 func (s *FederationSuite) TestCreateRemoteContainerRequestError(c *check.C) {
760         defer s.localServiceReturns404(c).Close()
761         // pass cluster_id via query parameter, this allows arvados-controller
762         // to avoid parsing the body
763         req := httptest.NewRequest("POST", "/arvados/v1/container_requests?cluster_id=zz404",
764                 strings.NewReader(`{
765   "container_request": {
766     "name": "hello world",
767     "state": "Uncommitted",
768     "output_path": "/",
769     "container_image": "123",
770     "command": ["abc"]
771   }
772 }
773 `))
774         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
775         req.Header.Set("Content-type", "application/json")
776         resp := s.testRequest(req).Result()
777         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
778 }
779
780 func (s *FederationSuite) TestGetRemoteContainer(c *check.C) {
781         defer s.localServiceReturns404(c).Close()
782         req := httptest.NewRequest("GET", "/arvados/v1/containers/"+arvadostest.QueuedContainerUUID, nil)
783         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
784         resp := s.testRequest(req)
785         c.Check(resp.Code, check.Equals, http.StatusOK)
786         var cn arvados.Container
787         c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
788         c.Check(cn.UUID, check.Equals, arvadostest.QueuedContainerUUID)
789 }
790
791 func (s *FederationSuite) TestListRemoteContainer(c *check.C) {
792         defer s.localServiceReturns404(c).Close()
793         req := httptest.NewRequest("GET", "/arvados/v1/containers?count=none&filters="+
794                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v"]]]`, arvadostest.QueuedContainerUUID)), nil)
795         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
796         resp := s.testRequest(req).Result()
797         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
798         var cn arvados.ContainerList
799         c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
800         c.Assert(cn.Items, check.HasLen, 1)
801         c.Check(cn.Items[0].UUID, check.Equals, arvadostest.QueuedContainerUUID)
802 }
803
804 func (s *FederationSuite) TestListMultiRemoteContainers(c *check.C) {
805         defer s.localServiceHandler(c, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
806                 bd, _ := ioutil.ReadAll(req.Body)
807                 c.Check(string(bd), check.Equals, `_method=GET&count=none&filters=%5B%5B%22uuid%22%2C+%22in%22%2C+%5B%22zhome-xvhdp-cr5queuedcontnr%22%5D%5D%5D&select=%5B%22uuid%22%2C+%22command%22%5D`)
808                 w.WriteHeader(200)
809                 w.Write([]byte(`{"kind": "arvados#containerList", "items": [{"uuid": "zhome-xvhdp-cr5queuedcontnr", "command": ["abc"]}]}`))
810         })).Close()
811         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&select=%s",
812                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
813                         arvadostest.QueuedContainerUUID)),
814                 url.QueryEscape(`["uuid", "command"]`)),
815                 nil)
816         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
817         resp := s.testRequest(req).Result()
818         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
819         var cn arvados.ContainerList
820         c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
821         c.Check(cn.Items, check.HasLen, 2)
822         mp := make(map[string]arvados.Container)
823         for _, cr := range cn.Items {
824                 mp[cr.UUID] = cr
825         }
826         c.Check(mp[arvadostest.QueuedContainerUUID].Command, check.DeepEquals, []string{"echo", "hello"})
827         c.Check(mp[arvadostest.QueuedContainerUUID].ContainerImage, check.Equals, "")
828         c.Check(mp["zhome-xvhdp-cr5queuedcontnr"].Command, check.DeepEquals, []string{"abc"})
829         c.Check(mp["zhome-xvhdp-cr5queuedcontnr"].ContainerImage, check.Equals, "")
830 }
831
832 func (s *FederationSuite) TestListMultiRemoteContainerError(c *check.C) {
833         defer s.localServiceReturns404(c).Close()
834         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&select=%s",
835                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
836                         arvadostest.QueuedContainerUUID)),
837                 url.QueryEscape(`["uuid", "command"]`)),
838                 nil)
839         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
840         resp := s.testRequest(req).Result()
841         c.Check(resp.StatusCode, check.Equals, http.StatusBadGateway)
842         s.checkJSONErrorMatches(c, resp, `error fetching from zhome \(404 Not Found\): EOF`)
843 }
844
845 func (s *FederationSuite) TestListMultiRemoteContainersPaged(c *check.C) {
846
847         callCount := 0
848         defer s.localServiceHandler(c, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
849                 bd, _ := ioutil.ReadAll(req.Body)
850                 if callCount == 0 {
851                         c.Check(string(bd), check.Equals, `_method=GET&count=none&filters=%5B%5B%22uuid%22%2C+%22in%22%2C+%5B%22zhome-xvhdp-cr5queuedcontnr%22%2C%22zhome-xvhdp-cr6queuedcontnr%22%5D%5D%5D`)
852                         w.WriteHeader(200)
853                         w.Write([]byte(`{"kind": "arvados#containerList", "items": [{"uuid": "zhome-xvhdp-cr5queuedcontnr", "command": ["abc"]}]}`))
854                 } else if callCount == 1 {
855                         c.Check(string(bd), check.Equals, `_method=GET&count=none&filters=%5B%5B%22uuid%22%2C+%22in%22%2C+%5B%22zhome-xvhdp-cr6queuedcontnr%22%5D%5D%5D`)
856                         w.WriteHeader(200)
857                         w.Write([]byte(`{"kind": "arvados#containerList", "items": [{"uuid": "zhome-xvhdp-cr6queuedcontnr", "command": ["efg"]}]}`))
858                 }
859                 callCount++
860         })).Close()
861         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s",
862                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr", "zhome-xvhdp-cr6queuedcontnr"]]]`,
863                         arvadostest.QueuedContainerUUID))),
864                 nil)
865         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
866         resp := s.testRequest(req).Result()
867         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
868         c.Check(callCount, check.Equals, 2)
869         var cn arvados.ContainerList
870         c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
871         c.Check(cn.Items, check.HasLen, 3)
872         mp := make(map[string]arvados.Container)
873         for _, cr := range cn.Items {
874                 mp[cr.UUID] = cr
875         }
876         c.Check(mp[arvadostest.QueuedContainerUUID].Command, check.DeepEquals, []string{"echo", "hello"})
877         c.Check(mp["zhome-xvhdp-cr5queuedcontnr"].Command, check.DeepEquals, []string{"abc"})
878         c.Check(mp["zhome-xvhdp-cr6queuedcontnr"].Command, check.DeepEquals, []string{"efg"})
879 }
880
881 func (s *FederationSuite) TestListMultiRemoteContainersMissing(c *check.C) {
882
883         callCount := 0
884         defer s.localServiceHandler(c, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
885                 bd, _ := ioutil.ReadAll(req.Body)
886                 if callCount == 0 {
887                         c.Check(string(bd), check.Equals, `_method=GET&count=none&filters=%5B%5B%22uuid%22%2C+%22in%22%2C+%5B%22zhome-xvhdp-cr5queuedcontnr%22%2C%22zhome-xvhdp-cr6queuedcontnr%22%5D%5D%5D`)
888                         w.WriteHeader(200)
889                         w.Write([]byte(`{"kind": "arvados#containerList", "items": [{"uuid": "zhome-xvhdp-cr6queuedcontnr", "command": ["efg"]}]}`))
890                 } else if callCount == 1 {
891                         c.Check(string(bd), check.Equals, `_method=GET&count=none&filters=%5B%5B%22uuid%22%2C+%22in%22%2C+%5B%22zhome-xvhdp-cr5queuedcontnr%22%5D%5D%5D`)
892                         w.WriteHeader(200)
893                         w.Write([]byte(`{"kind": "arvados#containerList", "items": []}`))
894                 }
895                 callCount++
896         })).Close()
897         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s",
898                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr", "zhome-xvhdp-cr6queuedcontnr"]]]`,
899                         arvadostest.QueuedContainerUUID))),
900                 nil)
901         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
902         resp := s.testRequest(req).Result()
903         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
904         c.Check(callCount, check.Equals, 2)
905         var cn arvados.ContainerList
906         c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
907         c.Check(cn.Items, check.HasLen, 2)
908         mp := make(map[string]arvados.Container)
909         for _, cr := range cn.Items {
910                 mp[cr.UUID] = cr
911         }
912         c.Check(mp[arvadostest.QueuedContainerUUID].Command, check.DeepEquals, []string{"echo", "hello"})
913         c.Check(mp["zhome-xvhdp-cr6queuedcontnr"].Command, check.DeepEquals, []string{"efg"})
914 }
915
916 func (s *FederationSuite) TestListMultiRemoteContainerPageSizeError(c *check.C) {
917         s.testHandler.Cluster.API.MaxItemsPerResponse = 1
918         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s",
919                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
920                         arvadostest.QueuedContainerUUID))),
921                 nil)
922         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
923         resp := s.testRequest(req).Result()
924         c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
925         s.checkJSONErrorMatches(c, resp, `Federated multi-object request for 2 objects which is more than max page size 1.`)
926 }
927
928 func (s *FederationSuite) TestListMultiRemoteContainerLimitError(c *check.C) {
929         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&limit=1",
930                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
931                         arvadostest.QueuedContainerUUID))),
932                 nil)
933         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
934         resp := s.testRequest(req).Result()
935         c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
936         s.checkJSONErrorMatches(c, resp, `Federated multi-object may not provide 'limit', 'offset' or 'order'.`)
937 }
938
939 func (s *FederationSuite) TestListMultiRemoteContainerOffsetError(c *check.C) {
940         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&offset=1",
941                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
942                         arvadostest.QueuedContainerUUID))),
943                 nil)
944         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
945         resp := s.testRequest(req).Result()
946         c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
947         s.checkJSONErrorMatches(c, resp, `Federated multi-object may not provide 'limit', 'offset' or 'order'.`)
948 }
949
950 func (s *FederationSuite) TestListMultiRemoteContainerOrderError(c *check.C) {
951         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&order=uuid",
952                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
953                         arvadostest.QueuedContainerUUID))),
954                 nil)
955         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
956         resp := s.testRequest(req).Result()
957         c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
958         s.checkJSONErrorMatches(c, resp, `Federated multi-object may not provide 'limit', 'offset' or 'order'.`)
959 }
960
961 func (s *FederationSuite) TestListMultiRemoteContainerSelectError(c *check.C) {
962         req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s&select=%s",
963                 url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr"]]]`,
964                         arvadostest.QueuedContainerUUID)),
965                 url.QueryEscape(`["command"]`)),
966                 nil)
967         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
968         resp := s.testRequest(req).Result()
969         c.Check(resp.StatusCode, check.Equals, http.StatusBadRequest)
970         s.checkJSONErrorMatches(c, resp, `Federated multi-object request must include 'uuid' in 'select'`)
971 }