14087: Rate limit the number of parallel requests
[arvados.git] / lib / controller / federation_test.go
1 // Copyright (C) The Arvados Authors. All rights reserved.
2 //
3 // SPDX-License-Identifier: AGPL-3.0
4
5 package controller
6
7 import (
8         "encoding/json"
9         "io/ioutil"
10         "net/http"
11         "net/http/httptest"
12         "net/url"
13         "os"
14         "strings"
15         "time"
16
17         "git.curoverse.com/arvados.git/sdk/go/arvados"
18         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
19         "git.curoverse.com/arvados.git/sdk/go/httpserver"
20         "git.curoverse.com/arvados.git/sdk/go/keepclient"
21         "github.com/Sirupsen/logrus"
22         check "gopkg.in/check.v1"
23 )
24
25 // Gocheck boilerplate
26 var _ = check.Suite(&FederationSuite{})
27
28 type FederationSuite struct {
29         log *logrus.Logger
30         // testServer and testHandler are the controller being tested,
31         // "zhome".
32         testServer  *httpserver.Server
33         testHandler *Handler
34         // remoteServer ("zzzzz") forwards requests to the Rails API
35         // provided by the integration test environment.
36         remoteServer *httpserver.Server
37         // remoteMock ("zmock") appends each incoming request to
38         // remoteMockRequests, and returns an empty 200 response.
39         remoteMock         *httpserver.Server
40         remoteMockRequests []http.Request
41 }
42
43 func (s *FederationSuite) SetUpTest(c *check.C) {
44         s.log = logrus.New()
45         s.log.Formatter = &logrus.JSONFormatter{}
46         s.log.Out = &logWriter{c.Log}
47
48         s.remoteServer = newServerFromIntegrationTestEnv(c)
49         c.Assert(s.remoteServer.Start(), check.IsNil)
50
51         s.remoteMock = newServerFromIntegrationTestEnv(c)
52         s.remoteMock.Server.Handler = http.HandlerFunc(s.remoteMockHandler)
53         c.Assert(s.remoteMock.Start(), check.IsNil)
54
55         nodeProfile := arvados.NodeProfile{
56                 Controller: arvados.SystemServiceInstance{Listen: ":"},
57                 RailsAPI:   arvados.SystemServiceInstance{Listen: ":1"}, // local reqs will error "connection refused"
58         }
59         s.testHandler = &Handler{Cluster: &arvados.Cluster{
60                 ClusterID:  "zhome",
61                 PostgreSQL: integrationTestCluster().PostgreSQL,
62                 NodeProfiles: map[string]arvados.NodeProfile{
63                         "*": nodeProfile,
64                 },
65         }, NodeProfile: &nodeProfile}
66         s.testServer = newServerFromIntegrationTestEnv(c)
67         s.testServer.Server.Handler = httpserver.AddRequestIDs(httpserver.LogRequests(s.log, s.testHandler))
68
69         s.testHandler.Cluster.RemoteClusters = map[string]arvados.RemoteCluster{
70                 "zzzzz": {
71                         Host:   s.remoteServer.Addr,
72                         Proxy:  true,
73                         Scheme: "http",
74                 },
75                 "zmock": {
76                         Host:   s.remoteMock.Addr,
77                         Proxy:  true,
78                         Scheme: "http",
79                 },
80         }
81
82         c.Assert(s.testServer.Start(), check.IsNil)
83
84         s.remoteMockRequests = nil
85 }
86
87 func (s *FederationSuite) remoteMockHandler(w http.ResponseWriter, req *http.Request) {
88         s.remoteMockRequests = append(s.remoteMockRequests, *req)
89 }
90
91 func (s *FederationSuite) TearDownTest(c *check.C) {
92         if s.remoteServer != nil {
93                 s.remoteServer.Close()
94         }
95         if s.testServer != nil {
96                 s.testServer.Close()
97         }
98 }
99
100 func (s *FederationSuite) testRequest(req *http.Request) *http.Response {
101         resp := httptest.NewRecorder()
102         s.testServer.Server.Handler.ServeHTTP(resp, req)
103         return resp.Result()
104 }
105
106 func (s *FederationSuite) TestLocalRequest(c *check.C) {
107         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zhome-", 1), nil)
108         resp := s.testRequest(req)
109         s.checkHandledLocally(c, resp)
110 }
111
112 func (s *FederationSuite) checkHandledLocally(c *check.C, resp *http.Response) {
113         // Our "home" controller can't handle local requests because
114         // it doesn't have its own stub/test Rails API, so we rely on
115         // "connection refused" to indicate the controller tried to
116         // proxy the request to its local Rails API.
117         c.Check(resp.StatusCode, check.Equals, http.StatusBadGateway)
118         s.checkJSONErrorMatches(c, resp, `.*connection refused`)
119 }
120
121 func (s *FederationSuite) TestNoAuth(c *check.C) {
122         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
123         resp := s.testRequest(req)
124         c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
125         s.checkJSONErrorMatches(c, resp, `Not logged in`)
126 }
127
128 func (s *FederationSuite) TestBadAuth(c *check.C) {
129         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
130         req.Header.Set("Authorization", "Bearer aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
131         resp := s.testRequest(req)
132         c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
133         s.checkJSONErrorMatches(c, resp, `Not logged in`)
134 }
135
136 func (s *FederationSuite) TestNoAccess(c *check.C) {
137         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
138         req.Header.Set("Authorization", "Bearer "+arvadostest.SpectatorToken)
139         resp := s.testRequest(req)
140         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
141         s.checkJSONErrorMatches(c, resp, `.*not found`)
142 }
143
144 func (s *FederationSuite) TestGetUnknownRemote(c *check.C) {
145         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zz404-", 1), nil)
146         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
147         resp := s.testRequest(req)
148         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
149         s.checkJSONErrorMatches(c, resp, `.*no proxy available for cluster zz404`)
150 }
151
152 func (s *FederationSuite) TestRemoteError(c *check.C) {
153         rc := s.testHandler.Cluster.RemoteClusters["zzzzz"]
154         rc.Scheme = "https"
155         s.testHandler.Cluster.RemoteClusters["zzzzz"] = rc
156
157         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
158         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
159         resp := s.testRequest(req)
160         c.Check(resp.StatusCode, check.Equals, http.StatusBadGateway)
161         s.checkJSONErrorMatches(c, resp, `.*HTTP response to HTTPS client`)
162 }
163
164 func (s *FederationSuite) TestGetRemoteWorkflow(c *check.C) {
165         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
166         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
167         resp := s.testRequest(req)
168         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
169         var wf arvados.Workflow
170         c.Check(json.NewDecoder(resp.Body).Decode(&wf), check.IsNil)
171         c.Check(wf.UUID, check.Equals, arvadostest.WorkflowWithDefinitionYAMLUUID)
172         c.Check(wf.OwnerUUID, check.Equals, arvadostest.ActiveUserUUID)
173 }
174
175 func (s *FederationSuite) TestOptionsMethod(c *check.C) {
176         req := httptest.NewRequest("OPTIONS", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
177         req.Header.Set("Origin", "https://example.com")
178         resp := s.testRequest(req)
179         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
180         body, err := ioutil.ReadAll(resp.Body)
181         c.Check(err, check.IsNil)
182         c.Check(string(body), check.Equals, "")
183         c.Check(resp.Header.Get("Access-Control-Allow-Origin"), check.Equals, "*")
184         for _, hdr := range []string{"Authorization", "Content-Type"} {
185                 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), check.Matches, ".*"+hdr+".*")
186         }
187         for _, method := range []string{"GET", "HEAD", "PUT", "POST", "DELETE"} {
188                 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), check.Matches, ".*"+method+".*")
189         }
190 }
191
192 func (s *FederationSuite) TestRemoteWithTokenInQuery(c *check.C) {
193         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1)+"?api_token="+arvadostest.ActiveToken, nil)
194         s.testRequest(req)
195         c.Assert(len(s.remoteMockRequests), check.Equals, 1)
196         pr := s.remoteMockRequests[0]
197         // Token is salted and moved from query to Authorization header.
198         c.Check(pr.URL.String(), check.Not(check.Matches), `.*api_token=.*`)
199         c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/7fd31b61f39c0e82a4155592163218272cedacdc")
200 }
201
202 func (s *FederationSuite) TestLocalTokenSalted(c *check.C) {
203         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1), nil)
204         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
205         s.testRequest(req)
206         c.Assert(len(s.remoteMockRequests), check.Equals, 1)
207         pr := s.remoteMockRequests[0]
208         // The salted token here has a "zzzzz-" UUID instead of a
209         // "ztest-" UUID because ztest's local database has the
210         // "zzzzz-" test fixtures. The "secret" part is HMAC(sha1,
211         // arvadostest.ActiveToken, "zmock") = "7fd3...".
212         c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/7fd31b61f39c0e82a4155592163218272cedacdc")
213 }
214
215 func (s *FederationSuite) TestRemoteTokenNotSalted(c *check.C) {
216         // remoteToken can be any v1 token that doesn't appear in
217         // ztest's local db.
218         remoteToken := "abcdef00000000000000000000000000000000000000000000"
219         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1), nil)
220         req.Header.Set("Authorization", "Bearer "+remoteToken)
221         s.testRequest(req)
222         c.Assert(len(s.remoteMockRequests), check.Equals, 1)
223         pr := s.remoteMockRequests[0]
224         c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer "+remoteToken)
225 }
226
227 func (s *FederationSuite) TestWorkflowCRUD(c *check.C) {
228         wf := arvados.Workflow{
229                 Description: "TestCRUD",
230         }
231         {
232                 body := &strings.Builder{}
233                 json.NewEncoder(body).Encode(&wf)
234                 req := httptest.NewRequest("POST", "/arvados/v1/workflows", strings.NewReader(url.Values{
235                         "workflow": {body.String()},
236                 }.Encode()))
237                 req.Header.Set("Content-type", "application/x-www-form-urlencoded")
238                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
239                 rec := httptest.NewRecorder()
240                 s.remoteServer.Server.Handler.ServeHTTP(rec, req) // direct to remote -- can't proxy a create req because no uuid
241                 resp := rec.Result()
242                 s.checkResponseOK(c, resp)
243                 json.NewDecoder(resp.Body).Decode(&wf)
244
245                 defer func() {
246                         req := httptest.NewRequest("DELETE", "/arvados/v1/workflows/"+wf.UUID, nil)
247                         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
248                         s.remoteServer.Server.Handler.ServeHTTP(httptest.NewRecorder(), req)
249                 }()
250                 c.Check(wf.UUID, check.Not(check.Equals), "")
251
252                 c.Assert(wf.ModifiedAt, check.NotNil)
253                 c.Logf("wf.ModifiedAt: %v", wf.ModifiedAt)
254                 c.Check(time.Since(*wf.ModifiedAt) < time.Minute, check.Equals, true)
255         }
256         for _, method := range []string{"PATCH", "PUT", "POST"} {
257                 form := url.Values{
258                         "workflow": {`{"description": "Updated with ` + method + `"}`},
259                 }
260                 if method == "POST" {
261                         form["_method"] = []string{"PATCH"}
262                 }
263                 req := httptest.NewRequest(method, "/arvados/v1/workflows/"+wf.UUID, strings.NewReader(form.Encode()))
264                 req.Header.Set("Content-type", "application/x-www-form-urlencoded")
265                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
266                 resp := s.testRequest(req)
267                 s.checkResponseOK(c, resp)
268                 err := json.NewDecoder(resp.Body).Decode(&wf)
269                 c.Check(err, check.IsNil)
270
271                 c.Check(wf.Description, check.Equals, "Updated with "+method)
272         }
273         {
274                 req := httptest.NewRequest("DELETE", "/arvados/v1/workflows/"+wf.UUID, nil)
275                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
276                 resp := s.testRequest(req)
277                 s.checkResponseOK(c, resp)
278                 err := json.NewDecoder(resp.Body).Decode(&wf)
279                 c.Check(err, check.IsNil)
280         }
281         {
282                 req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+wf.UUID, nil)
283                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
284                 resp := s.testRequest(req)
285                 c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
286         }
287 }
288
289 func (s *FederationSuite) checkResponseOK(c *check.C, resp *http.Response) {
290         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
291         if resp.StatusCode != http.StatusOK {
292                 body, err := ioutil.ReadAll(resp.Body)
293                 c.Logf("... response body = %q, %v\n", body, err)
294         }
295 }
296
297 func (s *FederationSuite) checkJSONErrorMatches(c *check.C, resp *http.Response, re string) {
298         var jresp httpserver.ErrorResponse
299         err := json.NewDecoder(resp.Body).Decode(&jresp)
300         c.Check(err, check.IsNil)
301         c.Assert(len(jresp.Errors), check.Equals, 1)
302         c.Check(jresp.Errors[0], check.Matches, re)
303 }
304
305 func (s *FederationSuite) TestGetLocalCollection(c *check.C) {
306         np := arvados.NodeProfile{
307                 Controller: arvados.SystemServiceInstance{Listen: ":"},
308                 RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"),
309                         TLS: true, Insecure: true}}
310         s.testHandler.Cluster.ClusterID = "zzzzz"
311         s.testHandler.Cluster.NodeProfiles["*"] = np
312         s.testHandler.NodeProfile = &np
313
314         req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementCollection, nil)
315         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
316         resp := s.testRequest(req)
317
318         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
319         var col arvados.Collection
320         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
321         c.Check(col.UUID, check.Equals, arvadostest.UserAgreementCollection)
322         c.Check(col.ManifestText, check.Matches,
323                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+A[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
324 `)
325 }
326
327 func (s *FederationSuite) TestGetRemoteCollection(c *check.C) {
328         req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementCollection, nil)
329         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
330         resp := s.testRequest(req)
331         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
332         var col arvados.Collection
333         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
334         c.Check(col.UUID, check.Equals, arvadostest.UserAgreementCollection)
335         c.Check(col.ManifestText, check.Matches,
336                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+Rzzzzz-[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
337 `)
338 }
339
340 func (s *FederationSuite) TestGetRemoteCollectionError(c *check.C) {
341         req := httptest.NewRequest("GET", "/arvados/v1/collections/zzzzz-4zz18-fakefakefakefak", nil)
342         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
343         resp := s.testRequest(req)
344         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
345 }
346
347 func (s *FederationSuite) TestSignedLocatorPattern(c *check.C) {
348         // Confirm the regular expression identifies other groups of hints correctly
349         c.Check(keepclient.SignedLocatorRe.FindStringSubmatch(`6a4ff0499484c6c79c95cd8c566bd25f+249025+B1+C2+A05227438989d04712ea9ca1c91b556cef01d5cc7@5ba5405b+D3+E4`),
350                 check.DeepEquals,
351                 []string{"6a4ff0499484c6c79c95cd8c566bd25f+249025+B1+C2+A05227438989d04712ea9ca1c91b556cef01d5cc7@5ba5405b+D3+E4",
352                         "6a4ff0499484c6c79c95cd8c566bd25f",
353                         "+249025",
354                         "+B1+C2", "+C2",
355                         "+A05227438989d04712ea9ca1c91b556cef01d5cc7@5ba5405b",
356                         "05227438989d04712ea9ca1c91b556cef01d5cc7", "5ba5405b",
357                         "+D3+E4", "+E4"})
358 }
359
360 func (s *FederationSuite) TestGetLocalCollectionByPDH(c *check.C) {
361         np := arvados.NodeProfile{
362                 Controller: arvados.SystemServiceInstance{Listen: ":"},
363                 RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"),
364                         TLS: true, Insecure: true}}
365         s.testHandler.Cluster.NodeProfiles["*"] = np
366         s.testHandler.NodeProfile = &np
367
368         req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
369         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
370         resp := s.testRequest(req)
371
372         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
373         var col arvados.Collection
374         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
375         c.Check(col.PortableDataHash, check.Equals, arvadostest.UserAgreementPDH)
376         c.Check(col.ManifestText, check.Matches,
377                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+A[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
378 `)
379 }
380
381 func (s *FederationSuite) TestGetRemoteCollectionByPDH(c *check.C) {
382         srv := &httpserver.Server{
383                 Server: http.Server{
384                         Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
385                                 w.WriteHeader(404)
386                         }),
387                 },
388         }
389
390         c.Assert(srv.Start(), check.IsNil)
391         defer srv.Close()
392
393         np := arvados.NodeProfile{
394                 Controller: arvados.SystemServiceInstance{Listen: ":"},
395                 RailsAPI: arvados.SystemServiceInstance{Listen: srv.Addr,
396                         TLS: false, Insecure: true}}
397         s.testHandler.Cluster.NodeProfiles["*"] = np
398         s.testHandler.NodeProfile = &np
399
400         req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
401         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
402         resp := s.testRequest(req)
403
404         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
405         var col arvados.Collection
406         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
407         c.Check(col.PortableDataHash, check.Equals, arvadostest.UserAgreementPDH)
408         c.Check(col.ManifestText, check.Matches,
409                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+Rzzzzz-[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
410 `)
411 }
412
413 func (s *FederationSuite) TestGetCollectionByPDHError(c *check.C) {
414         srv := &httpserver.Server{
415                 Server: http.Server{
416                         Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
417                                 w.WriteHeader(404)
418                         }),
419                 },
420         }
421
422         c.Assert(srv.Start(), check.IsNil)
423         defer srv.Close()
424
425         // Make the "local" cluster to a service that always returns 404
426         np := arvados.NodeProfile{
427                 Controller: arvados.SystemServiceInstance{Listen: ":"},
428                 RailsAPI: arvados.SystemServiceInstance{Listen: srv.Addr,
429                         TLS: false, Insecure: true}}
430         s.testHandler.Cluster.NodeProfiles["*"] = np
431         s.testHandler.NodeProfile = &np
432
433         req := httptest.NewRequest("GET", "/arvados/v1/collections/99999999999999999999999999999999+99", nil)
434         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
435
436         resp := s.testRequest(req)
437         defer resp.Body.Close()
438
439         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
440 }
441
442 func (s *FederationSuite) TestSaltedTokenGetCollectionByPDH(c *check.C) {
443         np := arvados.NodeProfile{
444                 Controller: arvados.SystemServiceInstance{Listen: ":"},
445                 RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"),
446                         TLS: true, Insecure: true}}
447         s.testHandler.Cluster.NodeProfiles["*"] = np
448         s.testHandler.NodeProfile = &np
449
450         req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
451         req.Header.Set("Authorization", "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/282d7d172b6cfdce364c5ed12ddf7417b2d00065")
452         resp := s.testRequest(req)
453
454         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
455         var col arvados.Collection
456         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
457         c.Check(col.PortableDataHash, check.Equals, arvadostest.UserAgreementPDH)
458         c.Check(col.ManifestText, check.Matches,
459                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+A[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
460 `)
461 }
462
463 func (s *FederationSuite) TestSaltedTokenGetCollectionByPDHError(c *check.C) {
464         np := arvados.NodeProfile{
465                 Controller: arvados.SystemServiceInstance{Listen: ":"},
466                 RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"),
467                         TLS: true, Insecure: true}}
468         s.testHandler.Cluster.NodeProfiles["*"] = np
469         s.testHandler.NodeProfile = &np
470
471         req := httptest.NewRequest("GET", "/arvados/v1/collections/99999999999999999999999999999999+99", nil)
472         req.Header.Set("Authorization", "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/282d7d172b6cfdce364c5ed12ddf7417b2d00065")
473         resp := s.testRequest(req)
474
475         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
476 }