1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
17 "git.curoverse.com/arvados.git/sdk/go/arvados"
18 "git.curoverse.com/arvados.git/sdk/go/arvadostest"
19 "git.curoverse.com/arvados.git/sdk/go/httpserver"
20 "git.curoverse.com/arvados.git/sdk/go/keepclient"
21 "github.com/Sirupsen/logrus"
22 check "gopkg.in/check.v1"
25 // Gocheck boilerplate
26 var _ = check.Suite(&FederationSuite{})
28 type FederationSuite struct {
30 // testServer and testHandler are the controller being tested,
32 testServer *httpserver.Server
34 // remoteServer ("zzzzz") forwards requests to the Rails API
35 // provided by the integration test environment.
36 remoteServer *httpserver.Server
37 // remoteMock ("zmock") appends each incoming request to
38 // remoteMockRequests, and returns an empty 200 response.
39 remoteMock *httpserver.Server
40 remoteMockRequests []http.Request
43 func (s *FederationSuite) SetUpTest(c *check.C) {
45 s.log.Formatter = &logrus.JSONFormatter{}
46 s.log.Out = &logWriter{c.Log}
48 s.remoteServer = newServerFromIntegrationTestEnv(c)
49 c.Assert(s.remoteServer.Start(), check.IsNil)
51 s.remoteMock = newServerFromIntegrationTestEnv(c)
52 s.remoteMock.Server.Handler = http.HandlerFunc(s.remoteMockHandler)
53 c.Assert(s.remoteMock.Start(), check.IsNil)
55 nodeProfile := arvados.NodeProfile{
56 Controller: arvados.SystemServiceInstance{Listen: ":"},
57 RailsAPI: arvados.SystemServiceInstance{Listen: ":1"}, // local reqs will error "connection refused"
59 s.testHandler = &Handler{Cluster: &arvados.Cluster{
61 PostgreSQL: integrationTestCluster().PostgreSQL,
62 NodeProfiles: map[string]arvados.NodeProfile{
65 }, NodeProfile: &nodeProfile}
66 s.testServer = newServerFromIntegrationTestEnv(c)
67 s.testServer.Server.Handler = httpserver.AddRequestIDs(httpserver.LogRequests(s.log, s.testHandler))
69 s.testHandler.Cluster.RemoteClusters = map[string]arvados.RemoteCluster{
71 Host: s.remoteServer.Addr,
76 Host: s.remoteMock.Addr,
82 c.Assert(s.testServer.Start(), check.IsNil)
84 s.remoteMockRequests = nil
87 func (s *FederationSuite) remoteMockHandler(w http.ResponseWriter, req *http.Request) {
88 s.remoteMockRequests = append(s.remoteMockRequests, *req)
91 func (s *FederationSuite) TearDownTest(c *check.C) {
92 if s.remoteServer != nil {
93 s.remoteServer.Close()
95 if s.testServer != nil {
100 func (s *FederationSuite) testRequest(req *http.Request) *http.Response {
101 resp := httptest.NewRecorder()
102 s.testServer.Server.Handler.ServeHTTP(resp, req)
106 func (s *FederationSuite) TestLocalRequest(c *check.C) {
107 req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zhome-", 1), nil)
108 resp := s.testRequest(req)
109 s.checkHandledLocally(c, resp)
112 func (s *FederationSuite) checkHandledLocally(c *check.C, resp *http.Response) {
113 // Our "home" controller can't handle local requests because
114 // it doesn't have its own stub/test Rails API, so we rely on
115 // "connection refused" to indicate the controller tried to
116 // proxy the request to its local Rails API.
117 c.Check(resp.StatusCode, check.Equals, http.StatusBadGateway)
118 s.checkJSONErrorMatches(c, resp, `.*connection refused`)
121 func (s *FederationSuite) TestNoAuth(c *check.C) {
122 req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
123 resp := s.testRequest(req)
124 c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
125 s.checkJSONErrorMatches(c, resp, `Not logged in`)
128 func (s *FederationSuite) TestBadAuth(c *check.C) {
129 req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
130 req.Header.Set("Authorization", "Bearer aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
131 resp := s.testRequest(req)
132 c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
133 s.checkJSONErrorMatches(c, resp, `Not logged in`)
136 func (s *FederationSuite) TestNoAccess(c *check.C) {
137 req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
138 req.Header.Set("Authorization", "Bearer "+arvadostest.SpectatorToken)
139 resp := s.testRequest(req)
140 c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
141 s.checkJSONErrorMatches(c, resp, `.*not found`)
144 func (s *FederationSuite) TestGetUnknownRemote(c *check.C) {
145 req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zz404-", 1), nil)
146 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
147 resp := s.testRequest(req)
148 c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
149 s.checkJSONErrorMatches(c, resp, `.*no proxy available for cluster zz404`)
152 func (s *FederationSuite) TestRemoteError(c *check.C) {
153 rc := s.testHandler.Cluster.RemoteClusters["zzzzz"]
155 s.testHandler.Cluster.RemoteClusters["zzzzz"] = rc
157 req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
158 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
159 resp := s.testRequest(req)
160 c.Check(resp.StatusCode, check.Equals, http.StatusBadGateway)
161 s.checkJSONErrorMatches(c, resp, `.*HTTP response to HTTPS client`)
164 func (s *FederationSuite) TestGetRemoteWorkflow(c *check.C) {
165 req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
166 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
167 resp := s.testRequest(req)
168 c.Check(resp.StatusCode, check.Equals, http.StatusOK)
169 var wf arvados.Workflow
170 c.Check(json.NewDecoder(resp.Body).Decode(&wf), check.IsNil)
171 c.Check(wf.UUID, check.Equals, arvadostest.WorkflowWithDefinitionYAMLUUID)
172 c.Check(wf.OwnerUUID, check.Equals, arvadostest.ActiveUserUUID)
175 func (s *FederationSuite) TestOptionsMethod(c *check.C) {
176 req := httptest.NewRequest("OPTIONS", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
177 req.Header.Set("Origin", "https://example.com")
178 resp := s.testRequest(req)
179 c.Check(resp.StatusCode, check.Equals, http.StatusOK)
180 body, err := ioutil.ReadAll(resp.Body)
181 c.Check(err, check.IsNil)
182 c.Check(string(body), check.Equals, "")
183 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), check.Equals, "*")
184 for _, hdr := range []string{"Authorization", "Content-Type"} {
185 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), check.Matches, ".*"+hdr+".*")
187 for _, method := range []string{"GET", "HEAD", "PUT", "POST", "DELETE"} {
188 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), check.Matches, ".*"+method+".*")
192 func (s *FederationSuite) TestRemoteWithTokenInQuery(c *check.C) {
193 req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1)+"?api_token="+arvadostest.ActiveToken, nil)
195 c.Assert(len(s.remoteMockRequests), check.Equals, 1)
196 pr := s.remoteMockRequests[0]
197 // Token is salted and moved from query to Authorization header.
198 c.Check(pr.URL.String(), check.Not(check.Matches), `.*api_token=.*`)
199 c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/7fd31b61f39c0e82a4155592163218272cedacdc")
202 func (s *FederationSuite) TestLocalTokenSalted(c *check.C) {
203 req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1), nil)
204 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
206 c.Assert(len(s.remoteMockRequests), check.Equals, 1)
207 pr := s.remoteMockRequests[0]
208 // The salted token here has a "zzzzz-" UUID instead of a
209 // "ztest-" UUID because ztest's local database has the
210 // "zzzzz-" test fixtures. The "secret" part is HMAC(sha1,
211 // arvadostest.ActiveToken, "zmock") = "7fd3...".
212 c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/7fd31b61f39c0e82a4155592163218272cedacdc")
215 func (s *FederationSuite) TestRemoteTokenNotSalted(c *check.C) {
216 // remoteToken can be any v1 token that doesn't appear in
218 remoteToken := "abcdef00000000000000000000000000000000000000000000"
219 req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1), nil)
220 req.Header.Set("Authorization", "Bearer "+remoteToken)
222 c.Assert(len(s.remoteMockRequests), check.Equals, 1)
223 pr := s.remoteMockRequests[0]
224 c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer "+remoteToken)
227 func (s *FederationSuite) TestWorkflowCRUD(c *check.C) {
228 wf := arvados.Workflow{
229 Description: "TestCRUD",
232 body := &strings.Builder{}
233 json.NewEncoder(body).Encode(&wf)
234 req := httptest.NewRequest("POST", "/arvados/v1/workflows", strings.NewReader(url.Values{
235 "workflow": {body.String()},
237 req.Header.Set("Content-type", "application/x-www-form-urlencoded")
238 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
239 rec := httptest.NewRecorder()
240 s.remoteServer.Server.Handler.ServeHTTP(rec, req) // direct to remote -- can't proxy a create req because no uuid
242 s.checkResponseOK(c, resp)
243 json.NewDecoder(resp.Body).Decode(&wf)
246 req := httptest.NewRequest("DELETE", "/arvados/v1/workflows/"+wf.UUID, nil)
247 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
248 s.remoteServer.Server.Handler.ServeHTTP(httptest.NewRecorder(), req)
250 c.Check(wf.UUID, check.Not(check.Equals), "")
252 c.Assert(wf.ModifiedAt, check.NotNil)
253 c.Logf("wf.ModifiedAt: %v", wf.ModifiedAt)
254 c.Check(time.Since(*wf.ModifiedAt) < time.Minute, check.Equals, true)
256 for _, method := range []string{"PATCH", "PUT", "POST"} {
258 "workflow": {`{"description": "Updated with ` + method + `"}`},
260 if method == "POST" {
261 form["_method"] = []string{"PATCH"}
263 req := httptest.NewRequest(method, "/arvados/v1/workflows/"+wf.UUID, strings.NewReader(form.Encode()))
264 req.Header.Set("Content-type", "application/x-www-form-urlencoded")
265 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
266 resp := s.testRequest(req)
267 s.checkResponseOK(c, resp)
268 err := json.NewDecoder(resp.Body).Decode(&wf)
269 c.Check(err, check.IsNil)
271 c.Check(wf.Description, check.Equals, "Updated with "+method)
274 req := httptest.NewRequest("DELETE", "/arvados/v1/workflows/"+wf.UUID, nil)
275 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
276 resp := s.testRequest(req)
277 s.checkResponseOK(c, resp)
278 err := json.NewDecoder(resp.Body).Decode(&wf)
279 c.Check(err, check.IsNil)
282 req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+wf.UUID, nil)
283 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
284 resp := s.testRequest(req)
285 c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
289 func (s *FederationSuite) checkResponseOK(c *check.C, resp *http.Response) {
290 c.Check(resp.StatusCode, check.Equals, http.StatusOK)
291 if resp.StatusCode != http.StatusOK {
292 body, err := ioutil.ReadAll(resp.Body)
293 c.Logf("... response body = %q, %v\n", body, err)
297 func (s *FederationSuite) checkJSONErrorMatches(c *check.C, resp *http.Response, re string) {
298 var jresp httpserver.ErrorResponse
299 err := json.NewDecoder(resp.Body).Decode(&jresp)
300 c.Check(err, check.IsNil)
301 c.Assert(len(jresp.Errors), check.Equals, 1)
302 c.Check(jresp.Errors[0], check.Matches, re)
305 func (s *FederationSuite) TestGetLocalCollection(c *check.C) {
306 np := arvados.NodeProfile{
307 Controller: arvados.SystemServiceInstance{Listen: ":"},
308 RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"),
309 TLS: true, Insecure: true}}
310 s.testHandler.Cluster.ClusterID = "zzzzz"
311 s.testHandler.Cluster.NodeProfiles["*"] = np
312 s.testHandler.NodeProfile = &np
314 req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementCollection, nil)
315 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
316 resp := s.testRequest(req)
318 c.Check(resp.StatusCode, check.Equals, http.StatusOK)
319 var col arvados.Collection
320 c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
321 c.Check(col.UUID, check.Equals, arvadostest.UserAgreementCollection)
322 c.Check(col.ManifestText, check.Matches,
323 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+A[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
327 func (s *FederationSuite) TestGetRemoteCollection(c *check.C) {
328 req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementCollection, nil)
329 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
330 resp := s.testRequest(req)
331 c.Check(resp.StatusCode, check.Equals, http.StatusOK)
332 var col arvados.Collection
333 c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
334 c.Check(col.UUID, check.Equals, arvadostest.UserAgreementCollection)
335 c.Check(col.ManifestText, check.Matches,
336 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+Rzzzzz-[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
340 func (s *FederationSuite) TestGetRemoteCollectionError(c *check.C) {
341 req := httptest.NewRequest("GET", "/arvados/v1/collections/zzzzz-4zz18-fakefakefakefak", nil)
342 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
343 resp := s.testRequest(req)
344 c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
347 func (s *FederationSuite) TestSignedLocatorPattern(c *check.C) {
348 // Confirm the regular expression identifies other groups of hints correctly
349 c.Check(keepclient.SignedLocatorRe.FindStringSubmatch(`6a4ff0499484c6c79c95cd8c566bd25f+249025+B1+C2+A05227438989d04712ea9ca1c91b556cef01d5cc7@5ba5405b+D3+E4`),
351 []string{"6a4ff0499484c6c79c95cd8c566bd25f+249025+B1+C2+A05227438989d04712ea9ca1c91b556cef01d5cc7@5ba5405b+D3+E4",
352 "6a4ff0499484c6c79c95cd8c566bd25f",
355 "+A05227438989d04712ea9ca1c91b556cef01d5cc7@5ba5405b",
356 "05227438989d04712ea9ca1c91b556cef01d5cc7", "5ba5405b",
360 func (s *FederationSuite) TestGetLocalCollectionByPDH(c *check.C) {
361 np := arvados.NodeProfile{
362 Controller: arvados.SystemServiceInstance{Listen: ":"},
363 RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"),
364 TLS: true, Insecure: true}}
365 s.testHandler.Cluster.NodeProfiles["*"] = np
366 s.testHandler.NodeProfile = &np
368 req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
369 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
370 resp := s.testRequest(req)
372 c.Check(resp.StatusCode, check.Equals, http.StatusOK)
373 var col arvados.Collection
374 c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
375 c.Check(col.PortableDataHash, check.Equals, arvadostest.UserAgreementPDH)
376 c.Check(col.ManifestText, check.Matches,
377 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+A[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
381 func (s *FederationSuite) TestGetRemoteCollectionByPDH(c *check.C) {
382 srv := &httpserver.Server{
384 Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
390 c.Assert(srv.Start(), check.IsNil)
393 np := arvados.NodeProfile{
394 Controller: arvados.SystemServiceInstance{Listen: ":"},
395 RailsAPI: arvados.SystemServiceInstance{Listen: srv.Addr,
396 TLS: false, Insecure: true}}
397 s.testHandler.Cluster.NodeProfiles["*"] = np
398 s.testHandler.NodeProfile = &np
400 req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
401 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
402 resp := s.testRequest(req)
404 c.Check(resp.StatusCode, check.Equals, http.StatusOK)
405 var col arvados.Collection
406 c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
407 c.Check(col.PortableDataHash, check.Equals, arvadostest.UserAgreementPDH)
408 c.Check(col.ManifestText, check.Matches,
409 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+Rzzzzz-[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
413 func (s *FederationSuite) TestGetCollectionByPDHError(c *check.C) {
414 srv := &httpserver.Server{
416 Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
422 c.Assert(srv.Start(), check.IsNil)
425 // Make the "local" cluster to a service that always returns 404
426 np := arvados.NodeProfile{
427 Controller: arvados.SystemServiceInstance{Listen: ":"},
428 RailsAPI: arvados.SystemServiceInstance{Listen: srv.Addr,
429 TLS: false, Insecure: true}}
430 s.testHandler.Cluster.NodeProfiles["*"] = np
431 s.testHandler.NodeProfile = &np
433 req := httptest.NewRequest("GET", "/arvados/v1/collections/99999999999999999999999999999999+99", nil)
434 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
436 resp := s.testRequest(req)
437 defer resp.Body.Close()
439 c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
442 func (s *FederationSuite) TestSaltedTokenGetCollectionByPDH(c *check.C) {
443 np := arvados.NodeProfile{
444 Controller: arvados.SystemServiceInstance{Listen: ":"},
445 RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"),
446 TLS: true, Insecure: true}}
447 s.testHandler.Cluster.NodeProfiles["*"] = np
448 s.testHandler.NodeProfile = &np
450 req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
451 req.Header.Set("Authorization", "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/282d7d172b6cfdce364c5ed12ddf7417b2d00065")
452 resp := s.testRequest(req)
454 c.Check(resp.StatusCode, check.Equals, http.StatusOK)
455 var col arvados.Collection
456 c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
457 c.Check(col.PortableDataHash, check.Equals, arvadostest.UserAgreementPDH)
458 c.Check(col.ManifestText, check.Matches,
459 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+A[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
463 func (s *FederationSuite) TestSaltedTokenGetCollectionByPDHError(c *check.C) {
464 np := arvados.NodeProfile{
465 Controller: arvados.SystemServiceInstance{Listen: ":"},
466 RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"),
467 TLS: true, Insecure: true}}
468 s.testHandler.Cluster.NodeProfiles["*"] = np
469 s.testHandler.NodeProfile = &np
471 req := httptest.NewRequest("GET", "/arvados/v1/collections/99999999999999999999999999999999+99", nil)
472 req.Header.Set("Authorization", "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/282d7d172b6cfdce364c5ed12ddf7417b2d00065")
473 resp := s.testRequest(req)
475 c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)