lib/controller/federation_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package controller
   6
   7 import (
   8         "encoding/json"
   9         "io/ioutil"
  10         "net/http"
  11         "net/http/httptest"
  12         "net/url"
  13         "os"
  14         "strings"
  15         "time"
  16
  17         "git.curoverse.com/arvados.git/sdk/go/arvados"
  18         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  19         "git.curoverse.com/arvados.git/sdk/go/httpserver"
  20         "git.curoverse.com/arvados.git/sdk/go/keepclient"
  21         "github.com/Sirupsen/logrus"
  22         check "gopkg.in/check.v1"
  23 )
  24
  25 // Gocheck boilerplate
  26 var _ = check.Suite(&FederationSuite{})
  27
  28 type FederationSuite struct {
  29         log *logrus.Logger
  30         // testServer and testHandler are the controller being tested,
  31         // "zhome".
  32         testServer  *httpserver.Server
  33         testHandler *Handler
  34         // remoteServer ("zzzzz") forwards requests to the Rails API
  35         // provided by the integration test environment.
  36         remoteServer *httpserver.Server
  37         // remoteMock ("zmock") appends each incoming request to
  38         // remoteMockRequests, and returns an empty 200 response.
  39         remoteMock         *httpserver.Server
  40         remoteMockRequests []http.Request
  41 }
  42
  43 func (s *FederationSuite) SetUpTest(c *check.C) {
  44         s.log = logrus.New()
  45         s.log.Formatter = &logrus.JSONFormatter{}
  46         s.log.Out = &logWriter{c.Log}
  47
  48         s.remoteServer = newServerFromIntegrationTestEnv(c)
  49         c.Assert(s.remoteServer.Start(), check.IsNil)
  50
  51         s.remoteMock = newServerFromIntegrationTestEnv(c)
  52         s.remoteMock.Server.Handler = http.HandlerFunc(s.remoteMockHandler)
  53         c.Assert(s.remoteMock.Start(), check.IsNil)
  54
  55         nodeProfile := arvados.NodeProfile{
  56                 Controller: arvados.SystemServiceInstance{Listen: ":"},
  57                 RailsAPI:   arvados.SystemServiceInstance{Listen: ":1"}, // local reqs will error "connection refused"
  58         }
  59         s.testHandler = &Handler{Cluster: &arvados.Cluster{
  60                 ClusterID:  "zhome",
  61                 PostgreSQL: integrationTestCluster().PostgreSQL,
  62                 NodeProfiles: map[string]arvados.NodeProfile{
  63                         "*": nodeProfile,
  64                 },
  65         }, NodeProfile: &nodeProfile}
  66         s.testServer = newServerFromIntegrationTestEnv(c)
  67         s.testServer.Server.Handler = httpserver.AddRequestIDs(httpserver.LogRequests(s.log, s.testHandler))
  68
  69         s.testHandler.Cluster.RemoteClusters = map[string]arvados.RemoteCluster{
  70                 "zzzzz": {
  71                         Host:   s.remoteServer.Addr,
  72                         Proxy:  true,
  73                         Scheme: "http",
  74                 },
  75                 "zmock": {
  76                         Host:   s.remoteMock.Addr,
  77                         Proxy:  true,
  78                         Scheme: "http",
  79                 },
  80         }
  81
  82         c.Assert(s.testServer.Start(), check.IsNil)
  83
  84         s.remoteMockRequests = nil
  85 }
  86
  87 func (s *FederationSuite) remoteMockHandler(w http.ResponseWriter, req *http.Request) {
  88         s.remoteMockRequests = append(s.remoteMockRequests, *req)
  89 }
  90
  91 func (s *FederationSuite) TearDownTest(c *check.C) {
  92         if s.remoteServer != nil {
  93                 s.remoteServer.Close()
  94         }
  95         if s.testServer != nil {
  96                 s.testServer.Close()
  97         }
  98 }
  99
 100 func (s *FederationSuite) testRequest(req *http.Request) *http.Response {
 101         resp := httptest.NewRecorder()
 102         s.testServer.Server.Handler.ServeHTTP(resp, req)
 103         return resp.Result()
 104 }
 105
 106 func (s *FederationSuite) TestLocalRequest(c *check.C) {
 107         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zhome-", 1), nil)
 108         resp := s.testRequest(req)
 109         s.checkHandledLocally(c, resp)
 110 }
 111
 112 func (s *FederationSuite) checkHandledLocally(c *check.C, resp *http.Response) {
 113         // Our "home" controller can't handle local requests because
 114         // it doesn't have its own stub/test Rails API, so we rely on
 115         // "connection refused" to indicate the controller tried to
 116         // proxy the request to its local Rails API.
 117         c.Check(resp.StatusCode, check.Equals, http.StatusBadGateway)
 118         s.checkJSONErrorMatches(c, resp, `.*connection refused`)
 119 }
 120
 121 func (s *FederationSuite) TestNoAuth(c *check.C) {
 122         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
 123         resp := s.testRequest(req)
 124         c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
 125         s.checkJSONErrorMatches(c, resp, `Not logged in`)
 126 }
 127
 128 func (s *FederationSuite) TestBadAuth(c *check.C) {
 129         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
 130         req.Header.Set("Authorization", "Bearer aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
 131         resp := s.testRequest(req)
 132         c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
 133         s.checkJSONErrorMatches(c, resp, `Not logged in`)
 134 }
 135
 136 func (s *FederationSuite) TestNoAccess(c *check.C) {
 137         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
 138         req.Header.Set("Authorization", "Bearer "+arvadostest.SpectatorToken)
 139         resp := s.testRequest(req)
 140         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 141         s.checkJSONErrorMatches(c, resp, `.*not found`)
 142 }
 143
 144 func (s *FederationSuite) TestGetUnknownRemote(c *check.C) {
 145         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zz404-", 1), nil)
 146         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 147         resp := s.testRequest(req)
 148         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 149         s.checkJSONErrorMatches(c, resp, `.*no proxy available for cluster zz404`)
 150 }
 151
 152 func (s *FederationSuite) TestRemoteError(c *check.C) {
 153         rc := s.testHandler.Cluster.RemoteClusters["zzzzz"]
 154         rc.Scheme = "https"
 155         s.testHandler.Cluster.RemoteClusters["zzzzz"] = rc
 156
 157         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
 158         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 159         resp := s.testRequest(req)
 160         c.Check(resp.StatusCode, check.Equals, http.StatusBadGateway)
 161         s.checkJSONErrorMatches(c, resp, `.*HTTP response to HTTPS client`)
 162 }
 163
 164 func (s *FederationSuite) TestGetRemoteWorkflow(c *check.C) {
 165         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
 166         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 167         resp := s.testRequest(req)
 168         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 169         var wf arvados.Workflow
 170         c.Check(json.NewDecoder(resp.Body).Decode(&wf), check.IsNil)
 171         c.Check(wf.UUID, check.Equals, arvadostest.WorkflowWithDefinitionYAMLUUID)
 172         c.Check(wf.OwnerUUID, check.Equals, arvadostest.ActiveUserUUID)
 173 }
 174
 175 func (s *FederationSuite) TestOptionsMethod(c *check.C) {
 176         req := httptest.NewRequest("OPTIONS", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
 177         req.Header.Set("Origin", "https://example.com")
 178         resp := s.testRequest(req)
 179         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 180         body, err := ioutil.ReadAll(resp.Body)
 181         c.Check(err, check.IsNil)
 182         c.Check(string(body), check.Equals, "")
 183         c.Check(resp.Header.Get("Access-Control-Allow-Origin"), check.Equals, "*")
 184         for _, hdr := range []string{"Authorization", "Content-Type"} {
 185                 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), check.Matches, ".*"+hdr+".*")
 186         }
 187         for _, method := range []string{"GET", "HEAD", "PUT", "POST", "DELETE"} {
 188                 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), check.Matches, ".*"+method+".*")
 189         }
 190 }
 191
 192 func (s *FederationSuite) TestRemoteWithTokenInQuery(c *check.C) {
 193         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1)+"?api_token="+arvadostest.ActiveToken, nil)
 194         s.testRequest(req)
 195         c.Assert(len(s.remoteMockRequests), check.Equals, 1)
 196         pr := s.remoteMockRequests[0]
 197         // Token is salted and moved from query to Authorization header.
 198         c.Check(pr.URL.String(), check.Not(check.Matches), `.*api_token=.*`)
 199         c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/7fd31b61f39c0e82a4155592163218272cedacdc")
 200 }
 201
 202 func (s *FederationSuite) TestLocalTokenSalted(c *check.C) {
 203         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1), nil)
 204         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 205         s.testRequest(req)
 206         c.Assert(len(s.remoteMockRequests), check.Equals, 1)
 207         pr := s.remoteMockRequests[0]
 208         // The salted token here has a "zzzzz-" UUID instead of a
 209         // "ztest-" UUID because ztest's local database has the
 210         // "zzzzz-" test fixtures. The "secret" part is HMAC(sha1,
 211         // arvadostest.ActiveToken, "zmock") = "7fd3...".
 212         c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/7fd31b61f39c0e82a4155592163218272cedacdc")
 213 }
 214
 215 func (s *FederationSuite) TestRemoteTokenNotSalted(c *check.C) {
 216         // remoteToken can be any v1 token that doesn't appear in
 217         // ztest's local db.
 218         remoteToken := "abcdef00000000000000000000000000000000000000000000"
 219         req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+strings.Replace(arvadostest.WorkflowWithDefinitionYAMLUUID, "zzzzz-", "zmock-", 1), nil)
 220         req.Header.Set("Authorization", "Bearer "+remoteToken)
 221         s.testRequest(req)
 222         c.Assert(len(s.remoteMockRequests), check.Equals, 1)
 223         pr := s.remoteMockRequests[0]
 224         c.Check(pr.Header.Get("Authorization"), check.Equals, "Bearer "+remoteToken)
 225 }
 226
 227 func (s *FederationSuite) TestWorkflowCRUD(c *check.C) {
 228         wf := arvados.Workflow{
 229                 Description: "TestCRUD",
 230         }
 231         {
 232                 body := &strings.Builder{}
 233                 json.NewEncoder(body).Encode(&wf)
 234                 req := httptest.NewRequest("POST", "/arvados/v1/workflows", strings.NewReader(url.Values{
 235                         "workflow": {body.String()},
 236                 }.Encode()))
 237                 req.Header.Set("Content-type", "application/x-www-form-urlencoded")
 238                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 239                 rec := httptest.NewRecorder()
 240                 s.remoteServer.Server.Handler.ServeHTTP(rec, req) // direct to remote -- can't proxy a create req because no uuid
 241                 resp := rec.Result()
 242                 s.checkResponseOK(c, resp)
 243                 json.NewDecoder(resp.Body).Decode(&wf)
 244
 245                 defer func() {
 246                         req := httptest.NewRequest("DELETE", "/arvados/v1/workflows/"+wf.UUID, nil)
 247                         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 248                         s.remoteServer.Server.Handler.ServeHTTP(httptest.NewRecorder(), req)
 249                 }()
 250                 c.Check(wf.UUID, check.Not(check.Equals), "")
 251
 252                 c.Assert(wf.ModifiedAt, check.NotNil)
 253                 c.Logf("wf.ModifiedAt: %v", wf.ModifiedAt)
 254                 c.Check(time.Since(*wf.ModifiedAt) < time.Minute, check.Equals, true)
 255         }
 256         for _, method := range []string{"PATCH", "PUT", "POST"} {
 257                 form := url.Values{
 258                         "workflow": {`{"description": "Updated with ` + method + `"}`},
 259                 }
 260                 if method == "POST" {
 261                         form["_method"] = []string{"PATCH"}
 262                 }
 263                 req := httptest.NewRequest(method, "/arvados/v1/workflows/"+wf.UUID, strings.NewReader(form.Encode()))
 264                 req.Header.Set("Content-type", "application/x-www-form-urlencoded")
 265                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 266                 resp := s.testRequest(req)
 267                 s.checkResponseOK(c, resp)
 268                 err := json.NewDecoder(resp.Body).Decode(&wf)
 269                 c.Check(err, check.IsNil)
 270
 271                 c.Check(wf.Description, check.Equals, "Updated with "+method)
 272         }
 273         {
 274                 req := httptest.NewRequest("DELETE", "/arvados/v1/workflows/"+wf.UUID, nil)
 275                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 276                 resp := s.testRequest(req)
 277                 s.checkResponseOK(c, resp)
 278                 err := json.NewDecoder(resp.Body).Decode(&wf)
 279                 c.Check(err, check.IsNil)
 280         }
 281         {
 282                 req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+wf.UUID, nil)
 283                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 284                 resp := s.testRequest(req)
 285                 c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 286         }
 287 }
 288
 289 func (s *FederationSuite) checkResponseOK(c *check.C, resp *http.Response) {
 290         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 291         if resp.StatusCode != http.StatusOK {
 292                 body, err := ioutil.ReadAll(resp.Body)
 293                 c.Logf("... response body = %q, %v\n", body, err)
 294         }
 295 }
 296
 297 func (s *FederationSuite) checkJSONErrorMatches(c *check.C, resp *http.Response, re string) {
 298         var jresp httpserver.ErrorResponse
 299         err := json.NewDecoder(resp.Body).Decode(&jresp)
 300         c.Check(err, check.IsNil)
 301         c.Assert(len(jresp.Errors), check.Equals, 1)
 302         c.Check(jresp.Errors[0], check.Matches, re)
 303 }
 304
 305 func (s *FederationSuite) localServiceReturns404(c *check.C) *httpserver.Server {
 306         srv := &httpserver.Server{
 307                 Server: http.Server{
 308                         Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 309                                 w.WriteHeader(404)
 310                         }),
 311                 },
 312         }
 313
 314         c.Assert(srv.Start(), check.IsNil)
 315
 316         np := arvados.NodeProfile{
 317                 Controller: arvados.SystemServiceInstance{Listen: ":"},
 318                 RailsAPI: arvados.SystemServiceInstance{Listen: srv.Addr,
 319                         TLS: false, Insecure: true}}
 320         s.testHandler.Cluster.NodeProfiles["*"] = np
 321         s.testHandler.NodeProfile = &np
 322
 323         return srv
 324 }
 325
 326 func (s *FederationSuite) TestGetLocalCollection(c *check.C) {
 327         np := arvados.NodeProfile{
 328                 Controller: arvados.SystemServiceInstance{Listen: ":"},
 329                 RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"),
 330                         TLS: true, Insecure: true}}
 331         s.testHandler.Cluster.ClusterID = "zzzzz"
 332         s.testHandler.Cluster.NodeProfiles["*"] = np
 333         s.testHandler.NodeProfile = &np
 334
 335         req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementCollection, nil)
 336         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 337         resp := s.testRequest(req)
 338
 339         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 340         var col arvados.Collection
 341         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
 342         c.Check(col.UUID, check.Equals, arvadostest.UserAgreementCollection)
 343         c.Check(col.ManifestText, check.Matches,
 344                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+A[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
 345 `)
 346 }
 347
 348 func (s *FederationSuite) TestGetRemoteCollection(c *check.C) {
 349         defer s.localServiceReturns404(c).Close()
 350
 351         req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementCollection, nil)
 352         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 353         resp := s.testRequest(req)
 354         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 355         var col arvados.Collection
 356         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
 357         c.Check(col.UUID, check.Equals, arvadostest.UserAgreementCollection)
 358         c.Check(col.ManifestText, check.Matches,
 359                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+Rzzzzz-[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
 360 `)
 361 }
 362
 363 func (s *FederationSuite) TestGetRemoteCollectionError(c *check.C) {
 364         defer s.localServiceReturns404(c).Close()
 365
 366         req := httptest.NewRequest("GET", "/arvados/v1/collections/zzzzz-4zz18-fakefakefakefak", nil)
 367         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 368         resp := s.testRequest(req)
 369         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 370 }
 371
 372 func (s *FederationSuite) TestSignedLocatorPattern(c *check.C) {
 373         // Confirm the regular expression identifies other groups of hints correctly
 374         c.Check(keepclient.SignedLocatorRe.FindStringSubmatch(`6a4ff0499484c6c79c95cd8c566bd25f+249025+B1+C2+A05227438989d04712ea9ca1c91b556cef01d5cc7@5ba5405b+D3+E4`),
 375                 check.DeepEquals,
 376                 []string{"6a4ff0499484c6c79c95cd8c566bd25f+249025+B1+C2+A05227438989d04712ea9ca1c91b556cef01d5cc7@5ba5405b+D3+E4",
 377                         "6a4ff0499484c6c79c95cd8c566bd25f",
 378                         "+249025",
 379                         "+B1+C2", "+C2",
 380                         "+A05227438989d04712ea9ca1c91b556cef01d5cc7@5ba5405b",
 381                         "05227438989d04712ea9ca1c91b556cef01d5cc7", "5ba5405b",
 382                         "+D3+E4", "+E4"})
 383 }
 384
 385 func (s *FederationSuite) TestGetLocalCollectionByPDH(c *check.C) {
 386         np := arvados.NodeProfile{
 387                 Controller: arvados.SystemServiceInstance{Listen: ":"},
 388                 RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"),
 389                         TLS: true, Insecure: true}}
 390         s.testHandler.Cluster.NodeProfiles["*"] = np
 391         s.testHandler.NodeProfile = &np
 392
 393         req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
 394         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 395         resp := s.testRequest(req)
 396
 397         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 398         var col arvados.Collection
 399         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
 400         c.Check(col.PortableDataHash, check.Equals, arvadostest.UserAgreementPDH)
 401         c.Check(col.ManifestText, check.Matches,
 402                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+A[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
 403 `)
 404 }
 405
 406 func (s *FederationSuite) TestGetRemoteCollectionByPDH(c *check.C) {
 407         defer s.localServiceReturns404(c).Close()
 408
 409         req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
 410         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 411         resp := s.testRequest(req)
 412
 413         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 414
 415         var col arvados.Collection
 416         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
 417         c.Check(col.PortableDataHash, check.Equals, arvadostest.UserAgreementPDH)
 418         c.Check(col.ManifestText, check.Matches,
 419                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+Rzzzzz-[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
 420 `)
 421 }
 422
 423 func (s *FederationSuite) TestGetCollectionByPDHError(c *check.C) {
 424         defer s.localServiceReturns404(c).Close()
 425
 426         req := httptest.NewRequest("GET", "/arvados/v1/collections/99999999999999999999999999999999+99", nil)
 427         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 428
 429         resp := s.testRequest(req)
 430         defer resp.Body.Close()
 431
 432         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 433 }
 434
 435 func (s *FederationSuite) TestGetCollectionByPDHErrorBadHash(c *check.C) {
 436         defer s.localServiceReturns404(c).Close()
 437
 438         srv2 := &httpserver.Server{
 439                 Server: http.Server{
 440                         Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 441                                 w.WriteHeader(200)
 442                                 // Return a collection where the hash
 443                                 // of the manifest text doesn't match
 444                                 // PDH that was requested.
 445                                 var col arvados.Collection
 446                                 col.PortableDataHash = "99999999999999999999999999999999+99"
 447                                 col.ManifestText = `. 6a4ff0499484c6c79c95cd8c566bd25f\+249025 0:249025:GNU_General_Public_License,_version_3.pdf
 448 `
 449                                 enc := json.NewEncoder(w)
 450                                 enc.Encode(col)
 451                         }),
 452                 },
 453         }
 454
 455         c.Assert(srv2.Start(), check.IsNil)
 456         defer srv2.Close()
 457
 458         // Direct zzzzz to service that returns a 200 result with a bogus manifest_text
 459         s.testHandler.Cluster.RemoteClusters["zzzzz"] = arvados.RemoteCluster{
 460                 Host:   srv2.Addr,
 461                 Proxy:  true,
 462                 Scheme: "http",
 463         }
 464
 465         req := httptest.NewRequest("GET", "/arvados/v1/collections/99999999999999999999999999999999+99", nil)
 466         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 467
 468         resp := s.testRequest(req)
 469         defer resp.Body.Close()
 470
 471         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 472 }
 473
 474 func (s *FederationSuite) TestSaltedTokenGetCollectionByPDH(c *check.C) {
 475         np := arvados.NodeProfile{
 476                 Controller: arvados.SystemServiceInstance{Listen: ":"},
 477                 RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"),
 478                         TLS: true, Insecure: true}}
 479         s.testHandler.Cluster.NodeProfiles["*"] = np
 480         s.testHandler.NodeProfile = &np
 481
 482         req := httptest.NewRequest("GET", "/arvados/v1/collections/"+arvadostest.UserAgreementPDH, nil)
 483         req.Header.Set("Authorization", "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/282d7d172b6cfdce364c5ed12ddf7417b2d00065")
 484         resp := s.testRequest(req)
 485
 486         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 487         var col arvados.Collection
 488         c.Check(json.NewDecoder(resp.Body).Decode(&col), check.IsNil)
 489         c.Check(col.PortableDataHash, check.Equals, arvadostest.UserAgreementPDH)
 490         c.Check(col.ManifestText, check.Matches,
 491                 `\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025\+A[0-9a-f]{40}@[0-9a-f]{8} 0:249025:GNU_General_Public_License,_version_3.pdf
 492 `)
 493 }
 494
 495 func (s *FederationSuite) TestSaltedTokenGetCollectionByPDHError(c *check.C) {
 496         np := arvados.NodeProfile{
 497                 Controller: arvados.SystemServiceInstance{Listen: ":"},
 498                 RailsAPI: arvados.SystemServiceInstance{Listen: os.Getenv("ARVADOS_TEST_API_HOST"),
 499                         TLS: true, Insecure: true}}
 500         s.testHandler.Cluster.NodeProfiles["*"] = np
 501         s.testHandler.NodeProfile = &np
 502
 503         req := httptest.NewRequest("GET", "/arvados/v1/collections/99999999999999999999999999999999+99", nil)
 504         req.Header.Set("Authorization", "Bearer v2/zzzzz-gj3su-077z32aux8dg2s1/282d7d172b6cfdce364c5ed12ddf7417b2d00065")
 505         resp := s.testRequest(req)
 506
 507         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 508 }
 509
 510 func (s *FederationSuite) TestGetRemoteContainerRequest(c *check.C) {
 511         defer s.localServiceReturns404(c).Close()
 512         req := httptest.NewRequest("GET", "/arvados/v1/container_requests/"+arvadostest.QueuedContainerRequestUUID, nil)
 513         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 514         resp := s.testRequest(req)
 515         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 516         var cr arvados.ContainerRequest
 517         c.Check(json.NewDecoder(resp.Body).Decode(&cr), check.IsNil)
 518         c.Check(cr.UUID, check.Equals, arvadostest.QueuedContainerRequestUUID)
 519         c.Check(cr.Priority, check.Equals, 1)
 520 }
 521
 522 func (s *FederationSuite) TestUpdateRemoteContainerRequest(c *check.C) {
 523         defer s.localServiceReturns404(c).Close()
 524         req := httptest.NewRequest("PATCH", "/arvados/v1/container_requests/"+arvadostest.QueuedContainerRequestUUID,
 525                 strings.NewReader(`{"container_request": {"priority": 696}}`))
 526         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 527         req.Header.Set("Content-type", "application/json")
 528         resp := s.testRequest(req)
 529         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 530         var cr arvados.ContainerRequest
 531         c.Check(json.NewDecoder(resp.Body).Decode(&cr), check.IsNil)
 532         c.Check(cr.UUID, check.Equals, arvadostest.QueuedContainerRequestUUID)
 533         c.Check(cr.Priority, check.Equals, 696)
 534 }
 535
 536 func (s *FederationSuite) TestCreateRemoteContainerRequest1(c *check.C) {
 537         defer s.localServiceReturns404(c).Close()
 538         req := httptest.NewRequest("POST", "/arvados/v1/container_requests",
 539                 strings.NewReader(`{
 540   "cluster_id": "zzzzz",
 541   "container_request": {
 542     "name": "hello world",
 543     "state": "Uncommitted",
 544     "output_path": "/",
 545     "container_image": "123",
 546     "command": ["abc"]
 547   }
 548 }
 549 `))
 550         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 551         req.Header.Set("Content-type", "application/json")
 552         resp := s.testRequest(req)
 553         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 554         var cr arvados.ContainerRequest
 555         c.Check(json.NewDecoder(resp.Body).Decode(&cr), check.IsNil)
 556         c.Check(cr.Name, check.Equals, "hello world")
 557 }
 558
 559 func (s *FederationSuite) TestCreateRemoteContainerRequest2(c *check.C) {
 560         defer s.localServiceReturns404(c).Close()
 561         // pass cluster_id via query parameter, this allows arvados-controller
 562         // to avoid parsing the body
 563         req := httptest.NewRequest("POST", "/arvados/v1/container_requests?cluster_id=zzzzz",
 564                 strings.NewReader(`{
 565   "container_request": {
 566     "name": "hello world",
 567     "state": "Uncommitted",
 568     "output_path": "/",
 569     "container_image": "123",
 570     "command": ["abc"]
 571   }
 572 }
 573 `))
 574         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 575         req.Header.Set("Content-type", "application/json")
 576         resp := s.testRequest(req)
 577         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 578         var cr arvados.ContainerRequest
 579         c.Check(json.NewDecoder(resp.Body).Decode(&cr), check.IsNil)
 580         c.Check(cr.Name, check.Equals, "hello world")
 581 }
 582
 583 func (s *FederationSuite) TestGetRemoteContainer(c *check.C) {
 584         defer s.localServiceReturns404(c).Close()
 585         req := httptest.NewRequest("GET", "/arvados/v1/containers/"+arvadostest.QueuedContainerUUID, nil)
 586         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 587         resp := s.testRequest(req)
 588         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 589         var cn arvados.Container
 590         c.Check(json.NewDecoder(resp.Body).Decode(&cn), check.IsNil)
 591         c.Check(cn.UUID, check.Equals, arvadostest.QueuedContainerUUID)
 592 }