services/keep-web/handler_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "fmt"
  10         "html"
  11         "io/ioutil"
  12         "net/http"
  13         "net/http/httptest"
  14         "net/url"
  15         "os"
  16         "path/filepath"
  17         "regexp"
  18         "strings"
  19
  20         "git.curoverse.com/arvados.git/lib/config"
  21         "git.curoverse.com/arvados.git/sdk/go/arvados"
  22         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  23         "git.curoverse.com/arvados.git/sdk/go/auth"
  24         check "gopkg.in/check.v1"
  25 )
  26
  27 var _ = check.Suite(&UnitSuite{})
  28
  29 type UnitSuite struct {
  30         Config *arvados.Config
  31 }
  32
  33 func (s *UnitSuite) SetUpTest(c *check.C) {
  34         ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), nil)
  35         ldr.Path = "-"
  36         cfg, err := ldr.Load()
  37         c.Assert(err, check.IsNil)
  38         s.Config = cfg
  39 }
  40
  41 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
  42         h := handler{Config: newConfig(s.Config)}
  43         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
  44         req := &http.Request{
  45                 Method:     "OPTIONS",
  46                 Host:       u.Host,
  47                 URL:        u,
  48                 RequestURI: u.RequestURI(),
  49                 Header: http.Header{
  50                         "Origin":                        {"https://workbench.example"},
  51                         "Access-Control-Request-Method": {"POST"},
  52                 },
  53         }
  54
  55         // Check preflight for an allowed request
  56         resp := httptest.NewRecorder()
  57         h.ServeHTTP(resp, req)
  58         c.Check(resp.Code, check.Equals, http.StatusOK)
  59         c.Check(resp.Body.String(), check.Equals, "")
  60         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
  61         c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
  62         c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout")
  63
  64         // Check preflight for a disallowed request
  65         resp = httptest.NewRecorder()
  66         req.Header.Set("Access-Control-Request-Method", "MAKE-COFFEE")
  67         h.ServeHTTP(resp, req)
  68         c.Check(resp.Body.String(), check.Equals, "")
  69         c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
  70 }
  71
  72 func (s *UnitSuite) TestInvalidUUID(c *check.C) {
  73         bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-"
  74         token := arvadostest.ActiveToken
  75         for _, trial := range []string{
  76                 "http://keep-web/c=" + bogusID + "/foo",
  77                 "http://keep-web/c=" + bogusID + "/t=" + token + "/foo",
  78                 "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo",
  79                 "http://keep-web/collections/" + bogusID + "/foo",
  80                 "http://" + bogusID + ".keep-web/" + bogusID + "/foo",
  81                 "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
  82         } {
  83                 c.Log(trial)
  84                 u := mustParseURL(trial)
  85                 req := &http.Request{
  86                         Method:     "GET",
  87                         Host:       u.Host,
  88                         URL:        u,
  89                         RequestURI: u.RequestURI(),
  90                 }
  91                 resp := httptest.NewRecorder()
  92                 cfg := newConfig(s.Config)
  93                 cfg.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
  94                 h := handler{Config: cfg}
  95                 h.ServeHTTP(resp, req)
  96                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
  97         }
  98 }
  99
 100 func mustParseURL(s string) *url.URL {
 101         r, err := url.Parse(s)
 102         if err != nil {
 103                 panic("parse URL: " + s)
 104         }
 105         return r
 106 }
 107
 108 func (s *IntegrationSuite) TestVhost404(c *check.C) {
 109         for _, testURL := range []string{
 110                 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
 111                 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
 112         } {
 113                 resp := httptest.NewRecorder()
 114                 u := mustParseURL(testURL)
 115                 req := &http.Request{
 116                         Method:     "GET",
 117                         URL:        u,
 118                         RequestURI: u.RequestURI(),
 119                 }
 120                 s.testServer.Handler.ServeHTTP(resp, req)
 121                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 122                 c.Check(resp.Body.String(), check.Equals, "")
 123         }
 124 }
 125
 126 // An authorizer modifies an HTTP request to make use of the given
 127 // token -- by adding it to a header, cookie, query param, or whatever
 128 // -- and returns the HTTP status code we should expect from keep-web if
 129 // the token is invalid.
 130 type authorizer func(*http.Request, string) int
 131
 132 func (s *IntegrationSuite) TestVhostViaAuthzHeader(c *check.C) {
 133         s.doVhostRequests(c, authzViaAuthzHeader)
 134 }
 135 func authzViaAuthzHeader(r *http.Request, tok string) int {
 136         r.Header.Add("Authorization", "OAuth2 "+tok)
 137         return http.StatusUnauthorized
 138 }
 139
 140 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
 141         s.doVhostRequests(c, authzViaCookieValue)
 142 }
 143 func authzViaCookieValue(r *http.Request, tok string) int {
 144         r.AddCookie(&http.Cookie{
 145                 Name:  "arvados_api_token",
 146                 Value: auth.EncodeTokenCookie([]byte(tok)),
 147         })
 148         return http.StatusUnauthorized
 149 }
 150
 151 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
 152         s.doVhostRequests(c, authzViaPath)
 153 }
 154 func authzViaPath(r *http.Request, tok string) int {
 155         r.URL.Path = "/t=" + tok + r.URL.Path
 156         return http.StatusNotFound
 157 }
 158
 159 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
 160         s.doVhostRequests(c, authzViaQueryString)
 161 }
 162 func authzViaQueryString(r *http.Request, tok string) int {
 163         r.URL.RawQuery = "api_token=" + tok
 164         return http.StatusUnauthorized
 165 }
 166
 167 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
 168         s.doVhostRequests(c, authzViaPOST)
 169 }
 170 func authzViaPOST(r *http.Request, tok string) int {
 171         r.Method = "POST"
 172         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 173         r.Body = ioutil.NopCloser(strings.NewReader(
 174                 url.Values{"api_token": {tok}}.Encode()))
 175         return http.StatusUnauthorized
 176 }
 177
 178 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
 179         s.doVhostRequests(c, authzViaPOST)
 180 }
 181 func authzViaXHRPOST(r *http.Request, tok string) int {
 182         r.Method = "POST"
 183         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 184         r.Header.Add("Origin", "https://origin.example")
 185         r.Body = ioutil.NopCloser(strings.NewReader(
 186                 url.Values{
 187                         "api_token":   {tok},
 188                         "disposition": {"attachment"},
 189                 }.Encode()))
 190         return http.StatusUnauthorized
 191 }
 192
 193 // Try some combinations of {url, token} using the given authorization
 194 // mechanism, and verify the result is correct.
 195 func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
 196         for _, hostPath := range []string{
 197                 arvadostest.FooCollection + ".example.com/foo",
 198                 arvadostest.FooCollection + "--collections.example.com/foo",
 199                 arvadostest.FooCollection + "--collections.example.com/_/foo",
 200                 arvadostest.FooCollectionPDH + ".example.com/foo",
 201                 strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + "--collections.example.com/foo",
 202                 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
 203         } {
 204                 c.Log("doRequests: ", hostPath)
 205                 s.doVhostRequestsWithHostPath(c, authz, hostPath)
 206         }
 207 }
 208
 209 func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
 210         for _, tok := range []string{
 211                 arvadostest.ActiveToken,
 212                 arvadostest.ActiveToken[:15],
 213                 arvadostest.SpectatorToken,
 214                 "bogus",
 215                 "",
 216         } {
 217                 u := mustParseURL("http://" + hostPath)
 218                 req := &http.Request{
 219                         Method:     "GET",
 220                         Host:       u.Host,
 221                         URL:        u,
 222                         RequestURI: u.RequestURI(),
 223                         Header:     http.Header{},
 224                 }
 225                 failCode := authz(req, tok)
 226                 req, resp := s.doReq(req)
 227                 code, body := resp.Code, resp.Body.String()
 228
 229                 // If the initial request had a (non-empty) token
 230                 // showing in the query string, we should have been
 231                 // redirected in order to hide it in a cookie.
 232                 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
 233
 234                 if tok == arvadostest.ActiveToken {
 235                         c.Check(code, check.Equals, http.StatusOK)
 236                         c.Check(body, check.Equals, "foo")
 237
 238                 } else {
 239                         c.Check(code >= 400, check.Equals, true)
 240                         c.Check(code < 500, check.Equals, true)
 241                         if tok == arvadostest.SpectatorToken {
 242                                 // Valid token never offers to retry
 243                                 // with different credentials.
 244                                 c.Check(code, check.Equals, http.StatusNotFound)
 245                         } else {
 246                                 // Invalid token can ask to retry
 247                                 // depending on the authz method.
 248                                 c.Check(code, check.Equals, failCode)
 249                         }
 250                         c.Check(body, check.Equals, "")
 251                 }
 252         }
 253 }
 254
 255 func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
 256         resp := httptest.NewRecorder()
 257         s.testServer.Handler.ServeHTTP(resp, req)
 258         if resp.Code != http.StatusSeeOther {
 259                 return req, resp
 260         }
 261         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 262         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 263         req = &http.Request{
 264                 Method:     "GET",
 265                 Host:       u.Host,
 266                 URL:        u,
 267                 RequestURI: u.RequestURI(),
 268                 Header:     http.Header{},
 269         }
 270         for _, c := range cookies {
 271                 req.AddCookie(c)
 272         }
 273         return s.doReq(req)
 274 }
 275
 276 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
 277         s.testVhostRedirectTokenToCookie(c, "GET",
 278                 arvadostest.FooCollection+".example.com/foo",
 279                 "?api_token="+arvadostest.ActiveToken,
 280                 "",
 281                 "",
 282                 http.StatusOK,
 283                 "foo",
 284         )
 285 }
 286
 287 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
 288         s.testVhostRedirectTokenToCookie(c, "GET",
 289                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
 290                 "",
 291                 "",
 292                 "",
 293                 http.StatusOK,
 294                 "foo",
 295         )
 296 }
 297
 298 // Bad token in URL is 404 Not Found because it doesn't make sense to
 299 // retry the same URL with different authorization.
 300 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
 301         s.testVhostRedirectTokenToCookie(c, "GET",
 302                 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
 303                 "",
 304                 "",
 305                 "",
 306                 http.StatusNotFound,
 307                 "",
 308         )
 309 }
 310
 311 // Bad token in a cookie (even if it got there via our own
 312 // query-string-to-cookie redirect) is, in principle, retryable at the
 313 // same URL so it's 401 Unauthorized.
 314 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
 315         s.testVhostRedirectTokenToCookie(c, "GET",
 316                 arvadostest.FooCollection+".example.com/foo",
 317                 "?api_token=thisisabogustoken",
 318                 "",
 319                 "",
 320                 http.StatusUnauthorized,
 321                 "",
 322         )
 323 }
 324
 325 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
 326         s.testVhostRedirectTokenToCookie(c, "GET",
 327                 "example.com/c="+arvadostest.FooCollection+"/foo",
 328                 "?api_token="+arvadostest.ActiveToken,
 329                 "",
 330                 "",
 331                 http.StatusBadRequest,
 332                 "",
 333         )
 334 }
 335
 336 // If client requests an attachment by putting ?disposition=attachment
 337 // in the query string, and gets redirected, the redirect target
 338 // should respond with an attachment.
 339 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
 340         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 341                 arvadostest.FooCollection+".example.com/foo",
 342                 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
 343                 "",
 344                 "",
 345                 http.StatusOK,
 346                 "foo",
 347         )
 348         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 349 }
 350
 351 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
 352         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 353         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 354                 "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
 355                 "?api_token="+arvadostest.ActiveToken,
 356                 "",
 357                 "",
 358                 http.StatusOK,
 359                 "foo",
 360         )
 361         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 362 }
 363
 364 func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
 365         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 366         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 367                 "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
 368                 "?api_token="+arvadostest.ActiveToken,
 369                 "",
 370                 "",
 371                 http.StatusOK,
 372                 "waz",
 373         )
 374         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 375         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 376                 "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz",
 377                 "?api_token="+arvadostest.ActiveToken,
 378                 "",
 379                 "",
 380                 http.StatusOK,
 381                 "waz",
 382         )
 383         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 384 }
 385
 386 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
 387         s.testServer.Config.cluster.Collections.TrustAllContent = true
 388         s.testVhostRedirectTokenToCookie(c, "GET",
 389                 "example.com/c="+arvadostest.FooCollection+"/foo",
 390                 "?api_token="+arvadostest.ActiveToken,
 391                 "",
 392                 "",
 393                 http.StatusOK,
 394                 "foo",
 395         )
 396 }
 397
 398 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
 399         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
 400
 401         s.testVhostRedirectTokenToCookie(c, "GET",
 402                 "example.com/c="+arvadostest.FooCollection+"/foo",
 403                 "?api_token="+arvadostest.ActiveToken,
 404                 "",
 405                 "",
 406                 http.StatusBadRequest,
 407                 "",
 408         )
 409
 410         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 411                 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
 412                 "?api_token="+arvadostest.ActiveToken,
 413                 "",
 414                 "",
 415                 http.StatusOK,
 416                 "foo",
 417         )
 418         c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
 419 }
 420
 421 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
 422         s.testVhostRedirectTokenToCookie(c, "POST",
 423                 arvadostest.FooCollection+".example.com/foo",
 424                 "",
 425                 "application/x-www-form-urlencoded",
 426                 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
 427                 http.StatusOK,
 428                 "foo",
 429         )
 430 }
 431
 432 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
 433         s.testVhostRedirectTokenToCookie(c, "POST",
 434                 arvadostest.FooCollection+".example.com/foo",
 435                 "",
 436                 "application/x-www-form-urlencoded",
 437                 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
 438                 http.StatusNotFound,
 439                 "",
 440         )
 441 }
 442
 443 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
 444         s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 445         s.testVhostRedirectTokenToCookie(c, "GET",
 446                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 447                 "",
 448                 "",
 449                 "",
 450                 http.StatusOK,
 451                 "Hello world\n",
 452         )
 453 }
 454
 455 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
 456         s.testServer.Config.cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
 457         s.testVhostRedirectTokenToCookie(c, "GET",
 458                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 459                 "",
 460                 "",
 461                 "",
 462                 http.StatusNotFound,
 463                 "",
 464         )
 465 }
 466
 467 func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
 468         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 469
 470         client := s.testServer.Config.Client
 471         client.AuthToken = arvadostest.ActiveToken
 472         fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
 473         c.Assert(err, check.IsNil)
 474         f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777)
 475         c.Assert(err, check.IsNil)
 476         f.Close()
 477         mtxt, err := fs.MarshalManifest(".")
 478         c.Assert(err, check.IsNil)
 479         var coll arvados.Collection
 480         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 481                 "collection": map[string]string{
 482                         "manifest_text": mtxt,
 483                 },
 484         })
 485         c.Assert(err, check.IsNil)
 486
 487         u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
 488         req := &http.Request{
 489                 Method:     "GET",
 490                 Host:       u.Host,
 491                 URL:        u,
 492                 RequestURI: u.RequestURI(),
 493                 Header: http.Header{
 494                         "Authorization": {"Bearer " + client.AuthToken},
 495                 },
 496         }
 497         resp := httptest.NewRecorder()
 498         s.testServer.Handler.ServeHTTP(resp, req)
 499         c.Check(resp.Code, check.Equals, http.StatusOK)
 500         c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\&#34;odd&#39; path chars.*`)
 501 }
 502
 503 // XHRs can't follow redirect-with-cookie so they rely on method=POST
 504 // and disposition=attachment (telling us it's acceptable to respond
 505 // with content instead of a redirect) and an Origin header that gets
 506 // added automatically by the browser (telling us it's desirable to do
 507 // so).
 508 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
 509         u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
 510         req := &http.Request{
 511                 Method:     "POST",
 512                 Host:       u.Host,
 513                 URL:        u,
 514                 RequestURI: u.RequestURI(),
 515                 Header: http.Header{
 516                         "Origin":       {"https://origin.example"},
 517                         "Content-Type": {"application/x-www-form-urlencoded"},
 518                 },
 519                 Body: ioutil.NopCloser(strings.NewReader(url.Values{
 520                         "api_token":   {arvadostest.ActiveToken},
 521                         "disposition": {"attachment"},
 522                 }.Encode())),
 523         }
 524         resp := httptest.NewRecorder()
 525         s.testServer.Handler.ServeHTTP(resp, req)
 526         c.Check(resp.Code, check.Equals, http.StatusOK)
 527         c.Check(resp.Body.String(), check.Equals, "foo")
 528         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
 529 }
 530
 531 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder {
 532         u, _ := url.Parse(`http://` + hostPath + queryString)
 533         req := &http.Request{
 534                 Method:     method,
 535                 Host:       u.Host,
 536                 URL:        u,
 537                 RequestURI: u.RequestURI(),
 538                 Header:     http.Header{"Content-Type": {contentType}},
 539                 Body:       ioutil.NopCloser(strings.NewReader(reqBody)),
 540         }
 541
 542         resp := httptest.NewRecorder()
 543         defer func() {
 544                 c.Check(resp.Code, check.Equals, expectStatus)
 545                 c.Check(resp.Body.String(), check.Equals, expectRespBody)
 546         }()
 547
 548         s.testServer.Handler.ServeHTTP(resp, req)
 549         if resp.Code != http.StatusSeeOther {
 550                 return resp
 551         }
 552         c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
 553         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 554
 555         u, _ = u.Parse(resp.Header().Get("Location"))
 556         req = &http.Request{
 557                 Method:     "GET",
 558                 Host:       u.Host,
 559                 URL:        u,
 560                 RequestURI: u.RequestURI(),
 561                 Header:     http.Header{},
 562         }
 563         for _, c := range cookies {
 564                 req.AddCookie(c)
 565         }
 566
 567         resp = httptest.NewRecorder()
 568         s.testServer.Handler.ServeHTTP(resp, req)
 569         c.Check(resp.Header().Get("Location"), check.Equals, "")
 570         return resp
 571 }
 572
 573 func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) {
 574         s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 575         s.testDirectoryListing(c)
 576 }
 577
 578 func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) {
 579         s.testServer.Config.cluster.Users.AnonymousUserToken = ""
 580         s.testDirectoryListing(c)
 581 }
 582
 583 func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
 584         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 585         authHeader := http.Header{
 586                 "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
 587         }
 588         for _, trial := range []struct {
 589                 uri      string
 590                 header   http.Header
 591                 expect   []string
 592                 redirect string
 593                 cutDirs  int
 594         }{
 595                 {
 596                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
 597                         header:  authHeader,
 598                         expect:  []string{"dir1/foo", "dir1/bar"},
 599                         cutDirs: 0,
 600                 },
 601                 {
 602                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
 603                         header:  authHeader,
 604                         expect:  []string{"foo", "bar"},
 605                         cutDirs: 1,
 606                 },
 607                 {
 608                         // URLs of this form ignore authHeader, and
 609                         // FooAndBarFilesInDirUUID isn't public, so
 610                         // this returns 404.
 611                         uri:    "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
 612                         header: authHeader,
 613                         expect: nil,
 614                 },
 615                 {
 616                         uri:     "download.example.com/users/active/foo_file_in_dir/",
 617                         header:  authHeader,
 618                         expect:  []string{"dir1/"},
 619                         cutDirs: 3,
 620                 },
 621                 {
 622                         uri:     "download.example.com/users/active/foo_file_in_dir/dir1/",
 623                         header:  authHeader,
 624                         expect:  []string{"bar"},
 625                         cutDirs: 4,
 626                 },
 627                 {
 628                         uri:     "download.example.com/",
 629                         header:  authHeader,
 630                         expect:  []string{"users/"},
 631                         cutDirs: 0,
 632                 },
 633                 {
 634                         uri:      "download.example.com/users",
 635                         header:   authHeader,
 636                         redirect: "/users/",
 637                         expect:   []string{"active/"},
 638                         cutDirs:  1,
 639                 },
 640                 {
 641                         uri:     "download.example.com/users/",
 642                         header:  authHeader,
 643                         expect:  []string{"active/"},
 644                         cutDirs: 1,
 645                 },
 646                 {
 647                         uri:      "download.example.com/users/active",
 648                         header:   authHeader,
 649                         redirect: "/users/active/",
 650                         expect:   []string{"foo_file_in_dir/"},
 651                         cutDirs:  2,
 652                 },
 653                 {
 654                         uri:     "download.example.com/users/active/",
 655                         header:  authHeader,
 656                         expect:  []string{"foo_file_in_dir/"},
 657                         cutDirs: 2,
 658                 },
 659                 {
 660                         uri:     "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
 661                         header:  nil,
 662                         expect:  []string{"dir1/foo", "dir1/bar"},
 663                         cutDirs: 4,
 664                 },
 665                 {
 666                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
 667                         header:  nil,
 668                         expect:  []string{"dir1/foo", "dir1/bar"},
 669                         cutDirs: 2,
 670                 },
 671                 {
 672                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken,
 673                         header:  nil,
 674                         expect:  []string{"dir1/foo", "dir1/bar"},
 675                         cutDirs: 2,
 676                 },
 677                 {
 678                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID,
 679                         header:  authHeader,
 680                         expect:  []string{"dir1/foo", "dir1/bar"},
 681                         cutDirs: 1,
 682                 },
 683                 {
 684                         uri:      "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
 685                         header:   authHeader,
 686                         redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
 687                         expect:   []string{"foo", "bar"},
 688                         cutDirs:  2,
 689                 },
 690                 {
 691                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
 692                         header:  authHeader,
 693                         expect:  []string{"foo", "bar"},
 694                         cutDirs: 3,
 695                 },
 696                 {
 697                         uri:      arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
 698                         header:   authHeader,
 699                         redirect: "/dir1/",
 700                         expect:   []string{"foo", "bar"},
 701                         cutDirs:  1,
 702                 },
 703                 {
 704                         uri:    "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
 705                         header: authHeader,
 706                         expect: nil,
 707                 },
 708                 {
 709                         uri:     "download.example.com/c=" + arvadostest.WazVersion1Collection,
 710                         header:  authHeader,
 711                         expect:  []string{"waz"},
 712                         cutDirs: 1,
 713                 },
 714                 {
 715                         uri:     "download.example.com/by_id/" + arvadostest.WazVersion1Collection,
 716                         header:  authHeader,
 717                         expect:  []string{"waz"},
 718                         cutDirs: 2,
 719                 },
 720         } {
 721                 comment := check.Commentf("HTML: %q => %q", trial.uri, trial.expect)
 722                 resp := httptest.NewRecorder()
 723                 u := mustParseURL("//" + trial.uri)
 724                 req := &http.Request{
 725                         Method:     "GET",
 726                         Host:       u.Host,
 727                         URL:        u,
 728                         RequestURI: u.RequestURI(),
 729                         Header:     copyHeader(trial.header),
 730                 }
 731                 s.testServer.Handler.ServeHTTP(resp, req)
 732                 var cookies []*http.Cookie
 733                 for resp.Code == http.StatusSeeOther {
 734                         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 735                         req = &http.Request{
 736                                 Method:     "GET",
 737                                 Host:       u.Host,
 738                                 URL:        u,
 739                                 RequestURI: u.RequestURI(),
 740                                 Header:     copyHeader(trial.header),
 741                         }
 742                         cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
 743                         for _, c := range cookies {
 744                                 req.AddCookie(c)
 745                         }
 746                         resp = httptest.NewRecorder()
 747                         s.testServer.Handler.ServeHTTP(resp, req)
 748                 }
 749                 if trial.redirect != "" {
 750                         c.Check(req.URL.Path, check.Equals, trial.redirect, comment)
 751                 }
 752                 if trial.expect == nil {
 753                         c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
 754                 } else {
 755                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
 756                         for _, e := range trial.expect {
 757                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`, comment)
 758                         }
 759                         c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`, comment)
 760                 }
 761
 762                 comment = check.Commentf("WebDAV: %q => %q", trial.uri, trial.expect)
 763                 req = &http.Request{
 764                         Method:     "OPTIONS",
 765                         Host:       u.Host,
 766                         URL:        u,
 767                         RequestURI: u.RequestURI(),
 768                         Header:     copyHeader(trial.header),
 769                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
 770                 }
 771                 resp = httptest.NewRecorder()
 772                 s.testServer.Handler.ServeHTTP(resp, req)
 773                 if trial.expect == nil {
 774                         c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
 775                 } else {
 776                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
 777                 }
 778
 779                 req = &http.Request{
 780                         Method:     "PROPFIND",
 781                         Host:       u.Host,
 782                         URL:        u,
 783                         RequestURI: u.RequestURI(),
 784                         Header:     copyHeader(trial.header),
 785                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
 786                 }
 787                 resp = httptest.NewRecorder()
 788                 s.testServer.Handler.ServeHTTP(resp, req)
 789                 if trial.expect == nil {
 790                         c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
 791                 } else {
 792                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment)
 793                         for _, e := range trial.expect {
 794                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+filepath.Join(u.Path, e)+`</D:href>.*`, comment)
 795                         }
 796                 }
 797         }
 798 }
 799
 800 func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
 801         arv := arvados.NewClientFromEnv()
 802         var newCollection arvados.Collection
 803         err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 804                 "collection": map[string]string{
 805                         "owner_uuid":    arvadostest.ActiveUserUUID,
 806                         "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
 807                         "name":          "keep-web test collection",
 808                 },
 809                 "ensure_unique_name": true,
 810         })
 811         c.Assert(err, check.IsNil)
 812         defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
 813
 814         var updated arvados.Collection
 815         for _, fnm := range []string{"foo.txt", "bar.txt"} {
 816                 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
 817                 u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
 818                 req := &http.Request{
 819                         Method:     "DELETE",
 820                         Host:       u.Host,
 821                         URL:        u,
 822                         RequestURI: u.RequestURI(),
 823                         Header: http.Header{
 824                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
 825                         },
 826                 }
 827                 resp := httptest.NewRecorder()
 828                 s.testServer.Handler.ServeHTTP(resp, req)
 829                 c.Check(resp.Code, check.Equals, http.StatusNoContent)
 830
 831                 updated = arvados.Collection{}
 832                 err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
 833                 c.Check(err, check.IsNil)
 834                 c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`)
 835                 c.Logf("updated manifest_text %q", updated.ManifestText)
 836         }
 837         c.Check(updated.ManifestText, check.Equals, "")
 838 }
 839
 840 func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) {
 841         s.testServer.Config.cluster.ManagementToken = arvadostest.ManagementToken
 842         authHeader := http.Header{
 843                 "Authorization": {"Bearer " + arvadostest.ManagementToken},
 844         }
 845
 846         resp := httptest.NewRecorder()
 847         u := mustParseURL("http://download.example.com/_health/ping")
 848         req := &http.Request{
 849                 Method:     "GET",
 850                 Host:       u.Host,
 851                 URL:        u,
 852                 RequestURI: u.RequestURI(),
 853                 Header:     authHeader,
 854         }
 855         s.testServer.Handler.ServeHTTP(resp, req)
 856
 857         c.Check(resp.Code, check.Equals, http.StatusOK)
 858         c.Check(resp.Body.String(), check.Matches, `{"health":"OK"}\n`)
 859 }
 860
 861 func copyHeader(h http.Header) http.Header {
 862         hc := http.Header{}
 863         for k, v := range h {
 864                 hc[k] = append([]string(nil), v...)
 865         }
 866         return hc
 867 }