services/keep-web/handler_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "fmt"
  10         "html"
  11         "io/ioutil"
  12         "net/http"
  13         "net/http/httptest"
  14         "net/url"
  15         "os"
  16         "path/filepath"
  17         "regexp"
  18         "strings"
  19
  20         "git.curoverse.com/arvados.git/lib/config"
  21         "git.curoverse.com/arvados.git/sdk/go/arvados"
  22         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  23         "git.curoverse.com/arvados.git/sdk/go/auth"
  24         check "gopkg.in/check.v1"
  25 )
  26
  27 var _ = check.Suite(&UnitSuite{})
  28
  29 type UnitSuite struct {
  30         Config *arvados.Config
  31 }
  32
  33 func (s *UnitSuite) SetUpTest(c *check.C) {
  34         ldr := config.NewLoader(nil, nil)
  35         cfg, err := ldr.LoadDefaults()
  36         c.Assert(err, check.IsNil)
  37         s.Config = cfg
  38 }
  39
  40 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
  41         h := handler{Config: DefaultConfig(s.Config)}
  42         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
  43         req := &http.Request{
  44                 Method:     "OPTIONS",
  45                 Host:       u.Host,
  46                 URL:        u,
  47                 RequestURI: u.RequestURI(),
  48                 Header: http.Header{
  49                         "Origin":                        {"https://workbench.example"},
  50                         "Access-Control-Request-Method": {"POST"},
  51                 },
  52         }
  53
  54         // Check preflight for an allowed request
  55         resp := httptest.NewRecorder()
  56         h.ServeHTTP(resp, req)
  57         c.Check(resp.Code, check.Equals, http.StatusOK)
  58         c.Check(resp.Body.String(), check.Equals, "")
  59         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
  60         c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
  61         c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout")
  62
  63         // Check preflight for a disallowed request
  64         resp = httptest.NewRecorder()
  65         req.Header.Set("Access-Control-Request-Method", "MAKE-COFFEE")
  66         h.ServeHTTP(resp, req)
  67         c.Check(resp.Body.String(), check.Equals, "")
  68         c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
  69 }
  70
  71 func (s *UnitSuite) TestInvalidUUID(c *check.C) {
  72         bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-"
  73         token := arvadostest.ActiveToken
  74         for _, trial := range []string{
  75                 "http://keep-web/c=" + bogusID + "/foo",
  76                 "http://keep-web/c=" + bogusID + "/t=" + token + "/foo",
  77                 "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo",
  78                 "http://keep-web/collections/" + bogusID + "/foo",
  79                 "http://" + bogusID + ".keep-web/" + bogusID + "/foo",
  80                 "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
  81         } {
  82                 c.Log(trial)
  83                 u := mustParseURL(trial)
  84                 req := &http.Request{
  85                         Method:     "GET",
  86                         Host:       u.Host,
  87                         URL:        u,
  88                         RequestURI: u.RequestURI(),
  89                 }
  90                 resp := httptest.NewRecorder()
  91                 cfg := DefaultConfig(s.Config)
  92                 cfg.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
  93                 h := handler{Config: cfg}
  94                 h.ServeHTTP(resp, req)
  95                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
  96         }
  97 }
  98
  99 func mustParseURL(s string) *url.URL {
 100         r, err := url.Parse(s)
 101         if err != nil {
 102                 panic("parse URL: " + s)
 103         }
 104         return r
 105 }
 106
 107 func (s *IntegrationSuite) TestVhost404(c *check.C) {
 108         for _, testURL := range []string{
 109                 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
 110                 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
 111         } {
 112                 resp := httptest.NewRecorder()
 113                 u := mustParseURL(testURL)
 114                 req := &http.Request{
 115                         Method:     "GET",
 116                         URL:        u,
 117                         RequestURI: u.RequestURI(),
 118                 }
 119                 s.testServer.Handler.ServeHTTP(resp, req)
 120                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 121                 c.Check(resp.Body.String(), check.Equals, "")
 122         }
 123 }
 124
 125 // An authorizer modifies an HTTP request to make use of the given
 126 // token -- by adding it to a header, cookie, query param, or whatever
 127 // -- and returns the HTTP status code we should expect from keep-web if
 128 // the token is invalid.
 129 type authorizer func(*http.Request, string) int
 130
 131 func (s *IntegrationSuite) TestVhostViaAuthzHeader(c *check.C) {
 132         s.doVhostRequests(c, authzViaAuthzHeader)
 133 }
 134 func authzViaAuthzHeader(r *http.Request, tok string) int {
 135         r.Header.Add("Authorization", "OAuth2 "+tok)
 136         return http.StatusUnauthorized
 137 }
 138
 139 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
 140         s.doVhostRequests(c, authzViaCookieValue)
 141 }
 142 func authzViaCookieValue(r *http.Request, tok string) int {
 143         r.AddCookie(&http.Cookie{
 144                 Name:  "arvados_api_token",
 145                 Value: auth.EncodeTokenCookie([]byte(tok)),
 146         })
 147         return http.StatusUnauthorized
 148 }
 149
 150 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
 151         s.doVhostRequests(c, authzViaPath)
 152 }
 153 func authzViaPath(r *http.Request, tok string) int {
 154         r.URL.Path = "/t=" + tok + r.URL.Path
 155         return http.StatusNotFound
 156 }
 157
 158 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
 159         s.doVhostRequests(c, authzViaQueryString)
 160 }
 161 func authzViaQueryString(r *http.Request, tok string) int {
 162         r.URL.RawQuery = "api_token=" + tok
 163         return http.StatusUnauthorized
 164 }
 165
 166 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
 167         s.doVhostRequests(c, authzViaPOST)
 168 }
 169 func authzViaPOST(r *http.Request, tok string) int {
 170         r.Method = "POST"
 171         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 172         r.Body = ioutil.NopCloser(strings.NewReader(
 173                 url.Values{"api_token": {tok}}.Encode()))
 174         return http.StatusUnauthorized
 175 }
 176
 177 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
 178         s.doVhostRequests(c, authzViaPOST)
 179 }
 180 func authzViaXHRPOST(r *http.Request, tok string) int {
 181         r.Method = "POST"
 182         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 183         r.Header.Add("Origin", "https://origin.example")
 184         r.Body = ioutil.NopCloser(strings.NewReader(
 185                 url.Values{
 186                         "api_token":   {tok},
 187                         "disposition": {"attachment"},
 188                 }.Encode()))
 189         return http.StatusUnauthorized
 190 }
 191
 192 // Try some combinations of {url, token} using the given authorization
 193 // mechanism, and verify the result is correct.
 194 func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
 195         for _, hostPath := range []string{
 196                 arvadostest.FooCollection + ".example.com/foo",
 197                 arvadostest.FooCollection + "--collections.example.com/foo",
 198                 arvadostest.FooCollection + "--collections.example.com/_/foo",
 199                 arvadostest.FooCollectionPDH + ".example.com/foo",
 200                 strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + "--collections.example.com/foo",
 201                 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
 202         } {
 203                 c.Log("doRequests: ", hostPath)
 204                 s.doVhostRequestsWithHostPath(c, authz, hostPath)
 205         }
 206 }
 207
 208 func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
 209         for _, tok := range []string{
 210                 arvadostest.ActiveToken,
 211                 arvadostest.ActiveToken[:15],
 212                 arvadostest.SpectatorToken,
 213                 "bogus",
 214                 "",
 215         } {
 216                 u := mustParseURL("http://" + hostPath)
 217                 req := &http.Request{
 218                         Method:     "GET",
 219                         Host:       u.Host,
 220                         URL:        u,
 221                         RequestURI: u.RequestURI(),
 222                         Header:     http.Header{},
 223                 }
 224                 failCode := authz(req, tok)
 225                 req, resp := s.doReq(req)
 226                 code, body := resp.Code, resp.Body.String()
 227
 228                 // If the initial request had a (non-empty) token
 229                 // showing in the query string, we should have been
 230                 // redirected in order to hide it in a cookie.
 231                 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
 232
 233                 if tok == arvadostest.ActiveToken {
 234                         c.Check(code, check.Equals, http.StatusOK)
 235                         c.Check(body, check.Equals, "foo")
 236
 237                 } else {
 238                         c.Check(code >= 400, check.Equals, true)
 239                         c.Check(code < 500, check.Equals, true)
 240                         if tok == arvadostest.SpectatorToken {
 241                                 // Valid token never offers to retry
 242                                 // with different credentials.
 243                                 c.Check(code, check.Equals, http.StatusNotFound)
 244                         } else {
 245                                 // Invalid token can ask to retry
 246                                 // depending on the authz method.
 247                                 c.Check(code, check.Equals, failCode)
 248                         }
 249                         c.Check(body, check.Equals, "")
 250                 }
 251         }
 252 }
 253
 254 func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
 255         resp := httptest.NewRecorder()
 256         s.testServer.Handler.ServeHTTP(resp, req)
 257         if resp.Code != http.StatusSeeOther {
 258                 return req, resp
 259         }
 260         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 261         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 262         req = &http.Request{
 263                 Method:     "GET",
 264                 Host:       u.Host,
 265                 URL:        u,
 266                 RequestURI: u.RequestURI(),
 267                 Header:     http.Header{},
 268         }
 269         for _, c := range cookies {
 270                 req.AddCookie(c)
 271         }
 272         return s.doReq(req)
 273 }
 274
 275 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
 276         s.testVhostRedirectTokenToCookie(c, "GET",
 277                 arvadostest.FooCollection+".example.com/foo",
 278                 "?api_token="+arvadostest.ActiveToken,
 279                 "",
 280                 "",
 281                 http.StatusOK,
 282                 "foo",
 283         )
 284 }
 285
 286 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
 287         s.testVhostRedirectTokenToCookie(c, "GET",
 288                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
 289                 "",
 290                 "",
 291                 "",
 292                 http.StatusOK,
 293                 "foo",
 294         )
 295 }
 296
 297 // Bad token in URL is 404 Not Found because it doesn't make sense to
 298 // retry the same URL with different authorization.
 299 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
 300         s.testVhostRedirectTokenToCookie(c, "GET",
 301                 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
 302                 "",
 303                 "",
 304                 "",
 305                 http.StatusNotFound,
 306                 "",
 307         )
 308 }
 309
 310 // Bad token in a cookie (even if it got there via our own
 311 // query-string-to-cookie redirect) is, in principle, retryable at the
 312 // same URL so it's 401 Unauthorized.
 313 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
 314         s.testVhostRedirectTokenToCookie(c, "GET",
 315                 arvadostest.FooCollection+".example.com/foo",
 316                 "?api_token=thisisabogustoken",
 317                 "",
 318                 "",
 319                 http.StatusUnauthorized,
 320                 "",
 321         )
 322 }
 323
 324 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
 325         s.testVhostRedirectTokenToCookie(c, "GET",
 326                 "example.com/c="+arvadostest.FooCollection+"/foo",
 327                 "?api_token="+arvadostest.ActiveToken,
 328                 "",
 329                 "",
 330                 http.StatusBadRequest,
 331                 "",
 332         )
 333 }
 334
 335 // If client requests an attachment by putting ?disposition=attachment
 336 // in the query string, and gets redirected, the redirect target
 337 // should respond with an attachment.
 338 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
 339         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 340                 arvadostest.FooCollection+".example.com/foo",
 341                 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
 342                 "",
 343                 "",
 344                 http.StatusOK,
 345                 "foo",
 346         )
 347         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 348 }
 349
 350 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
 351         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 352         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 353                 "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
 354                 "?api_token="+arvadostest.ActiveToken,
 355                 "",
 356                 "",
 357                 http.StatusOK,
 358                 "foo",
 359         )
 360         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 361 }
 362
 363 func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
 364         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 365         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 366                 "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
 367                 "?api_token="+arvadostest.ActiveToken,
 368                 "",
 369                 "",
 370                 http.StatusOK,
 371                 "waz",
 372         )
 373         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 374         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 375                 "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz",
 376                 "?api_token="+arvadostest.ActiveToken,
 377                 "",
 378                 "",
 379                 http.StatusOK,
 380                 "waz",
 381         )
 382         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 383 }
 384
 385 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
 386         s.testServer.Config.cluster.Collections.TrustAllContent = true
 387         s.testVhostRedirectTokenToCookie(c, "GET",
 388                 "example.com/c="+arvadostest.FooCollection+"/foo",
 389                 "?api_token="+arvadostest.ActiveToken,
 390                 "",
 391                 "",
 392                 http.StatusOK,
 393                 "foo",
 394         )
 395 }
 396
 397 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
 398         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
 399
 400         s.testVhostRedirectTokenToCookie(c, "GET",
 401                 "example.com/c="+arvadostest.FooCollection+"/foo",
 402                 "?api_token="+arvadostest.ActiveToken,
 403                 "",
 404                 "",
 405                 http.StatusBadRequest,
 406                 "",
 407         )
 408
 409         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 410                 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
 411                 "?api_token="+arvadostest.ActiveToken,
 412                 "",
 413                 "",
 414                 http.StatusOK,
 415                 "foo",
 416         )
 417         c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
 418 }
 419
 420 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
 421         s.testVhostRedirectTokenToCookie(c, "POST",
 422                 arvadostest.FooCollection+".example.com/foo",
 423                 "",
 424                 "application/x-www-form-urlencoded",
 425                 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
 426                 http.StatusOK,
 427                 "foo",
 428         )
 429 }
 430
 431 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
 432         s.testVhostRedirectTokenToCookie(c, "POST",
 433                 arvadostest.FooCollection+".example.com/foo",
 434                 "",
 435                 "application/x-www-form-urlencoded",
 436                 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
 437                 http.StatusNotFound,
 438                 "",
 439         )
 440 }
 441
 442 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
 443         s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 444         s.testVhostRedirectTokenToCookie(c, "GET",
 445                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 446                 "",
 447                 "",
 448                 "",
 449                 http.StatusOK,
 450                 "Hello world\n",
 451         )
 452 }
 453
 454 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
 455         s.testServer.Config.cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
 456         s.testVhostRedirectTokenToCookie(c, "GET",
 457                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 458                 "",
 459                 "",
 460                 "",
 461                 http.StatusNotFound,
 462                 "",
 463         )
 464 }
 465
 466 func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
 467         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 468
 469         client := s.testServer.Config.Client
 470         client.AuthToken = arvadostest.ActiveToken
 471         fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
 472         c.Assert(err, check.IsNil)
 473         f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777)
 474         c.Assert(err, check.IsNil)
 475         f.Close()
 476         mtxt, err := fs.MarshalManifest(".")
 477         c.Assert(err, check.IsNil)
 478         var coll arvados.Collection
 479         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 480                 "collection": map[string]string{
 481                         "manifest_text": mtxt,
 482                 },
 483         })
 484         c.Assert(err, check.IsNil)
 485
 486         u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
 487         req := &http.Request{
 488                 Method:     "GET",
 489                 Host:       u.Host,
 490                 URL:        u,
 491                 RequestURI: u.RequestURI(),
 492                 Header: http.Header{
 493                         "Authorization": {"Bearer " + client.AuthToken},
 494                 },
 495         }
 496         resp := httptest.NewRecorder()
 497         s.testServer.Handler.ServeHTTP(resp, req)
 498         c.Check(resp.Code, check.Equals, http.StatusOK)
 499         c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\&#34;odd&#39; path chars.*`)
 500 }
 501
 502 // XHRs can't follow redirect-with-cookie so they rely on method=POST
 503 // and disposition=attachment (telling us it's acceptable to respond
 504 // with content instead of a redirect) and an Origin header that gets
 505 // added automatically by the browser (telling us it's desirable to do
 506 // so).
 507 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
 508         u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
 509         req := &http.Request{
 510                 Method:     "POST",
 511                 Host:       u.Host,
 512                 URL:        u,
 513                 RequestURI: u.RequestURI(),
 514                 Header: http.Header{
 515                         "Origin":       {"https://origin.example"},
 516                         "Content-Type": {"application/x-www-form-urlencoded"},
 517                 },
 518                 Body: ioutil.NopCloser(strings.NewReader(url.Values{
 519                         "api_token":   {arvadostest.ActiveToken},
 520                         "disposition": {"attachment"},
 521                 }.Encode())),
 522         }
 523         resp := httptest.NewRecorder()
 524         s.testServer.Handler.ServeHTTP(resp, req)
 525         c.Check(resp.Code, check.Equals, http.StatusOK)
 526         c.Check(resp.Body.String(), check.Equals, "foo")
 527         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
 528 }
 529
 530 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder {
 531         u, _ := url.Parse(`http://` + hostPath + queryString)
 532         req := &http.Request{
 533                 Method:     method,
 534                 Host:       u.Host,
 535                 URL:        u,
 536                 RequestURI: u.RequestURI(),
 537                 Header:     http.Header{"Content-Type": {contentType}},
 538                 Body:       ioutil.NopCloser(strings.NewReader(reqBody)),
 539         }
 540
 541         resp := httptest.NewRecorder()
 542         defer func() {
 543                 c.Check(resp.Code, check.Equals, expectStatus)
 544                 c.Check(resp.Body.String(), check.Equals, expectRespBody)
 545         }()
 546
 547         s.testServer.Handler.ServeHTTP(resp, req)
 548         if resp.Code != http.StatusSeeOther {
 549                 return resp
 550         }
 551         c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
 552         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 553
 554         u, _ = u.Parse(resp.Header().Get("Location"))
 555         req = &http.Request{
 556                 Method:     "GET",
 557                 Host:       u.Host,
 558                 URL:        u,
 559                 RequestURI: u.RequestURI(),
 560                 Header:     http.Header{},
 561         }
 562         for _, c := range cookies {
 563                 req.AddCookie(c)
 564         }
 565
 566         resp = httptest.NewRecorder()
 567         s.testServer.Handler.ServeHTTP(resp, req)
 568         c.Check(resp.Header().Get("Location"), check.Equals, "")
 569         return resp
 570 }
 571
 572 func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) {
 573         s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 574         s.testDirectoryListing(c)
 575 }
 576
 577 func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) {
 578         s.testServer.Config.cluster.Users.AnonymousUserToken = ""
 579         s.testDirectoryListing(c)
 580 }
 581
 582 func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
 583         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 584         authHeader := http.Header{
 585                 "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
 586         }
 587         for _, trial := range []struct {
 588                 uri      string
 589                 header   http.Header
 590                 expect   []string
 591                 redirect string
 592                 cutDirs  int
 593         }{
 594                 {
 595                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
 596                         header:  authHeader,
 597                         expect:  []string{"dir1/foo", "dir1/bar"},
 598                         cutDirs: 0,
 599                 },
 600                 {
 601                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
 602                         header:  authHeader,
 603                         expect:  []string{"foo", "bar"},
 604                         cutDirs: 1,
 605                 },
 606                 {
 607                         // URLs of this form ignore authHeader, and
 608                         // FooAndBarFilesInDirUUID isn't public, so
 609                         // this returns 404.
 610                         uri:    "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
 611                         header: authHeader,
 612                         expect: nil,
 613                 },
 614                 {
 615                         uri:     "download.example.com/users/active/foo_file_in_dir/",
 616                         header:  authHeader,
 617                         expect:  []string{"dir1/"},
 618                         cutDirs: 3,
 619                 },
 620                 {
 621                         uri:     "download.example.com/users/active/foo_file_in_dir/dir1/",
 622                         header:  authHeader,
 623                         expect:  []string{"bar"},
 624                         cutDirs: 4,
 625                 },
 626                 {
 627                         uri:     "download.example.com/",
 628                         header:  authHeader,
 629                         expect:  []string{"users/"},
 630                         cutDirs: 0,
 631                 },
 632                 {
 633                         uri:      "download.example.com/users",
 634                         header:   authHeader,
 635                         redirect: "/users/",
 636                         expect:   []string{"active/"},
 637                         cutDirs:  1,
 638                 },
 639                 {
 640                         uri:     "download.example.com/users/",
 641                         header:  authHeader,
 642                         expect:  []string{"active/"},
 643                         cutDirs: 1,
 644                 },
 645                 {
 646                         uri:      "download.example.com/users/active",
 647                         header:   authHeader,
 648                         redirect: "/users/active/",
 649                         expect:   []string{"foo_file_in_dir/"},
 650                         cutDirs:  2,
 651                 },
 652                 {
 653                         uri:     "download.example.com/users/active/",
 654                         header:  authHeader,
 655                         expect:  []string{"foo_file_in_dir/"},
 656                         cutDirs: 2,
 657                 },
 658                 {
 659                         uri:     "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
 660                         header:  nil,
 661                         expect:  []string{"dir1/foo", "dir1/bar"},
 662                         cutDirs: 4,
 663                 },
 664                 {
 665                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
 666                         header:  nil,
 667                         expect:  []string{"dir1/foo", "dir1/bar"},
 668                         cutDirs: 2,
 669                 },
 670                 {
 671                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken,
 672                         header:  nil,
 673                         expect:  []string{"dir1/foo", "dir1/bar"},
 674                         cutDirs: 2,
 675                 },
 676                 {
 677                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID,
 678                         header:  authHeader,
 679                         expect:  []string{"dir1/foo", "dir1/bar"},
 680                         cutDirs: 1,
 681                 },
 682                 {
 683                         uri:      "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
 684                         header:   authHeader,
 685                         redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
 686                         expect:   []string{"foo", "bar"},
 687                         cutDirs:  2,
 688                 },
 689                 {
 690                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
 691                         header:  authHeader,
 692                         expect:  []string{"foo", "bar"},
 693                         cutDirs: 3,
 694                 },
 695                 {
 696                         uri:      arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
 697                         header:   authHeader,
 698                         redirect: "/dir1/",
 699                         expect:   []string{"foo", "bar"},
 700                         cutDirs:  1,
 701                 },
 702                 {
 703                         uri:    "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
 704                         header: authHeader,
 705                         expect: nil,
 706                 },
 707                 {
 708                         uri:     "download.example.com/c=" + arvadostest.WazVersion1Collection,
 709                         header:  authHeader,
 710                         expect:  []string{"waz"},
 711                         cutDirs: 1,
 712                 },
 713                 {
 714                         uri:     "download.example.com/by_id/" + arvadostest.WazVersion1Collection,
 715                         header:  authHeader,
 716                         expect:  []string{"waz"},
 717                         cutDirs: 2,
 718                 },
 719         } {
 720                 comment := check.Commentf("HTML: %q => %q", trial.uri, trial.expect)
 721                 resp := httptest.NewRecorder()
 722                 u := mustParseURL("//" + trial.uri)
 723                 req := &http.Request{
 724                         Method:     "GET",
 725                         Host:       u.Host,
 726                         URL:        u,
 727                         RequestURI: u.RequestURI(),
 728                         Header:     copyHeader(trial.header),
 729                 }
 730                 s.testServer.Handler.ServeHTTP(resp, req)
 731                 var cookies []*http.Cookie
 732                 for resp.Code == http.StatusSeeOther {
 733                         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 734                         req = &http.Request{
 735                                 Method:     "GET",
 736                                 Host:       u.Host,
 737                                 URL:        u,
 738                                 RequestURI: u.RequestURI(),
 739                                 Header:     copyHeader(trial.header),
 740                         }
 741                         cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
 742                         for _, c := range cookies {
 743                                 req.AddCookie(c)
 744                         }
 745                         resp = httptest.NewRecorder()
 746                         s.testServer.Handler.ServeHTTP(resp, req)
 747                 }
 748                 if trial.redirect != "" {
 749                         c.Check(req.URL.Path, check.Equals, trial.redirect, comment)
 750                 }
 751                 if trial.expect == nil {
 752                         c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
 753                 } else {
 754                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
 755                         for _, e := range trial.expect {
 756                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`, comment)
 757                         }
 758                         c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`, comment)
 759                 }
 760
 761                 comment = check.Commentf("WebDAV: %q => %q", trial.uri, trial.expect)
 762                 req = &http.Request{
 763                         Method:     "OPTIONS",
 764                         Host:       u.Host,
 765                         URL:        u,
 766                         RequestURI: u.RequestURI(),
 767                         Header:     copyHeader(trial.header),
 768                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
 769                 }
 770                 resp = httptest.NewRecorder()
 771                 s.testServer.Handler.ServeHTTP(resp, req)
 772                 if trial.expect == nil {
 773                         c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
 774                 } else {
 775                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
 776                 }
 777
 778                 req = &http.Request{
 779                         Method:     "PROPFIND",
 780                         Host:       u.Host,
 781                         URL:        u,
 782                         RequestURI: u.RequestURI(),
 783                         Header:     copyHeader(trial.header),
 784                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
 785                 }
 786                 resp = httptest.NewRecorder()
 787                 s.testServer.Handler.ServeHTTP(resp, req)
 788                 if trial.expect == nil {
 789                         c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
 790                 } else {
 791                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment)
 792                         for _, e := range trial.expect {
 793                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+filepath.Join(u.Path, e)+`</D:href>.*`, comment)
 794                         }
 795                 }
 796         }
 797 }
 798
 799 func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
 800         arv := arvados.NewClientFromEnv()
 801         var newCollection arvados.Collection
 802         err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 803                 "collection": map[string]string{
 804                         "owner_uuid":    arvadostest.ActiveUserUUID,
 805                         "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
 806                         "name":          "keep-web test collection",
 807                 },
 808                 "ensure_unique_name": true,
 809         })
 810         c.Assert(err, check.IsNil)
 811         defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
 812
 813         var updated arvados.Collection
 814         for _, fnm := range []string{"foo.txt", "bar.txt"} {
 815                 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
 816                 u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
 817                 req := &http.Request{
 818                         Method:     "DELETE",
 819                         Host:       u.Host,
 820                         URL:        u,
 821                         RequestURI: u.RequestURI(),
 822                         Header: http.Header{
 823                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
 824                         },
 825                 }
 826                 resp := httptest.NewRecorder()
 827                 s.testServer.Handler.ServeHTTP(resp, req)
 828                 c.Check(resp.Code, check.Equals, http.StatusNoContent)
 829
 830                 updated = arvados.Collection{}
 831                 err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
 832                 c.Check(err, check.IsNil)
 833                 c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`)
 834                 c.Logf("updated manifest_text %q", updated.ManifestText)
 835         }
 836         c.Check(updated.ManifestText, check.Equals, "")
 837 }
 838
 839 func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) {
 840         s.testServer.Config.cluster.ManagementToken = arvadostest.ManagementToken
 841         authHeader := http.Header{
 842                 "Authorization": {"Bearer " + arvadostest.ManagementToken},
 843         }
 844
 845         resp := httptest.NewRecorder()
 846         u := mustParseURL("http://download.example.com/_health/ping")
 847         req := &http.Request{
 848                 Method:     "GET",
 849                 Host:       u.Host,
 850                 URL:        u,
 851                 RequestURI: u.RequestURI(),
 852                 Header:     authHeader,
 853         }
 854         s.testServer.Handler.ServeHTTP(resp, req)
 855
 856         c.Check(resp.Code, check.Equals, http.StatusOK)
 857         c.Check(resp.Body.String(), check.Matches, `{"health":"OK"}\n`)
 858 }
 859
 860 func copyHeader(h http.Header) http.Header {
 861         hc := http.Header{}
 862         for k, v := range h {
 863                 hc[k] = append([]string(nil), v...)
 864         }
 865         return hc
 866 }