services/keep-web/handler_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "fmt"
  10         "html"
  11         "io/ioutil"
  12         "net/http"
  13         "net/http/httptest"
  14         "net/url"
  15         "os"
  16         "path/filepath"
  17         "regexp"
  18         "strings"
  19
  20         "git.curoverse.com/arvados.git/lib/config"
  21         "git.curoverse.com/arvados.git/sdk/go/arvados"
  22         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  23         "git.curoverse.com/arvados.git/sdk/go/auth"
  24         "git.curoverse.com/arvados.git/sdk/go/keepclient"
  25         check "gopkg.in/check.v1"
  26 )
  27
  28 var _ = check.Suite(&UnitSuite{})
  29
  30 type UnitSuite struct {
  31         Config *arvados.Config
  32 }
  33
  34 func (s *UnitSuite) SetUpTest(c *check.C) {
  35         ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), nil)
  36         ldr.Path = "-"
  37         cfg, err := ldr.Load()
  38         c.Assert(err, check.IsNil)
  39         s.Config = cfg
  40 }
  41
  42 func (s *UnitSuite) TestKeepClientBlockCache(c *check.C) {
  43         cfg := newConfig(s.Config)
  44         cfg.cluster.Collections.WebDAVCache.MaxBlockEntries = 42
  45         h := handler{Config: cfg}
  46         c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Not(check.Equals), cfg.cluster.Collections.WebDAVCache.MaxBlockEntries)
  47         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/t=" + arvadostest.ActiveToken + "/foo")
  48         req := &http.Request{
  49                 Method:     "GET",
  50                 Host:       u.Host,
  51                 URL:        u,
  52                 RequestURI: u.RequestURI(),
  53         }
  54         resp := httptest.NewRecorder()
  55         h.ServeHTTP(resp, req)
  56         c.Check(resp.Code, check.Equals, http.StatusOK)
  57         c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Equals, cfg.cluster.Collections.WebDAVCache.MaxBlockEntries)
  58 }
  59
  60 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
  61         h := handler{Config: newConfig(s.Config)}
  62         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
  63         req := &http.Request{
  64                 Method:     "OPTIONS",
  65                 Host:       u.Host,
  66                 URL:        u,
  67                 RequestURI: u.RequestURI(),
  68                 Header: http.Header{
  69                         "Origin":                        {"https://workbench.example"},
  70                         "Access-Control-Request-Method": {"POST"},
  71                 },
  72         }
  73
  74         // Check preflight for an allowed request
  75         resp := httptest.NewRecorder()
  76         h.ServeHTTP(resp, req)
  77         c.Check(resp.Code, check.Equals, http.StatusOK)
  78         c.Check(resp.Body.String(), check.Equals, "")
  79         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
  80         c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
  81         c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout")
  82
  83         // Check preflight for a disallowed request
  84         resp = httptest.NewRecorder()
  85         req.Header.Set("Access-Control-Request-Method", "MAKE-COFFEE")
  86         h.ServeHTTP(resp, req)
  87         c.Check(resp.Body.String(), check.Equals, "")
  88         c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
  89 }
  90
  91 func (s *UnitSuite) TestInvalidUUID(c *check.C) {
  92         bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-"
  93         token := arvadostest.ActiveToken
  94         for _, trial := range []string{
  95                 "http://keep-web/c=" + bogusID + "/foo",
  96                 "http://keep-web/c=" + bogusID + "/t=" + token + "/foo",
  97                 "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo",
  98                 "http://keep-web/collections/" + bogusID + "/foo",
  99                 "http://" + bogusID + ".keep-web/" + bogusID + "/foo",
 100                 "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
 101         } {
 102                 c.Log(trial)
 103                 u := mustParseURL(trial)
 104                 req := &http.Request{
 105                         Method:     "GET",
 106                         Host:       u.Host,
 107                         URL:        u,
 108                         RequestURI: u.RequestURI(),
 109                 }
 110                 resp := httptest.NewRecorder()
 111                 cfg := newConfig(s.Config)
 112                 cfg.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 113                 h := handler{Config: cfg}
 114                 h.ServeHTTP(resp, req)
 115                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 116         }
 117 }
 118
 119 func mustParseURL(s string) *url.URL {
 120         r, err := url.Parse(s)
 121         if err != nil {
 122                 panic("parse URL: " + s)
 123         }
 124         return r
 125 }
 126
 127 func (s *IntegrationSuite) TestVhost404(c *check.C) {
 128         for _, testURL := range []string{
 129                 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
 130                 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
 131         } {
 132                 resp := httptest.NewRecorder()
 133                 u := mustParseURL(testURL)
 134                 req := &http.Request{
 135                         Method:     "GET",
 136                         URL:        u,
 137                         RequestURI: u.RequestURI(),
 138                 }
 139                 s.testServer.Handler.ServeHTTP(resp, req)
 140                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 141                 c.Check(resp.Body.String(), check.Equals, "")
 142         }
 143 }
 144
 145 // An authorizer modifies an HTTP request to make use of the given
 146 // token -- by adding it to a header, cookie, query param, or whatever
 147 // -- and returns the HTTP status code we should expect from keep-web if
 148 // the token is invalid.
 149 type authorizer func(*http.Request, string) int
 150
 151 func (s *IntegrationSuite) TestVhostViaAuthzHeader(c *check.C) {
 152         s.doVhostRequests(c, authzViaAuthzHeader)
 153 }
 154 func authzViaAuthzHeader(r *http.Request, tok string) int {
 155         r.Header.Add("Authorization", "OAuth2 "+tok)
 156         return http.StatusUnauthorized
 157 }
 158
 159 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
 160         s.doVhostRequests(c, authzViaCookieValue)
 161 }
 162 func authzViaCookieValue(r *http.Request, tok string) int {
 163         r.AddCookie(&http.Cookie{
 164                 Name:  "arvados_api_token",
 165                 Value: auth.EncodeTokenCookie([]byte(tok)),
 166         })
 167         return http.StatusUnauthorized
 168 }
 169
 170 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
 171         s.doVhostRequests(c, authzViaPath)
 172 }
 173 func authzViaPath(r *http.Request, tok string) int {
 174         r.URL.Path = "/t=" + tok + r.URL.Path
 175         return http.StatusNotFound
 176 }
 177
 178 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
 179         s.doVhostRequests(c, authzViaQueryString)
 180 }
 181 func authzViaQueryString(r *http.Request, tok string) int {
 182         r.URL.RawQuery = "api_token=" + tok
 183         return http.StatusUnauthorized
 184 }
 185
 186 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
 187         s.doVhostRequests(c, authzViaPOST)
 188 }
 189 func authzViaPOST(r *http.Request, tok string) int {
 190         r.Method = "POST"
 191         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 192         r.Body = ioutil.NopCloser(strings.NewReader(
 193                 url.Values{"api_token": {tok}}.Encode()))
 194         return http.StatusUnauthorized
 195 }
 196
 197 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
 198         s.doVhostRequests(c, authzViaPOST)
 199 }
 200 func authzViaXHRPOST(r *http.Request, tok string) int {
 201         r.Method = "POST"
 202         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 203         r.Header.Add("Origin", "https://origin.example")
 204         r.Body = ioutil.NopCloser(strings.NewReader(
 205                 url.Values{
 206                         "api_token":   {tok},
 207                         "disposition": {"attachment"},
 208                 }.Encode()))
 209         return http.StatusUnauthorized
 210 }
 211
 212 // Try some combinations of {url, token} using the given authorization
 213 // mechanism, and verify the result is correct.
 214 func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
 215         for _, hostPath := range []string{
 216                 arvadostest.FooCollection + ".example.com/foo",
 217                 arvadostest.FooCollection + "--collections.example.com/foo",
 218                 arvadostest.FooCollection + "--collections.example.com/_/foo",
 219                 arvadostest.FooCollectionPDH + ".example.com/foo",
 220                 strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + "--collections.example.com/foo",
 221                 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
 222         } {
 223                 c.Log("doRequests: ", hostPath)
 224                 s.doVhostRequestsWithHostPath(c, authz, hostPath)
 225         }
 226 }
 227
 228 func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
 229         for _, tok := range []string{
 230                 arvadostest.ActiveToken,
 231                 arvadostest.ActiveToken[:15],
 232                 arvadostest.SpectatorToken,
 233                 "bogus",
 234                 "",
 235         } {
 236                 u := mustParseURL("http://" + hostPath)
 237                 req := &http.Request{
 238                         Method:     "GET",
 239                         Host:       u.Host,
 240                         URL:        u,
 241                         RequestURI: u.RequestURI(),
 242                         Header:     http.Header{},
 243                 }
 244                 failCode := authz(req, tok)
 245                 req, resp := s.doReq(req)
 246                 code, body := resp.Code, resp.Body.String()
 247
 248                 // If the initial request had a (non-empty) token
 249                 // showing in the query string, we should have been
 250                 // redirected in order to hide it in a cookie.
 251                 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
 252
 253                 if tok == arvadostest.ActiveToken {
 254                         c.Check(code, check.Equals, http.StatusOK)
 255                         c.Check(body, check.Equals, "foo")
 256
 257                 } else {
 258                         c.Check(code >= 400, check.Equals, true)
 259                         c.Check(code < 500, check.Equals, true)
 260                         if tok == arvadostest.SpectatorToken {
 261                                 // Valid token never offers to retry
 262                                 // with different credentials.
 263                                 c.Check(code, check.Equals, http.StatusNotFound)
 264                         } else {
 265                                 // Invalid token can ask to retry
 266                                 // depending on the authz method.
 267                                 c.Check(code, check.Equals, failCode)
 268                         }
 269                         c.Check(body, check.Equals, "")
 270                 }
 271         }
 272 }
 273
 274 func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
 275         resp := httptest.NewRecorder()
 276         s.testServer.Handler.ServeHTTP(resp, req)
 277         if resp.Code != http.StatusSeeOther {
 278                 return req, resp
 279         }
 280         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 281         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 282         req = &http.Request{
 283                 Method:     "GET",
 284                 Host:       u.Host,
 285                 URL:        u,
 286                 RequestURI: u.RequestURI(),
 287                 Header:     http.Header{},
 288         }
 289         for _, c := range cookies {
 290                 req.AddCookie(c)
 291         }
 292         return s.doReq(req)
 293 }
 294
 295 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
 296         s.testVhostRedirectTokenToCookie(c, "GET",
 297                 arvadostest.FooCollection+".example.com/foo",
 298                 "?api_token="+arvadostest.ActiveToken,
 299                 "",
 300                 "",
 301                 http.StatusOK,
 302                 "foo",
 303         )
 304 }
 305
 306 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
 307         s.testVhostRedirectTokenToCookie(c, "GET",
 308                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
 309                 "",
 310                 "",
 311                 "",
 312                 http.StatusOK,
 313                 "foo",
 314         )
 315 }
 316
 317 // Bad token in URL is 404 Not Found because it doesn't make sense to
 318 // retry the same URL with different authorization.
 319 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
 320         s.testVhostRedirectTokenToCookie(c, "GET",
 321                 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
 322                 "",
 323                 "",
 324                 "",
 325                 http.StatusNotFound,
 326                 "",
 327         )
 328 }
 329
 330 // Bad token in a cookie (even if it got there via our own
 331 // query-string-to-cookie redirect) is, in principle, retryable at the
 332 // same URL so it's 401 Unauthorized.
 333 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
 334         s.testVhostRedirectTokenToCookie(c, "GET",
 335                 arvadostest.FooCollection+".example.com/foo",
 336                 "?api_token=thisisabogustoken",
 337                 "",
 338                 "",
 339                 http.StatusUnauthorized,
 340                 "",
 341         )
 342 }
 343
 344 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
 345         s.testVhostRedirectTokenToCookie(c, "GET",
 346                 "example.com/c="+arvadostest.FooCollection+"/foo",
 347                 "?api_token="+arvadostest.ActiveToken,
 348                 "",
 349                 "",
 350                 http.StatusBadRequest,
 351                 "",
 352         )
 353 }
 354
 355 // If client requests an attachment by putting ?disposition=attachment
 356 // in the query string, and gets redirected, the redirect target
 357 // should respond with an attachment.
 358 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
 359         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 360                 arvadostest.FooCollection+".example.com/foo",
 361                 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
 362                 "",
 363                 "",
 364                 http.StatusOK,
 365                 "foo",
 366         )
 367         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 368 }
 369
 370 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
 371         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 372         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 373                 "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
 374                 "?api_token="+arvadostest.ActiveToken,
 375                 "",
 376                 "",
 377                 http.StatusOK,
 378                 "foo",
 379         )
 380         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 381 }
 382
 383 func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
 384         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 385         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 386                 "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
 387                 "?api_token="+arvadostest.ActiveToken,
 388                 "",
 389                 "",
 390                 http.StatusOK,
 391                 "waz",
 392         )
 393         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 394         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 395                 "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz",
 396                 "?api_token="+arvadostest.ActiveToken,
 397                 "",
 398                 "",
 399                 http.StatusOK,
 400                 "waz",
 401         )
 402         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 403 }
 404
 405 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
 406         s.testServer.Config.cluster.Collections.TrustAllContent = true
 407         s.testVhostRedirectTokenToCookie(c, "GET",
 408                 "example.com/c="+arvadostest.FooCollection+"/foo",
 409                 "?api_token="+arvadostest.ActiveToken,
 410                 "",
 411                 "",
 412                 http.StatusOK,
 413                 "foo",
 414         )
 415 }
 416
 417 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
 418         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
 419
 420         s.testVhostRedirectTokenToCookie(c, "GET",
 421                 "example.com/c="+arvadostest.FooCollection+"/foo",
 422                 "?api_token="+arvadostest.ActiveToken,
 423                 "",
 424                 "",
 425                 http.StatusBadRequest,
 426                 "",
 427         )
 428
 429         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 430                 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
 431                 "?api_token="+arvadostest.ActiveToken,
 432                 "",
 433                 "",
 434                 http.StatusOK,
 435                 "foo",
 436         )
 437         c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
 438 }
 439
 440 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
 441         s.testVhostRedirectTokenToCookie(c, "POST",
 442                 arvadostest.FooCollection+".example.com/foo",
 443                 "",
 444                 "application/x-www-form-urlencoded",
 445                 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
 446                 http.StatusOK,
 447                 "foo",
 448         )
 449 }
 450
 451 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
 452         s.testVhostRedirectTokenToCookie(c, "POST",
 453                 arvadostest.FooCollection+".example.com/foo",
 454                 "",
 455                 "application/x-www-form-urlencoded",
 456                 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
 457                 http.StatusNotFound,
 458                 "",
 459         )
 460 }
 461
 462 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
 463         s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 464         s.testVhostRedirectTokenToCookie(c, "GET",
 465                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 466                 "",
 467                 "",
 468                 "",
 469                 http.StatusOK,
 470                 "Hello world\n",
 471         )
 472 }
 473
 474 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
 475         s.testServer.Config.cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
 476         s.testVhostRedirectTokenToCookie(c, "GET",
 477                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 478                 "",
 479                 "",
 480                 "",
 481                 http.StatusNotFound,
 482                 "",
 483         )
 484 }
 485
 486 func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
 487         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 488
 489         client := s.testServer.Config.Client
 490         client.AuthToken = arvadostest.ActiveToken
 491         fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
 492         c.Assert(err, check.IsNil)
 493         f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777)
 494         c.Assert(err, check.IsNil)
 495         f.Close()
 496         mtxt, err := fs.MarshalManifest(".")
 497         c.Assert(err, check.IsNil)
 498         var coll arvados.Collection
 499         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 500                 "collection": map[string]string{
 501                         "manifest_text": mtxt,
 502                 },
 503         })
 504         c.Assert(err, check.IsNil)
 505
 506         u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
 507         req := &http.Request{
 508                 Method:     "GET",
 509                 Host:       u.Host,
 510                 URL:        u,
 511                 RequestURI: u.RequestURI(),
 512                 Header: http.Header{
 513                         "Authorization": {"Bearer " + client.AuthToken},
 514                 },
 515         }
 516         resp := httptest.NewRecorder()
 517         s.testServer.Handler.ServeHTTP(resp, req)
 518         c.Check(resp.Code, check.Equals, http.StatusOK)
 519         c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\&#34;odd&#39; path chars.*`)
 520 }
 521
 522 // XHRs can't follow redirect-with-cookie so they rely on method=POST
 523 // and disposition=attachment (telling us it's acceptable to respond
 524 // with content instead of a redirect) and an Origin header that gets
 525 // added automatically by the browser (telling us it's desirable to do
 526 // so).
 527 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
 528         u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
 529         req := &http.Request{
 530                 Method:     "POST",
 531                 Host:       u.Host,
 532                 URL:        u,
 533                 RequestURI: u.RequestURI(),
 534                 Header: http.Header{
 535                         "Origin":       {"https://origin.example"},
 536                         "Content-Type": {"application/x-www-form-urlencoded"},
 537                 },
 538                 Body: ioutil.NopCloser(strings.NewReader(url.Values{
 539                         "api_token":   {arvadostest.ActiveToken},
 540                         "disposition": {"attachment"},
 541                 }.Encode())),
 542         }
 543         resp := httptest.NewRecorder()
 544         s.testServer.Handler.ServeHTTP(resp, req)
 545         c.Check(resp.Code, check.Equals, http.StatusOK)
 546         c.Check(resp.Body.String(), check.Equals, "foo")
 547         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
 548 }
 549
 550 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder {
 551         u, _ := url.Parse(`http://` + hostPath + queryString)
 552         req := &http.Request{
 553                 Method:     method,
 554                 Host:       u.Host,
 555                 URL:        u,
 556                 RequestURI: u.RequestURI(),
 557                 Header:     http.Header{"Content-Type": {contentType}},
 558                 Body:       ioutil.NopCloser(strings.NewReader(reqBody)),
 559         }
 560
 561         resp := httptest.NewRecorder()
 562         defer func() {
 563                 c.Check(resp.Code, check.Equals, expectStatus)
 564                 c.Check(resp.Body.String(), check.Equals, expectRespBody)
 565         }()
 566
 567         s.testServer.Handler.ServeHTTP(resp, req)
 568         if resp.Code != http.StatusSeeOther {
 569                 return resp
 570         }
 571         c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
 572         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 573
 574         u, _ = u.Parse(resp.Header().Get("Location"))
 575         req = &http.Request{
 576                 Method:     "GET",
 577                 Host:       u.Host,
 578                 URL:        u,
 579                 RequestURI: u.RequestURI(),
 580                 Header:     http.Header{},
 581         }
 582         for _, c := range cookies {
 583                 req.AddCookie(c)
 584         }
 585
 586         resp = httptest.NewRecorder()
 587         s.testServer.Handler.ServeHTTP(resp, req)
 588         c.Check(resp.Header().Get("Location"), check.Equals, "")
 589         return resp
 590 }
 591
 592 func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) {
 593         s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 594         s.testDirectoryListing(c)
 595 }
 596
 597 func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) {
 598         s.testServer.Config.cluster.Users.AnonymousUserToken = ""
 599         s.testDirectoryListing(c)
 600 }
 601
 602 func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
 603         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 604         authHeader := http.Header{
 605                 "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
 606         }
 607         for _, trial := range []struct {
 608                 uri      string
 609                 header   http.Header
 610                 expect   []string
 611                 redirect string
 612                 cutDirs  int
 613         }{
 614                 {
 615                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
 616                         header:  authHeader,
 617                         expect:  []string{"dir1/foo", "dir1/bar"},
 618                         cutDirs: 0,
 619                 },
 620                 {
 621                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
 622                         header:  authHeader,
 623                         expect:  []string{"foo", "bar"},
 624                         cutDirs: 1,
 625                 },
 626                 {
 627                         // URLs of this form ignore authHeader, and
 628                         // FooAndBarFilesInDirUUID isn't public, so
 629                         // this returns 404.
 630                         uri:    "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
 631                         header: authHeader,
 632                         expect: nil,
 633                 },
 634                 {
 635                         uri:     "download.example.com/users/active/foo_file_in_dir/",
 636                         header:  authHeader,
 637                         expect:  []string{"dir1/"},
 638                         cutDirs: 3,
 639                 },
 640                 {
 641                         uri:     "download.example.com/users/active/foo_file_in_dir/dir1/",
 642                         header:  authHeader,
 643                         expect:  []string{"bar"},
 644                         cutDirs: 4,
 645                 },
 646                 {
 647                         uri:     "download.example.com/",
 648                         header:  authHeader,
 649                         expect:  []string{"users/"},
 650                         cutDirs: 0,
 651                 },
 652                 {
 653                         uri:      "download.example.com/users",
 654                         header:   authHeader,
 655                         redirect: "/users/",
 656                         expect:   []string{"active/"},
 657                         cutDirs:  1,
 658                 },
 659                 {
 660                         uri:     "download.example.com/users/",
 661                         header:  authHeader,
 662                         expect:  []string{"active/"},
 663                         cutDirs: 1,
 664                 },
 665                 {
 666                         uri:      "download.example.com/users/active",
 667                         header:   authHeader,
 668                         redirect: "/users/active/",
 669                         expect:   []string{"foo_file_in_dir/"},
 670                         cutDirs:  2,
 671                 },
 672                 {
 673                         uri:     "download.example.com/users/active/",
 674                         header:  authHeader,
 675                         expect:  []string{"foo_file_in_dir/"},
 676                         cutDirs: 2,
 677                 },
 678                 {
 679                         uri:     "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
 680                         header:  nil,
 681                         expect:  []string{"dir1/foo", "dir1/bar"},
 682                         cutDirs: 4,
 683                 },
 684                 {
 685                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
 686                         header:  nil,
 687                         expect:  []string{"dir1/foo", "dir1/bar"},
 688                         cutDirs: 2,
 689                 },
 690                 {
 691                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken,
 692                         header:  nil,
 693                         expect:  []string{"dir1/foo", "dir1/bar"},
 694                         cutDirs: 2,
 695                 },
 696                 {
 697                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID,
 698                         header:  authHeader,
 699                         expect:  []string{"dir1/foo", "dir1/bar"},
 700                         cutDirs: 1,
 701                 },
 702                 {
 703                         uri:      "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
 704                         header:   authHeader,
 705                         redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
 706                         expect:   []string{"foo", "bar"},
 707                         cutDirs:  2,
 708                 },
 709                 {
 710                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
 711                         header:  authHeader,
 712                         expect:  []string{"foo", "bar"},
 713                         cutDirs: 3,
 714                 },
 715                 {
 716                         uri:      arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
 717                         header:   authHeader,
 718                         redirect: "/dir1/",
 719                         expect:   []string{"foo", "bar"},
 720                         cutDirs:  1,
 721                 },
 722                 {
 723                         uri:    "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
 724                         header: authHeader,
 725                         expect: nil,
 726                 },
 727                 {
 728                         uri:     "download.example.com/c=" + arvadostest.WazVersion1Collection,
 729                         header:  authHeader,
 730                         expect:  []string{"waz"},
 731                         cutDirs: 1,
 732                 },
 733                 {
 734                         uri:     "download.example.com/by_id/" + arvadostest.WazVersion1Collection,
 735                         header:  authHeader,
 736                         expect:  []string{"waz"},
 737                         cutDirs: 2,
 738                 },
 739         } {
 740                 comment := check.Commentf("HTML: %q => %q", trial.uri, trial.expect)
 741                 resp := httptest.NewRecorder()
 742                 u := mustParseURL("//" + trial.uri)
 743                 req := &http.Request{
 744                         Method:     "GET",
 745                         Host:       u.Host,
 746                         URL:        u,
 747                         RequestURI: u.RequestURI(),
 748                         Header:     copyHeader(trial.header),
 749                 }
 750                 s.testServer.Handler.ServeHTTP(resp, req)
 751                 var cookies []*http.Cookie
 752                 for resp.Code == http.StatusSeeOther {
 753                         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 754                         req = &http.Request{
 755                                 Method:     "GET",
 756                                 Host:       u.Host,
 757                                 URL:        u,
 758                                 RequestURI: u.RequestURI(),
 759                                 Header:     copyHeader(trial.header),
 760                         }
 761                         cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
 762                         for _, c := range cookies {
 763                                 req.AddCookie(c)
 764                         }
 765                         resp = httptest.NewRecorder()
 766                         s.testServer.Handler.ServeHTTP(resp, req)
 767                 }
 768                 if trial.redirect != "" {
 769                         c.Check(req.URL.Path, check.Equals, trial.redirect, comment)
 770                 }
 771                 if trial.expect == nil {
 772                         c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
 773                 } else {
 774                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
 775                         for _, e := range trial.expect {
 776                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`, comment)
 777                         }
 778                         c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`, comment)
 779                 }
 780
 781                 comment = check.Commentf("WebDAV: %q => %q", trial.uri, trial.expect)
 782                 req = &http.Request{
 783                         Method:     "OPTIONS",
 784                         Host:       u.Host,
 785                         URL:        u,
 786                         RequestURI: u.RequestURI(),
 787                         Header:     copyHeader(trial.header),
 788                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
 789                 }
 790                 resp = httptest.NewRecorder()
 791                 s.testServer.Handler.ServeHTTP(resp, req)
 792                 if trial.expect == nil {
 793                         c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
 794                 } else {
 795                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
 796                 }
 797
 798                 req = &http.Request{
 799                         Method:     "PROPFIND",
 800                         Host:       u.Host,
 801                         URL:        u,
 802                         RequestURI: u.RequestURI(),
 803                         Header:     copyHeader(trial.header),
 804                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
 805                 }
 806                 resp = httptest.NewRecorder()
 807                 s.testServer.Handler.ServeHTTP(resp, req)
 808                 if trial.expect == nil {
 809                         c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
 810                 } else {
 811                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment)
 812                         for _, e := range trial.expect {
 813                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+filepath.Join(u.Path, e)+`</D:href>.*`, comment)
 814                         }
 815                 }
 816         }
 817 }
 818
 819 func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
 820         arv := arvados.NewClientFromEnv()
 821         var newCollection arvados.Collection
 822         err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 823                 "collection": map[string]string{
 824                         "owner_uuid":    arvadostest.ActiveUserUUID,
 825                         "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
 826                         "name":          "keep-web test collection",
 827                 },
 828                 "ensure_unique_name": true,
 829         })
 830         c.Assert(err, check.IsNil)
 831         defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
 832
 833         var updated arvados.Collection
 834         for _, fnm := range []string{"foo.txt", "bar.txt"} {
 835                 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
 836                 u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
 837                 req := &http.Request{
 838                         Method:     "DELETE",
 839                         Host:       u.Host,
 840                         URL:        u,
 841                         RequestURI: u.RequestURI(),
 842                         Header: http.Header{
 843                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
 844                         },
 845                 }
 846                 resp := httptest.NewRecorder()
 847                 s.testServer.Handler.ServeHTTP(resp, req)
 848                 c.Check(resp.Code, check.Equals, http.StatusNoContent)
 849
 850                 updated = arvados.Collection{}
 851                 err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
 852                 c.Check(err, check.IsNil)
 853                 c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`)
 854                 c.Logf("updated manifest_text %q", updated.ManifestText)
 855         }
 856         c.Check(updated.ManifestText, check.Equals, "")
 857 }
 858
 859 func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) {
 860         s.testServer.Config.cluster.ManagementToken = arvadostest.ManagementToken
 861         authHeader := http.Header{
 862                 "Authorization": {"Bearer " + arvadostest.ManagementToken},
 863         }
 864
 865         resp := httptest.NewRecorder()
 866         u := mustParseURL("http://download.example.com/_health/ping")
 867         req := &http.Request{
 868                 Method:     "GET",
 869                 Host:       u.Host,
 870                 URL:        u,
 871                 RequestURI: u.RequestURI(),
 872                 Header:     authHeader,
 873         }
 874         s.testServer.Handler.ServeHTTP(resp, req)
 875
 876         c.Check(resp.Code, check.Equals, http.StatusOK)
 877         c.Check(resp.Body.String(), check.Matches, `{"health":"OK"}\n`)
 878 }
 879
 880 func copyHeader(h http.Header) http.Header {
 881         hc := http.Header{}
 882         for k, v := range h {
 883                 hc[k] = append([]string(nil), v...)
 884         }
 885         return hc
 886 }