services/keep-web/handler_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "fmt"
  10         "html"
  11         "io/ioutil"
  12         "net/http"
  13         "net/http/httptest"
  14         "net/url"
  15         "os"
  16         "path/filepath"
  17         "regexp"
  18         "strings"
  19
  20         "git.curoverse.com/arvados.git/lib/config"
  21         "git.curoverse.com/arvados.git/sdk/go/arvados"
  22         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  23         "git.curoverse.com/arvados.git/sdk/go/auth"
  24         check "gopkg.in/check.v1"
  25 )
  26
  27 var _ = check.Suite(&UnitSuite{})
  28
  29 type UnitSuite struct {
  30         Config *arvados.Config
  31 }
  32
  33 func (s *UnitSuite) SetUpTest(c *check.C) {
  34         ldr := config.NewLoader(nil, nil)
  35         cfg, err := ldr.LoadDefaults()
  36         c.Assert(err, check.IsNil)
  37         s.Config = cfg
  38 }
  39
  40 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
  41         h := handler{Config: DefaultConfig(s.Config)}
  42         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
  43         req := &http.Request{
  44                 Method:     "OPTIONS",
  45                 Host:       u.Host,
  46                 URL:        u,
  47                 RequestURI: u.RequestURI(),
  48                 Header: http.Header{
  49                         "Origin":                        {"https://workbench.example"},
  50                         "Access-Control-Request-Method": {"POST"},
  51                 },
  52         }
  53
  54         // Check preflight for an allowed request
  55         resp := httptest.NewRecorder()
  56         h.ServeHTTP(resp, req)
  57         c.Check(resp.Code, check.Equals, http.StatusOK)
  58         c.Check(resp.Body.String(), check.Equals, "")
  59         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
  60         c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
  61         c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout")
  62
  63         // Check preflight for a disallowed request
  64         resp = httptest.NewRecorder()
  65         req.Header.Set("Access-Control-Request-Method", "MAKE-COFFEE")
  66         h.ServeHTTP(resp, req)
  67         c.Check(resp.Body.String(), check.Equals, "")
  68         c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
  69 }
  70
  71 func (s *UnitSuite) TestInvalidUUID(c *check.C) {
  72         bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-"
  73         token := arvadostest.ActiveToken
  74         for _, trial := range []string{
  75                 "http://keep-web/c=" + bogusID + "/foo",
  76                 "http://keep-web/c=" + bogusID + "/t=" + token + "/foo",
  77                 "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo",
  78                 "http://keep-web/collections/" + bogusID + "/foo",
  79                 "http://" + bogusID + ".keep-web/" + bogusID + "/foo",
  80                 "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
  81         } {
  82                 c.Log(trial)
  83                 u := mustParseURL(trial)
  84                 req := &http.Request{
  85                         Method:     "GET",
  86                         Host:       u.Host,
  87                         URL:        u,
  88                         RequestURI: u.RequestURI(),
  89                 }
  90                 resp := httptest.NewRecorder()
  91                 cfg := DefaultConfig(s.Config)
  92                 cfg.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
  93                 h := handler{Config: cfg}
  94                 h.ServeHTTP(resp, req)
  95                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
  96         }
  97 }
  98
  99 func mustParseURL(s string) *url.URL {
 100         r, err := url.Parse(s)
 101         if err != nil {
 102                 panic("parse URL: " + s)
 103         }
 104         return r
 105 }
 106
 107 func (s *IntegrationSuite) TestVhost404(c *check.C) {
 108         for _, testURL := range []string{
 109                 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
 110                 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
 111         } {
 112                 resp := httptest.NewRecorder()
 113                 u := mustParseURL(testURL)
 114                 req := &http.Request{
 115                         Method:     "GET",
 116                         URL:        u,
 117                         RequestURI: u.RequestURI(),
 118                 }
 119                 s.testServer.Handler.ServeHTTP(resp, req)
 120                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 121                 c.Check(resp.Body.String(), check.Equals, "")
 122         }
 123 }
 124
 125 // An authorizer modifies an HTTP request to make use of the given
 126 // token -- by adding it to a header, cookie, query param, or whatever
 127 // -- and returns the HTTP status code we should expect from keep-web if
 128 // the token is invalid.
 129 type authorizer func(*http.Request, string) int
 130
 131 func (s *IntegrationSuite) TestVhostViaAuthzHeader(c *check.C) {
 132         s.doVhostRequests(c, authzViaAuthzHeader)
 133 }
 134 func authzViaAuthzHeader(r *http.Request, tok string) int {
 135         r.Header.Add("Authorization", "OAuth2 "+tok)
 136         return http.StatusUnauthorized
 137 }
 138
 139 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
 140         s.doVhostRequests(c, authzViaCookieValue)
 141 }
 142 func authzViaCookieValue(r *http.Request, tok string) int {
 143         r.AddCookie(&http.Cookie{
 144                 Name:  "arvados_api_token",
 145                 Value: auth.EncodeTokenCookie([]byte(tok)),
 146         })
 147         return http.StatusUnauthorized
 148 }
 149
 150 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
 151         s.doVhostRequests(c, authzViaPath)
 152 }
 153 func authzViaPath(r *http.Request, tok string) int {
 154         r.URL.Path = "/t=" + tok + r.URL.Path
 155         return http.StatusNotFound
 156 }
 157
 158 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
 159         s.doVhostRequests(c, authzViaQueryString)
 160 }
 161 func authzViaQueryString(r *http.Request, tok string) int {
 162         r.URL.RawQuery = "api_token=" + tok
 163         return http.StatusUnauthorized
 164 }
 165
 166 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
 167         s.doVhostRequests(c, authzViaPOST)
 168 }
 169 func authzViaPOST(r *http.Request, tok string) int {
 170         r.Method = "POST"
 171         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 172         r.Body = ioutil.NopCloser(strings.NewReader(
 173                 url.Values{"api_token": {tok}}.Encode()))
 174         return http.StatusUnauthorized
 175 }
 176
 177 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
 178         s.doVhostRequests(c, authzViaPOST)
 179 }
 180 func authzViaXHRPOST(r *http.Request, tok string) int {
 181         r.Method = "POST"
 182         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 183         r.Header.Add("Origin", "https://origin.example")
 184         r.Body = ioutil.NopCloser(strings.NewReader(
 185                 url.Values{
 186                         "api_token":   {tok},
 187                         "disposition": {"attachment"},
 188                 }.Encode()))
 189         return http.StatusUnauthorized
 190 }
 191
 192 // Try some combinations of {url, token} using the given authorization
 193 // mechanism, and verify the result is correct.
 194 func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
 195         for _, hostPath := range []string{
 196                 arvadostest.FooCollection + ".example.com/foo",
 197                 arvadostest.FooCollection + "--collections.example.com/foo",
 198                 arvadostest.FooCollection + "--collections.example.com/_/foo",
 199                 arvadostest.FooCollectionPDH + ".example.com/foo",
 200                 strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + "--collections.example.com/foo",
 201                 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
 202         } {
 203                 c.Log("doRequests: ", hostPath)
 204                 s.doVhostRequestsWithHostPath(c, authz, hostPath)
 205         }
 206 }
 207
 208 func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
 209         for _, tok := range []string{
 210                 arvadostest.ActiveToken,
 211                 arvadostest.ActiveToken[:15],
 212                 arvadostest.SpectatorToken,
 213                 "bogus",
 214                 "",
 215         } {
 216                 u := mustParseURL("http://" + hostPath)
 217                 req := &http.Request{
 218                         Method:     "GET",
 219                         Host:       u.Host,
 220                         URL:        u,
 221                         RequestURI: u.RequestURI(),
 222                         Header:     http.Header{},
 223                 }
 224                 failCode := authz(req, tok)
 225                 req, resp := s.doReq(req)
 226                 code, body := resp.Code, resp.Body.String()
 227
 228                 // If the initial request had a (non-empty) token
 229                 // showing in the query string, we should have been
 230                 // redirected in order to hide it in a cookie.
 231                 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
 232
 233                 if tok == arvadostest.ActiveToken {
 234                         c.Check(code, check.Equals, http.StatusOK)
 235                         c.Check(body, check.Equals, "foo")
 236
 237                 } else {
 238                         c.Check(code >= 400, check.Equals, true)
 239                         c.Check(code < 500, check.Equals, true)
 240                         if tok == arvadostest.SpectatorToken {
 241                                 // Valid token never offers to retry
 242                                 // with different credentials.
 243                                 c.Check(code, check.Equals, http.StatusNotFound)
 244                         } else {
 245                                 // Invalid token can ask to retry
 246                                 // depending on the authz method.
 247                                 c.Check(code, check.Equals, failCode)
 248                         }
 249                         c.Check(body, check.Equals, "")
 250                 }
 251         }
 252 }
 253
 254 func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
 255         resp := httptest.NewRecorder()
 256         s.testServer.Handler.ServeHTTP(resp, req)
 257         if resp.Code != http.StatusSeeOther {
 258                 return req, resp
 259         }
 260         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 261         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 262         req = &http.Request{
 263                 Method:     "GET",
 264                 Host:       u.Host,
 265                 URL:        u,
 266                 RequestURI: u.RequestURI(),
 267                 Header:     http.Header{},
 268         }
 269         for _, c := range cookies {
 270                 req.AddCookie(c)
 271         }
 272         return s.doReq(req)
 273 }
 274
 275 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
 276         s.testVhostRedirectTokenToCookie(c, "GET",
 277                 arvadostest.FooCollection+".example.com/foo",
 278                 "?api_token="+arvadostest.ActiveToken,
 279                 "",
 280                 "",
 281                 http.StatusOK,
 282                 "foo",
 283         )
 284 }
 285
 286 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
 287         s.testVhostRedirectTokenToCookie(c, "GET",
 288                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
 289                 "",
 290                 "",
 291                 "",
 292                 http.StatusOK,
 293                 "foo",
 294         )
 295 }
 296
 297 // Bad token in URL is 404 Not Found because it doesn't make sense to
 298 // retry the same URL with different authorization.
 299 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
 300         s.testVhostRedirectTokenToCookie(c, "GET",
 301                 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
 302                 "",
 303                 "",
 304                 "",
 305                 http.StatusNotFound,
 306                 "",
 307         )
 308 }
 309
 310 // Bad token in a cookie (even if it got there via our own
 311 // query-string-to-cookie redirect) is, in principle, retryable at the
 312 // same URL so it's 401 Unauthorized.
 313 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
 314         s.testVhostRedirectTokenToCookie(c, "GET",
 315                 arvadostest.FooCollection+".example.com/foo",
 316                 "?api_token=thisisabogustoken",
 317                 "",
 318                 "",
 319                 http.StatusUnauthorized,
 320                 "",
 321         )
 322 }
 323
 324 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
 325         s.testVhostRedirectTokenToCookie(c, "GET",
 326                 "example.com/c="+arvadostest.FooCollection+"/foo",
 327                 "?api_token="+arvadostest.ActiveToken,
 328                 "",
 329                 "",
 330                 http.StatusBadRequest,
 331                 "",
 332         )
 333 }
 334
 335 // If client requests an attachment by putting ?disposition=attachment
 336 // in the query string, and gets redirected, the redirect target
 337 // should respond with an attachment.
 338 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
 339         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 340                 arvadostest.FooCollection+".example.com/foo",
 341                 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
 342                 "",
 343                 "",
 344                 http.StatusOK,
 345                 "foo",
 346         )
 347         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 348 }
 349
 350 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
 351         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 352         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 353                 "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
 354                 "?api_token="+arvadostest.ActiveToken,
 355                 "",
 356                 "",
 357                 http.StatusOK,
 358                 "foo",
 359         )
 360         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 361 }
 362
 363 func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
 364         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 365         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 366                 "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
 367                 "?api_token="+arvadostest.ActiveToken,
 368                 "",
 369                 "",
 370                 http.StatusOK,
 371                 "waz",
 372         )
 373         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 374         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 375                 "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz",
 376                 "?api_token="+arvadostest.ActiveToken,
 377                 "",
 378                 "",
 379                 http.StatusOK,
 380                 "waz",
 381         )
 382         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 383 }
 384
 385 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
 386         s.testServer.Config.cluster.Collections.TrustAllContent = true
 387         s.testVhostRedirectTokenToCookie(c, "GET",
 388                 "example.com/c="+arvadostest.FooCollection+"/foo",
 389                 "?api_token="+arvadostest.ActiveToken,
 390                 "",
 391                 "",
 392                 http.StatusOK,
 393                 "foo",
 394         )
 395 }
 396
 397 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
 398         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
 399
 400         s.testVhostRedirectTokenToCookie(c, "GET",
 401                 "example.com/c="+arvadostest.FooCollection+"/foo",
 402                 "?api_token="+arvadostest.ActiveToken,
 403                 "",
 404                 "",
 405                 http.StatusBadRequest,
 406                 "",
 407         )
 408
 409         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 410                 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
 411                 "?api_token="+arvadostest.ActiveToken,
 412                 "",
 413                 "",
 414                 http.StatusOK,
 415                 "foo",
 416         )
 417         c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
 418 }
 419
 420 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
 421         s.testVhostRedirectTokenToCookie(c, "POST",
 422                 arvadostest.FooCollection+".example.com/foo",
 423                 "",
 424                 "application/x-www-form-urlencoded",
 425                 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
 426                 http.StatusOK,
 427                 "foo",
 428         )
 429 }
 430
 431 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
 432         s.testVhostRedirectTokenToCookie(c, "POST",
 433                 arvadostest.FooCollection+".example.com/foo",
 434                 "",
 435                 "application/x-www-form-urlencoded",
 436                 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
 437                 http.StatusNotFound,
 438                 "",
 439         )
 440 }
 441
 442 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
 443         s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 444         s.testVhostRedirectTokenToCookie(c, "GET",
 445                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 446                 "",
 447                 "",
 448                 "",
 449                 http.StatusOK,
 450                 "Hello world\n",
 451         )
 452 }
 453
 454 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
 455         s.testServer.Config.cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
 456         s.testVhostRedirectTokenToCookie(c, "GET",
 457                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 458                 "",
 459                 "",
 460                 "",
 461                 http.StatusNotFound,
 462                 "",
 463         )
 464 }
 465
 466 func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
 467         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 468
 469         client := s.testServer.Config.Client
 470         client.AuthToken = arvadostest.ActiveToken
 471         fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
 472         c.Assert(err, check.IsNil)
 473         f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777)
 474         c.Assert(err, check.IsNil)
 475         f.Close()
 476         mtxt, err := fs.MarshalManifest(".")
 477         c.Assert(err, check.IsNil)
 478         var coll arvados.Collection
 479         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 480                 "collection": map[string]string{
 481                         "manifest_text": mtxt,
 482                 },
 483         })
 484         c.Assert(err, check.IsNil)
 485
 486         u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
 487         req := &http.Request{
 488                 Method:     "GET",
 489                 Host:       u.Host,
 490                 URL:        u,
 491                 RequestURI: u.RequestURI(),
 492                 Header: http.Header{
 493                         "Authorization": {"Bearer " + client.AuthToken},
 494                 },
 495         }
 496         resp := httptest.NewRecorder()
 497         s.testServer.Handler.ServeHTTP(resp, req)
 498         c.Check(resp.Code, check.Equals, http.StatusOK)
 499         c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\&#34;odd&#39; path chars.*`)
 500 }
 501
 502 // XHRs can't follow redirect-with-cookie so they rely on method=POST
 503 // and disposition=attachment (telling us it's acceptable to respond
 504 // with content instead of a redirect) and an Origin header that gets
 505 // added automatically by the browser (telling us it's desirable to do
 506 // so).
 507 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
 508         u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
 509         req := &http.Request{
 510                 Method:     "POST",
 511                 Host:       u.Host,
 512                 URL:        u,
 513                 RequestURI: u.RequestURI(),
 514                 Header: http.Header{
 515                         "Origin":       {"https://origin.example"},
 516                         "Content-Type": {"application/x-www-form-urlencoded"},
 517                 },
 518                 Body: ioutil.NopCloser(strings.NewReader(url.Values{
 519                         "api_token":   {arvadostest.ActiveToken},
 520                         "disposition": {"attachment"},
 521                 }.Encode())),
 522         }
 523         resp := httptest.NewRecorder()
 524         s.testServer.Handler.ServeHTTP(resp, req)
 525         c.Check(resp.Code, check.Equals, http.StatusOK)
 526         c.Check(resp.Body.String(), check.Equals, "foo")
 527         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
 528 }
 529
 530 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder {
 531         u, _ := url.Parse(`http://` + hostPath + queryString)
 532         req := &http.Request{
 533                 Method:     method,
 534                 Host:       u.Host,
 535                 URL:        u,
 536                 RequestURI: u.RequestURI(),
 537                 Header:     http.Header{"Content-Type": {contentType}},
 538                 Body:       ioutil.NopCloser(strings.NewReader(reqBody)),
 539         }
 540
 541         resp := httptest.NewRecorder()
 542         defer func() {
 543                 c.Check(resp.Code, check.Equals, expectStatus)
 544                 c.Check(resp.Body.String(), check.Equals, expectRespBody)
 545         }()
 546
 547         s.testServer.Handler.ServeHTTP(resp, req)
 548         if resp.Code != http.StatusSeeOther {
 549                 return resp
 550         }
 551         c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
 552         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 553
 554         u, _ = u.Parse(resp.Header().Get("Location"))
 555         req = &http.Request{
 556                 Method:     "GET",
 557                 Host:       u.Host,
 558                 URL:        u,
 559                 RequestURI: u.RequestURI(),
 560                 Header:     http.Header{},
 561         }
 562         for _, c := range cookies {
 563                 req.AddCookie(c)
 564         }
 565
 566         resp = httptest.NewRecorder()
 567         s.testServer.Handler.ServeHTTP(resp, req)
 568         c.Check(resp.Header().Get("Location"), check.Equals, "")
 569         return resp
 570 }
 571
 572 func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
 573         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 574         authHeader := http.Header{
 575                 "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
 576         }
 577         for _, trial := range []struct {
 578                 uri      string
 579                 header   http.Header
 580                 expect   []string
 581                 redirect string
 582                 cutDirs  int
 583         }{
 584                 {
 585                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
 586                         header:  authHeader,
 587                         expect:  []string{"dir1/foo", "dir1/bar"},
 588                         cutDirs: 0,
 589                 },
 590                 {
 591                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
 592                         header:  authHeader,
 593                         expect:  []string{"foo", "bar"},
 594                         cutDirs: 1,
 595                 },
 596                 // This test case fails
 597                 {
 598                         uri:     "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
 599                         header:  authHeader,
 600                         expect:  []string{"dir1/foo", "dir1/bar"},
 601                         cutDirs: 2,
 602                 },
 603                 {
 604                         uri:     "download.example.com/users/active/foo_file_in_dir/",
 605                         header:  authHeader,
 606                         expect:  []string{"dir1/"},
 607                         cutDirs: 3,
 608                 },
 609                 {
 610                         uri:     "download.example.com/users/active/foo_file_in_dir/dir1/",
 611                         header:  authHeader,
 612                         expect:  []string{"bar"},
 613                         cutDirs: 4,
 614                 },
 615                 {
 616                         uri:     "download.example.com/",
 617                         header:  authHeader,
 618                         expect:  []string{"users/"},
 619                         cutDirs: 0,
 620                 },
 621                 {
 622                         uri:      "download.example.com/users",
 623                         header:   authHeader,
 624                         redirect: "/users/",
 625                         expect:   []string{"active/"},
 626                         cutDirs:  1,
 627                 },
 628                 {
 629                         uri:     "download.example.com/users/",
 630                         header:  authHeader,
 631                         expect:  []string{"active/"},
 632                         cutDirs: 1,
 633                 },
 634                 {
 635                         uri:      "download.example.com/users/active",
 636                         header:   authHeader,
 637                         redirect: "/users/active/",
 638                         expect:   []string{"foo_file_in_dir/"},
 639                         cutDirs:  2,
 640                 },
 641                 {
 642                         uri:     "download.example.com/users/active/",
 643                         header:  authHeader,
 644                         expect:  []string{"foo_file_in_dir/"},
 645                         cutDirs: 2,
 646                 },
 647                 {
 648                         uri:     "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
 649                         header:  nil,
 650                         expect:  []string{"dir1/foo", "dir1/bar"},
 651                         cutDirs: 4,
 652                 },
 653                 {
 654                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
 655                         header:  nil,
 656                         expect:  []string{"dir1/foo", "dir1/bar"},
 657                         cutDirs: 2,
 658                 },
 659                 {
 660                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken,
 661                         header:  nil,
 662                         expect:  []string{"dir1/foo", "dir1/bar"},
 663                         cutDirs: 2,
 664                 },
 665                 {
 666                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID,
 667                         header:  authHeader,
 668                         expect:  []string{"dir1/foo", "dir1/bar"},
 669                         cutDirs: 1,
 670                 },
 671                 {
 672                         uri:      "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
 673                         header:   authHeader,
 674                         redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
 675                         expect:   []string{"foo", "bar"},
 676                         cutDirs:  2,
 677                 },
 678                 {
 679                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
 680                         header:  authHeader,
 681                         expect:  []string{"foo", "bar"},
 682                         cutDirs: 3,
 683                 },
 684                 {
 685                         uri:      arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
 686                         header:   authHeader,
 687                         redirect: "/dir1/",
 688                         expect:   []string{"foo", "bar"},
 689                         cutDirs:  1,
 690                 },
 691                 {
 692                         uri:    "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
 693                         header: authHeader,
 694                         expect: nil,
 695                 },
 696                 {
 697                         uri:     "download.example.com/c=" + arvadostest.WazVersion1Collection,
 698                         header:  authHeader,
 699                         expect:  []string{"waz"},
 700                         cutDirs: 1,
 701                 },
 702                 {
 703                         uri:     "download.example.com/by_id/" + arvadostest.WazVersion1Collection,
 704                         header:  authHeader,
 705                         expect:  []string{"waz"},
 706                         cutDirs: 2,
 707                 },
 708         } {
 709                 comment := check.Commentf("HTML: %q => %q", trial.uri, trial.expect)
 710                 resp := httptest.NewRecorder()
 711                 u := mustParseURL("//" + trial.uri)
 712                 req := &http.Request{
 713                         Method:     "GET",
 714                         Host:       u.Host,
 715                         URL:        u,
 716                         RequestURI: u.RequestURI(),
 717                         Header:     copyHeader(trial.header),
 718                 }
 719                 s.testServer.Handler.ServeHTTP(resp, req)
 720                 var cookies []*http.Cookie
 721                 for resp.Code == http.StatusSeeOther {
 722                         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 723                         req = &http.Request{
 724                                 Method:     "GET",
 725                                 Host:       u.Host,
 726                                 URL:        u,
 727                                 RequestURI: u.RequestURI(),
 728                                 Header:     copyHeader(trial.header),
 729                         }
 730                         cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
 731                         for _, c := range cookies {
 732                                 req.AddCookie(c)
 733                         }
 734                         resp = httptest.NewRecorder()
 735                         s.testServer.Handler.ServeHTTP(resp, req)
 736                 }
 737                 if trial.redirect != "" {
 738                         c.Check(req.URL.Path, check.Equals, trial.redirect, comment)
 739                 }
 740                 if trial.expect == nil {
 741                         c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
 742                 } else {
 743                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
 744                         for _, e := range trial.expect {
 745                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`, comment)
 746                         }
 747                         c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`, comment)
 748                 }
 749
 750                 comment = check.Commentf("WebDAV: %q => %q", trial.uri, trial.expect)
 751                 req = &http.Request{
 752                         Method:     "OPTIONS",
 753                         Host:       u.Host,
 754                         URL:        u,
 755                         RequestURI: u.RequestURI(),
 756                         Header:     copyHeader(trial.header),
 757                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
 758                 }
 759                 resp = httptest.NewRecorder()
 760                 s.testServer.Handler.ServeHTTP(resp, req)
 761                 if trial.expect == nil {
 762                         c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
 763                 } else {
 764                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
 765                 }
 766
 767                 req = &http.Request{
 768                         Method:     "PROPFIND",
 769                         Host:       u.Host,
 770                         URL:        u,
 771                         RequestURI: u.RequestURI(),
 772                         Header:     copyHeader(trial.header),
 773                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
 774                 }
 775                 resp = httptest.NewRecorder()
 776                 s.testServer.Handler.ServeHTTP(resp, req)
 777                 if trial.expect == nil {
 778                         c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
 779                 } else {
 780                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment)
 781                         for _, e := range trial.expect {
 782                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+filepath.Join(u.Path, e)+`</D:href>.*`, comment)
 783                         }
 784                 }
 785         }
 786 }
 787
 788 func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
 789         arv := arvados.NewClientFromEnv()
 790         var newCollection arvados.Collection
 791         err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 792                 "collection": map[string]string{
 793                         "owner_uuid":    arvadostest.ActiveUserUUID,
 794                         "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
 795                         "name":          "keep-web test collection",
 796                 },
 797                 "ensure_unique_name": true,
 798         })
 799         c.Assert(err, check.IsNil)
 800         defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
 801
 802         var updated arvados.Collection
 803         for _, fnm := range []string{"foo.txt", "bar.txt"} {
 804                 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
 805                 u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
 806                 req := &http.Request{
 807                         Method:     "DELETE",
 808                         Host:       u.Host,
 809                         URL:        u,
 810                         RequestURI: u.RequestURI(),
 811                         Header: http.Header{
 812                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
 813                         },
 814                 }
 815                 resp := httptest.NewRecorder()
 816                 s.testServer.Handler.ServeHTTP(resp, req)
 817                 c.Check(resp.Code, check.Equals, http.StatusNoContent)
 818
 819                 updated = arvados.Collection{}
 820                 err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
 821                 c.Check(err, check.IsNil)
 822                 c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`)
 823                 c.Logf("updated manifest_text %q", updated.ManifestText)
 824         }
 825         c.Check(updated.ManifestText, check.Equals, "")
 826 }
 827
 828 func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) {
 829         s.testServer.Config.cluster.ManagementToken = arvadostest.ManagementToken
 830         authHeader := http.Header{
 831                 "Authorization": {"Bearer " + arvadostest.ManagementToken},
 832         }
 833
 834         resp := httptest.NewRecorder()
 835         u := mustParseURL("http://download.example.com/_health/ping")
 836         req := &http.Request{
 837                 Method:     "GET",
 838                 Host:       u.Host,
 839                 URL:        u,
 840                 RequestURI: u.RequestURI(),
 841                 Header:     authHeader,
 842         }
 843         s.testServer.Handler.ServeHTTP(resp, req)
 844
 845         c.Check(resp.Code, check.Equals, http.StatusOK)
 846         c.Check(resp.Body.String(), check.Matches, `{"health":"OK"}\n`)
 847 }
 848
 849 func copyHeader(h http.Header) http.Header {
 850         hc := http.Header{}
 851         for k, v := range h {
 852                 hc[k] = append([]string(nil), v...)
 853         }
 854         return hc
 855 }