services/keep-web/handler_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "fmt"
  10         "html"
  11         "io/ioutil"
  12         "net/http"
  13         "net/http/httptest"
  14         "net/url"
  15         "os"
  16         "path/filepath"
  17         "regexp"
  18         "strings"
  19
  20         "git.curoverse.com/arvados.git/sdk/go/arvados"
  21         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  22         "git.curoverse.com/arvados.git/sdk/go/auth"
  23         check "gopkg.in/check.v1"
  24 )
  25
  26 var _ = check.Suite(&UnitSuite{})
  27
  28 type UnitSuite struct{}
  29
  30 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
  31         h := handler{Config: DefaultConfig()}
  32         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
  33         req := &http.Request{
  34                 Method:     "OPTIONS",
  35                 Host:       u.Host,
  36                 URL:        u,
  37                 RequestURI: u.RequestURI(),
  38                 Header: http.Header{
  39                         "Origin":                        {"https://workbench.example"},
  40                         "Access-Control-Request-Method": {"POST"},
  41                 },
  42         }
  43
  44         // Check preflight for an allowed request
  45         resp := httptest.NewRecorder()
  46         h.ServeHTTP(resp, req)
  47         c.Check(resp.Code, check.Equals, http.StatusOK)
  48         c.Check(resp.Body.String(), check.Equals, "")
  49         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
  50         c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
  51         c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout")
  52
  53         // Check preflight for a disallowed request
  54         resp = httptest.NewRecorder()
  55         req.Header.Set("Access-Control-Request-Method", "MAKE-COFFEE")
  56         h.ServeHTTP(resp, req)
  57         c.Check(resp.Body.String(), check.Equals, "")
  58         c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
  59 }
  60
  61 func (s *UnitSuite) TestInvalidUUID(c *check.C) {
  62         bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-"
  63         token := arvadostest.ActiveToken
  64         for _, trial := range []string{
  65                 "http://keep-web/c=" + bogusID + "/foo",
  66                 "http://keep-web/c=" + bogusID + "/t=" + token + "/foo",
  67                 "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo",
  68                 "http://keep-web/collections/" + bogusID + "/foo",
  69                 "http://" + bogusID + ".keep-web/" + bogusID + "/foo",
  70                 "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
  71         } {
  72                 c.Log(trial)
  73                 u := mustParseURL(trial)
  74                 req := &http.Request{
  75                         Method:     "GET",
  76                         Host:       u.Host,
  77                         URL:        u,
  78                         RequestURI: u.RequestURI(),
  79                 }
  80                 resp := httptest.NewRecorder()
  81                 cfg := DefaultConfig()
  82                 cfg.AnonymousTokens = []string{arvadostest.AnonymousToken}
  83                 h := handler{Config: cfg}
  84                 h.ServeHTTP(resp, req)
  85                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
  86         }
  87 }
  88
  89 func mustParseURL(s string) *url.URL {
  90         r, err := url.Parse(s)
  91         if err != nil {
  92                 panic("parse URL: " + s)
  93         }
  94         return r
  95 }
  96
  97 func (s *IntegrationSuite) TestVhost404(c *check.C) {
  98         for _, testURL := range []string{
  99                 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
 100                 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
 101         } {
 102                 resp := httptest.NewRecorder()
 103                 u := mustParseURL(testURL)
 104                 req := &http.Request{
 105                         Method:     "GET",
 106                         URL:        u,
 107                         RequestURI: u.RequestURI(),
 108                 }
 109                 s.testServer.Handler.ServeHTTP(resp, req)
 110                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 111                 c.Check(resp.Body.String(), check.Equals, "")
 112         }
 113 }
 114
 115 // An authorizer modifies an HTTP request to make use of the given
 116 // token -- by adding it to a header, cookie, query param, or whatever
 117 // -- and returns the HTTP status code we should expect from keep-web if
 118 // the token is invalid.
 119 type authorizer func(*http.Request, string) int
 120
 121 func (s *IntegrationSuite) TestVhostViaAuthzHeader(c *check.C) {
 122         s.doVhostRequests(c, authzViaAuthzHeader)
 123 }
 124 func authzViaAuthzHeader(r *http.Request, tok string) int {
 125         r.Header.Add("Authorization", "OAuth2 "+tok)
 126         return http.StatusUnauthorized
 127 }
 128
 129 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
 130         s.doVhostRequests(c, authzViaCookieValue)
 131 }
 132 func authzViaCookieValue(r *http.Request, tok string) int {
 133         r.AddCookie(&http.Cookie{
 134                 Name:  "arvados_api_token",
 135                 Value: auth.EncodeTokenCookie([]byte(tok)),
 136         })
 137         return http.StatusUnauthorized
 138 }
 139
 140 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
 141         s.doVhostRequests(c, authzViaPath)
 142 }
 143 func authzViaPath(r *http.Request, tok string) int {
 144         r.URL.Path = "/t=" + tok + r.URL.Path
 145         return http.StatusNotFound
 146 }
 147
 148 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
 149         s.doVhostRequests(c, authzViaQueryString)
 150 }
 151 func authzViaQueryString(r *http.Request, tok string) int {
 152         r.URL.RawQuery = "api_token=" + tok
 153         return http.StatusUnauthorized
 154 }
 155
 156 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
 157         s.doVhostRequests(c, authzViaPOST)
 158 }
 159 func authzViaPOST(r *http.Request, tok string) int {
 160         r.Method = "POST"
 161         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 162         r.Body = ioutil.NopCloser(strings.NewReader(
 163                 url.Values{"api_token": {tok}}.Encode()))
 164         return http.StatusUnauthorized
 165 }
 166
 167 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
 168         s.doVhostRequests(c, authzViaPOST)
 169 }
 170 func authzViaXHRPOST(r *http.Request, tok string) int {
 171         r.Method = "POST"
 172         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 173         r.Header.Add("Origin", "https://origin.example")
 174         r.Body = ioutil.NopCloser(strings.NewReader(
 175                 url.Values{
 176                         "api_token":   {tok},
 177                         "disposition": {"attachment"},
 178                 }.Encode()))
 179         return http.StatusUnauthorized
 180 }
 181
 182 // Try some combinations of {url, token} using the given authorization
 183 // mechanism, and verify the result is correct.
 184 func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
 185         for _, hostPath := range []string{
 186                 arvadostest.FooCollection + ".example.com/foo",
 187                 arvadostest.FooCollection + "--collections.example.com/foo",
 188                 arvadostest.FooCollection + "--collections.example.com/_/foo",
 189                 arvadostest.FooCollectionPDH + ".example.com/foo",
 190                 strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + "--collections.example.com/foo",
 191                 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
 192         } {
 193                 c.Log("doRequests: ", hostPath)
 194                 s.doVhostRequestsWithHostPath(c, authz, hostPath)
 195         }
 196 }
 197
 198 func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
 199         for _, tok := range []string{
 200                 arvadostest.ActiveToken,
 201                 arvadostest.ActiveToken[:15],
 202                 arvadostest.SpectatorToken,
 203                 "bogus",
 204                 "",
 205         } {
 206                 u := mustParseURL("http://" + hostPath)
 207                 req := &http.Request{
 208                         Method:     "GET",
 209                         Host:       u.Host,
 210                         URL:        u,
 211                         RequestURI: u.RequestURI(),
 212                         Header:     http.Header{},
 213                 }
 214                 failCode := authz(req, tok)
 215                 req, resp := s.doReq(req)
 216                 code, body := resp.Code, resp.Body.String()
 217
 218                 // If the initial request had a (non-empty) token
 219                 // showing in the query string, we should have been
 220                 // redirected in order to hide it in a cookie.
 221                 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
 222
 223                 if tok == arvadostest.ActiveToken {
 224                         c.Check(code, check.Equals, http.StatusOK)
 225                         c.Check(body, check.Equals, "foo")
 226
 227                 } else {
 228                         c.Check(code >= 400, check.Equals, true)
 229                         c.Check(code < 500, check.Equals, true)
 230                         if tok == arvadostest.SpectatorToken {
 231                                 // Valid token never offers to retry
 232                                 // with different credentials.
 233                                 c.Check(code, check.Equals, http.StatusNotFound)
 234                         } else {
 235                                 // Invalid token can ask to retry
 236                                 // depending on the authz method.
 237                                 c.Check(code, check.Equals, failCode)
 238                         }
 239                         c.Check(body, check.Equals, "")
 240                 }
 241         }
 242 }
 243
 244 func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
 245         resp := httptest.NewRecorder()
 246         s.testServer.Handler.ServeHTTP(resp, req)
 247         if resp.Code != http.StatusSeeOther {
 248                 return req, resp
 249         }
 250         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 251         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 252         req = &http.Request{
 253                 Method:     "GET",
 254                 Host:       u.Host,
 255                 URL:        u,
 256                 RequestURI: u.RequestURI(),
 257                 Header:     http.Header{},
 258         }
 259         for _, c := range cookies {
 260                 req.AddCookie(c)
 261         }
 262         return s.doReq(req)
 263 }
 264
 265 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
 266         s.testVhostRedirectTokenToCookie(c, "GET",
 267                 arvadostest.FooCollection+".example.com/foo",
 268                 "?api_token="+arvadostest.ActiveToken,
 269                 "",
 270                 "",
 271                 http.StatusOK,
 272                 "foo",
 273         )
 274 }
 275
 276 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
 277         s.testVhostRedirectTokenToCookie(c, "GET",
 278                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
 279                 "",
 280                 "",
 281                 "",
 282                 http.StatusOK,
 283                 "foo",
 284         )
 285 }
 286
 287 // Bad token in URL is 404 Not Found because it doesn't make sense to
 288 // retry the same URL with different authorization.
 289 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
 290         s.testVhostRedirectTokenToCookie(c, "GET",
 291                 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
 292                 "",
 293                 "",
 294                 "",
 295                 http.StatusNotFound,
 296                 "",
 297         )
 298 }
 299
 300 // Bad token in a cookie (even if it got there via our own
 301 // query-string-to-cookie redirect) is, in principle, retryable at the
 302 // same URL so it's 401 Unauthorized.
 303 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
 304         s.testVhostRedirectTokenToCookie(c, "GET",
 305                 arvadostest.FooCollection+".example.com/foo",
 306                 "?api_token=thisisabogustoken",
 307                 "",
 308                 "",
 309                 http.StatusUnauthorized,
 310                 "",
 311         )
 312 }
 313
 314 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
 315         s.testVhostRedirectTokenToCookie(c, "GET",
 316                 "example.com/c="+arvadostest.FooCollection+"/foo",
 317                 "?api_token="+arvadostest.ActiveToken,
 318                 "",
 319                 "",
 320                 http.StatusBadRequest,
 321                 "",
 322         )
 323 }
 324
 325 // If client requests an attachment by putting ?disposition=attachment
 326 // in the query string, and gets redirected, the redirect target
 327 // should respond with an attachment.
 328 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
 329         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 330                 arvadostest.FooCollection+".example.com/foo",
 331                 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
 332                 "",
 333                 "",
 334                 http.StatusOK,
 335                 "foo",
 336         )
 337         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 338 }
 339
 340 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
 341         s.testServer.Config.AttachmentOnlyHost = "download.example.com"
 342         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 343                 "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
 344                 "?api_token="+arvadostest.ActiveToken,
 345                 "",
 346                 "",
 347                 http.StatusOK,
 348                 "foo",
 349         )
 350         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 351 }
 352
 353 func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
 354         s.testServer.Config.AttachmentOnlyHost = "download.example.com"
 355         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 356                 "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
 357                 "?api_token="+arvadostest.ActiveToken,
 358                 "",
 359                 "",
 360                 http.StatusOK,
 361                 "waz",
 362         )
 363         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 364         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 365                 "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz",
 366                 "?api_token="+arvadostest.ActiveToken,
 367                 "",
 368                 "",
 369                 http.StatusOK,
 370                 "waz",
 371         )
 372         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 373 }
 374
 375 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
 376         s.testServer.Config.TrustAllContent = true
 377         s.testVhostRedirectTokenToCookie(c, "GET",
 378                 "example.com/c="+arvadostest.FooCollection+"/foo",
 379                 "?api_token="+arvadostest.ActiveToken,
 380                 "",
 381                 "",
 382                 http.StatusOK,
 383                 "foo",
 384         )
 385 }
 386
 387 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
 388         s.testServer.Config.AttachmentOnlyHost = "example.com:1234"
 389
 390         s.testVhostRedirectTokenToCookie(c, "GET",
 391                 "example.com/c="+arvadostest.FooCollection+"/foo",
 392                 "?api_token="+arvadostest.ActiveToken,
 393                 "",
 394                 "",
 395                 http.StatusBadRequest,
 396                 "",
 397         )
 398
 399         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 400                 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
 401                 "?api_token="+arvadostest.ActiveToken,
 402                 "",
 403                 "",
 404                 http.StatusOK,
 405                 "foo",
 406         )
 407         c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
 408 }
 409
 410 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
 411         s.testVhostRedirectTokenToCookie(c, "POST",
 412                 arvadostest.FooCollection+".example.com/foo",
 413                 "",
 414                 "application/x-www-form-urlencoded",
 415                 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
 416                 http.StatusOK,
 417                 "foo",
 418         )
 419 }
 420
 421 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
 422         s.testVhostRedirectTokenToCookie(c, "POST",
 423                 arvadostest.FooCollection+".example.com/foo",
 424                 "",
 425                 "application/x-www-form-urlencoded",
 426                 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
 427                 http.StatusNotFound,
 428                 "",
 429         )
 430 }
 431
 432 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
 433         s.testServer.Config.AnonymousTokens = []string{arvadostest.AnonymousToken}
 434         s.testVhostRedirectTokenToCookie(c, "GET",
 435                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 436                 "",
 437                 "",
 438                 "",
 439                 http.StatusOK,
 440                 "Hello world\n",
 441         )
 442 }
 443
 444 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
 445         s.testServer.Config.AnonymousTokens = []string{"anonymousTokenConfiguredButInvalid"}
 446         s.testVhostRedirectTokenToCookie(c, "GET",
 447                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 448                 "",
 449                 "",
 450                 "",
 451                 http.StatusNotFound,
 452                 "",
 453         )
 454 }
 455
 456 func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
 457         s.testServer.Config.AttachmentOnlyHost = "download.example.com"
 458
 459         client := s.testServer.Config.Client
 460         client.AuthToken = arvadostest.ActiveToken
 461         fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
 462         c.Assert(err, check.IsNil)
 463         f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777)
 464         c.Assert(err, check.IsNil)
 465         f.Close()
 466         mtxt, err := fs.MarshalManifest(".")
 467         c.Assert(err, check.IsNil)
 468         coll := arvados.Collection{ManifestText: mtxt}
 469         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", client.UpdateBody(coll), nil)
 470         c.Assert(err, check.IsNil)
 471
 472         u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
 473         req := &http.Request{
 474                 Method:     "GET",
 475                 Host:       u.Host,
 476                 URL:        u,
 477                 RequestURI: u.RequestURI(),
 478                 Header: http.Header{
 479                         "Authorization": {"Bearer " + client.AuthToken},
 480                 },
 481         }
 482         resp := httptest.NewRecorder()
 483         s.testServer.Handler.ServeHTTP(resp, req)
 484         c.Check(resp.Code, check.Equals, http.StatusOK)
 485         c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\&#34;odd&#39; path chars.*`)
 486 }
 487
 488 // XHRs can't follow redirect-with-cookie so they rely on method=POST
 489 // and disposition=attachment (telling us it's acceptable to respond
 490 // with content instead of a redirect) and an Origin header that gets
 491 // added automatically by the browser (telling us it's desirable to do
 492 // so).
 493 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
 494         u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
 495         req := &http.Request{
 496                 Method:     "POST",
 497                 Host:       u.Host,
 498                 URL:        u,
 499                 RequestURI: u.RequestURI(),
 500                 Header: http.Header{
 501                         "Origin":       {"https://origin.example"},
 502                         "Content-Type": {"application/x-www-form-urlencoded"},
 503                 },
 504                 Body: ioutil.NopCloser(strings.NewReader(url.Values{
 505                         "api_token":   {arvadostest.ActiveToken},
 506                         "disposition": {"attachment"},
 507                 }.Encode())),
 508         }
 509         resp := httptest.NewRecorder()
 510         s.testServer.Handler.ServeHTTP(resp, req)
 511         c.Check(resp.Code, check.Equals, http.StatusOK)
 512         c.Check(resp.Body.String(), check.Equals, "foo")
 513         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
 514 }
 515
 516 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder {
 517         u, _ := url.Parse(`http://` + hostPath + queryString)
 518         req := &http.Request{
 519                 Method:     method,
 520                 Host:       u.Host,
 521                 URL:        u,
 522                 RequestURI: u.RequestURI(),
 523                 Header:     http.Header{"Content-Type": {contentType}},
 524                 Body:       ioutil.NopCloser(strings.NewReader(reqBody)),
 525         }
 526
 527         resp := httptest.NewRecorder()
 528         defer func() {
 529                 c.Check(resp.Code, check.Equals, expectStatus)
 530                 c.Check(resp.Body.String(), check.Equals, expectRespBody)
 531         }()
 532
 533         s.testServer.Handler.ServeHTTP(resp, req)
 534         if resp.Code != http.StatusSeeOther {
 535                 return resp
 536         }
 537         c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
 538         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 539
 540         u, _ = u.Parse(resp.Header().Get("Location"))
 541         req = &http.Request{
 542                 Method:     "GET",
 543                 Host:       u.Host,
 544                 URL:        u,
 545                 RequestURI: u.RequestURI(),
 546                 Header:     http.Header{},
 547         }
 548         for _, c := range cookies {
 549                 req.AddCookie(c)
 550         }
 551
 552         resp = httptest.NewRecorder()
 553         s.testServer.Handler.ServeHTTP(resp, req)
 554         c.Check(resp.Header().Get("Location"), check.Equals, "")
 555         return resp
 556 }
 557
 558 func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
 559         s.testServer.Config.AttachmentOnlyHost = "download.example.com"
 560         authHeader := http.Header{
 561                 "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
 562         }
 563         for _, trial := range []struct {
 564                 uri      string
 565                 header   http.Header
 566                 expect   []string
 567                 redirect string
 568                 cutDirs  int
 569         }{
 570                 {
 571                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
 572                         header:  authHeader,
 573                         expect:  []string{"dir1/foo", "dir1/bar"},
 574                         cutDirs: 0,
 575                 },
 576                 {
 577                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
 578                         header:  authHeader,
 579                         expect:  []string{"foo", "bar"},
 580                         cutDirs: 1,
 581                 },
 582                 {
 583                         uri:     "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
 584                         header:  authHeader,
 585                         expect:  []string{"dir1/foo", "dir1/bar"},
 586                         cutDirs: 2,
 587                 },
 588                 {
 589                         uri:     "download.example.com/users/active/foo_file_in_dir/",
 590                         header:  authHeader,
 591                         expect:  []string{"dir1/"},
 592                         cutDirs: 3,
 593                 },
 594                 {
 595                         uri:     "download.example.com/users/active/foo_file_in_dir/dir1/",
 596                         header:  authHeader,
 597                         expect:  []string{"bar"},
 598                         cutDirs: 4,
 599                 },
 600                 {
 601                         uri:     "download.example.com/",
 602                         header:  authHeader,
 603                         expect:  []string{"users/"},
 604                         cutDirs: 0,
 605                 },
 606                 {
 607                         uri:      "download.example.com/users",
 608                         header:   authHeader,
 609                         redirect: "/users/",
 610                         expect:   []string{"active/"},
 611                         cutDirs:  1,
 612                 },
 613                 {
 614                         uri:     "download.example.com/users/",
 615                         header:  authHeader,
 616                         expect:  []string{"active/"},
 617                         cutDirs: 1,
 618                 },
 619                 {
 620                         uri:      "download.example.com/users/active",
 621                         header:   authHeader,
 622                         redirect: "/users/active/",
 623                         expect:   []string{"foo_file_in_dir/"},
 624                         cutDirs:  2,
 625                 },
 626                 {
 627                         uri:     "download.example.com/users/active/",
 628                         header:  authHeader,
 629                         expect:  []string{"foo_file_in_dir/"},
 630                         cutDirs: 2,
 631                 },
 632                 {
 633                         uri:     "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
 634                         header:  nil,
 635                         expect:  []string{"dir1/foo", "dir1/bar"},
 636                         cutDirs: 4,
 637                 },
 638                 {
 639                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
 640                         header:  nil,
 641                         expect:  []string{"dir1/foo", "dir1/bar"},
 642                         cutDirs: 2,
 643                 },
 644                 {
 645                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken,
 646                         header:  nil,
 647                         expect:  []string{"dir1/foo", "dir1/bar"},
 648                         cutDirs: 2,
 649                 },
 650                 {
 651                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID,
 652                         header:  authHeader,
 653                         expect:  []string{"dir1/foo", "dir1/bar"},
 654                         cutDirs: 1,
 655                 },
 656                 {
 657                         uri:      "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
 658                         header:   authHeader,
 659                         redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
 660                         expect:   []string{"foo", "bar"},
 661                         cutDirs:  2,
 662                 },
 663                 {
 664                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
 665                         header:  authHeader,
 666                         expect:  []string{"foo", "bar"},
 667                         cutDirs: 3,
 668                 },
 669                 {
 670                         uri:      arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
 671                         header:   authHeader,
 672                         redirect: "/dir1/",
 673                         expect:   []string{"foo", "bar"},
 674                         cutDirs:  1,
 675                 },
 676                 {
 677                         uri:    "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
 678                         header: authHeader,
 679                         expect: nil,
 680                 },
 681                 {
 682                         uri:     "download.example.com/c=" + arvadostest.WazVersion1Collection,
 683                         header:  authHeader,
 684                         expect:  []string{"waz"},
 685                         cutDirs: 1,
 686                 },
 687                 {
 688                         uri:     "download.example.com/by_id/" + arvadostest.WazVersion1Collection,
 689                         header:  authHeader,
 690                         expect:  []string{"waz"},
 691                         cutDirs: 2,
 692                 },
 693         } {
 694                 c.Logf("HTML: %q => %q", trial.uri, trial.expect)
 695                 resp := httptest.NewRecorder()
 696                 u := mustParseURL("//" + trial.uri)
 697                 req := &http.Request{
 698                         Method:     "GET",
 699                         Host:       u.Host,
 700                         URL:        u,
 701                         RequestURI: u.RequestURI(),
 702                         Header:     copyHeader(trial.header),
 703                 }
 704                 s.testServer.Handler.ServeHTTP(resp, req)
 705                 var cookies []*http.Cookie
 706                 for resp.Code == http.StatusSeeOther {
 707                         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 708                         req = &http.Request{
 709                                 Method:     "GET",
 710                                 Host:       u.Host,
 711                                 URL:        u,
 712                                 RequestURI: u.RequestURI(),
 713                                 Header:     copyHeader(trial.header),
 714                         }
 715                         cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
 716                         for _, c := range cookies {
 717                                 req.AddCookie(c)
 718                         }
 719                         resp = httptest.NewRecorder()
 720                         s.testServer.Handler.ServeHTTP(resp, req)
 721                 }
 722                 if trial.redirect != "" {
 723                         c.Check(req.URL.Path, check.Equals, trial.redirect)
 724                 }
 725                 if trial.expect == nil {
 726                         c.Check(resp.Code, check.Equals, http.StatusNotFound)
 727                 } else {
 728                         c.Check(resp.Code, check.Equals, http.StatusOK)
 729                         for _, e := range trial.expect {
 730                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`)
 731                         }
 732                         c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`)
 733                 }
 734
 735                 c.Logf("WebDAV: %q => %q", trial.uri, trial.expect)
 736                 req = &http.Request{
 737                         Method:     "OPTIONS",
 738                         Host:       u.Host,
 739                         URL:        u,
 740                         RequestURI: u.RequestURI(),
 741                         Header:     copyHeader(trial.header),
 742                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
 743                 }
 744                 resp = httptest.NewRecorder()
 745                 s.testServer.Handler.ServeHTTP(resp, req)
 746                 if trial.expect == nil {
 747                         c.Check(resp.Code, check.Equals, http.StatusNotFound)
 748                 } else {
 749                         c.Check(resp.Code, check.Equals, http.StatusOK)
 750                 }
 751
 752                 req = &http.Request{
 753                         Method:     "PROPFIND",
 754                         Host:       u.Host,
 755                         URL:        u,
 756                         RequestURI: u.RequestURI(),
 757                         Header:     copyHeader(trial.header),
 758                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
 759                 }
 760                 resp = httptest.NewRecorder()
 761                 s.testServer.Handler.ServeHTTP(resp, req)
 762                 if trial.expect == nil {
 763                         c.Check(resp.Code, check.Equals, http.StatusNotFound)
 764                 } else {
 765                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus)
 766                         for _, e := range trial.expect {
 767                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+filepath.Join(u.Path, e)+`</D:href>.*`)
 768                         }
 769                 }
 770         }
 771 }
 772
 773 func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
 774         arv := arvados.NewClientFromEnv()
 775         var newCollection arvados.Collection
 776         err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", arv.UpdateBody(&arvados.Collection{
 777                 OwnerUUID:    arvadostest.ActiveUserUUID,
 778                 ManifestText: ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
 779                 Name:         "keep-web test collection",
 780         }), map[string]bool{"ensure_unique_name": true})
 781         c.Assert(err, check.IsNil)
 782         defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
 783
 784         var updated arvados.Collection
 785         for _, fnm := range []string{"foo.txt", "bar.txt"} {
 786                 s.testServer.Config.AttachmentOnlyHost = "example.com"
 787                 u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
 788                 req := &http.Request{
 789                         Method:     "DELETE",
 790                         Host:       u.Host,
 791                         URL:        u,
 792                         RequestURI: u.RequestURI(),
 793                         Header: http.Header{
 794                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
 795                         },
 796                 }
 797                 resp := httptest.NewRecorder()
 798                 s.testServer.Handler.ServeHTTP(resp, req)
 799                 c.Check(resp.Code, check.Equals, http.StatusNoContent)
 800
 801                 updated = arvados.Collection{}
 802                 err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
 803                 c.Check(err, check.IsNil)
 804                 c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`)
 805                 c.Logf("updated manifest_text %q", updated.ManifestText)
 806         }
 807         c.Check(updated.ManifestText, check.Equals, "")
 808 }
 809
 810 func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) {
 811         s.testServer.Config.ManagementToken = arvadostest.ManagementToken
 812         authHeader := http.Header{
 813                 "Authorization": {"Bearer " + arvadostest.ManagementToken},
 814         }
 815
 816         resp := httptest.NewRecorder()
 817         u := mustParseURL("http://download.example.com/_health/ping")
 818         req := &http.Request{
 819                 Method:     "GET",
 820                 Host:       u.Host,
 821                 URL:        u,
 822                 RequestURI: u.RequestURI(),
 823                 Header:     authHeader,
 824         }
 825         s.testServer.Handler.ServeHTTP(resp, req)
 826
 827         c.Check(resp.Code, check.Equals, http.StatusOK)
 828         c.Check(resp.Body.String(), check.Matches, `{"health":"OK"}\n`)
 829 }
 830
 831 func copyHeader(h http.Header) http.Header {
 832         hc := http.Header{}
 833         for k, v := range h {
 834                 hc[k] = append([]string(nil), v...)
 835         }
 836         return hc
 837 }