services/keep-web/handler_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "fmt"
  10         "html"
  11         "io/ioutil"
  12         "net/http"
  13         "net/http/httptest"
  14         "net/url"
  15         "os"
  16         "path/filepath"
  17         "regexp"
  18         "strings"
  19
  20         "git.curoverse.com/arvados.git/sdk/go/arvados"
  21         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  22         "git.curoverse.com/arvados.git/sdk/go/auth"
  23         check "gopkg.in/check.v1"
  24 )
  25
  26 var _ = check.Suite(&UnitSuite{})
  27
  28 type UnitSuite struct{}
  29
  30 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
  31         h := handler{Config: DefaultConfig()}
  32         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
  33         req := &http.Request{
  34                 Method:     "OPTIONS",
  35                 Host:       u.Host,
  36                 URL:        u,
  37                 RequestURI: u.RequestURI(),
  38                 Header: http.Header{
  39                         "Origin":                        {"https://workbench.example"},
  40                         "Access-Control-Request-Method": {"POST"},
  41                 },
  42         }
  43
  44         // Check preflight for an allowed request
  45         resp := httptest.NewRecorder()
  46         h.ServeHTTP(resp, req)
  47         c.Check(resp.Code, check.Equals, http.StatusOK)
  48         c.Check(resp.Body.String(), check.Equals, "")
  49         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
  50         c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
  51         c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout")
  52
  53         // Check preflight for a disallowed request
  54         resp = httptest.NewRecorder()
  55         req.Header.Set("Access-Control-Request-Method", "MAKE-COFFEE")
  56         h.ServeHTTP(resp, req)
  57         c.Check(resp.Body.String(), check.Equals, "")
  58         c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
  59 }
  60
  61 func (s *UnitSuite) TestInvalidUUID(c *check.C) {
  62         bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-"
  63         token := arvadostest.ActiveToken
  64         for _, trial := range []string{
  65                 "http://keep-web/c=" + bogusID + "/foo",
  66                 "http://keep-web/c=" + bogusID + "/t=" + token + "/foo",
  67                 "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo",
  68                 "http://keep-web/collections/" + bogusID + "/foo",
  69                 "http://" + bogusID + ".keep-web/" + bogusID + "/foo",
  70                 "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
  71         } {
  72                 c.Log(trial)
  73                 u := mustParseURL(trial)
  74                 req := &http.Request{
  75                         Method:     "GET",
  76                         Host:       u.Host,
  77                         URL:        u,
  78                         RequestURI: u.RequestURI(),
  79                 }
  80                 resp := httptest.NewRecorder()
  81                 cfg := DefaultConfig()
  82                 cfg.AnonymousTokens = []string{arvadostest.AnonymousToken}
  83                 h := handler{Config: cfg}
  84                 h.ServeHTTP(resp, req)
  85                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
  86         }
  87 }
  88
  89 func mustParseURL(s string) *url.URL {
  90         r, err := url.Parse(s)
  91         if err != nil {
  92                 panic("parse URL: " + s)
  93         }
  94         return r
  95 }
  96
  97 func (s *IntegrationSuite) TestVhost404(c *check.C) {
  98         for _, testURL := range []string{
  99                 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
 100                 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
 101         } {
 102                 resp := httptest.NewRecorder()
 103                 u := mustParseURL(testURL)
 104                 req := &http.Request{
 105                         Method:     "GET",
 106                         URL:        u,
 107                         RequestURI: u.RequestURI(),
 108                 }
 109                 s.testServer.Handler.ServeHTTP(resp, req)
 110                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 111                 c.Check(resp.Body.String(), check.Equals, "")
 112         }
 113 }
 114
 115 // An authorizer modifies an HTTP request to make use of the given
 116 // token -- by adding it to a header, cookie, query param, or whatever
 117 // -- and returns the HTTP status code we should expect from keep-web if
 118 // the token is invalid.
 119 type authorizer func(*http.Request, string) int
 120
 121 func (s *IntegrationSuite) TestVhostViaAuthzHeader(c *check.C) {
 122         s.doVhostRequests(c, authzViaAuthzHeader)
 123 }
 124 func authzViaAuthzHeader(r *http.Request, tok string) int {
 125         r.Header.Add("Authorization", "OAuth2 "+tok)
 126         return http.StatusUnauthorized
 127 }
 128
 129 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
 130         s.doVhostRequests(c, authzViaCookieValue)
 131 }
 132 func authzViaCookieValue(r *http.Request, tok string) int {
 133         r.AddCookie(&http.Cookie{
 134                 Name:  "arvados_api_token",
 135                 Value: auth.EncodeTokenCookie([]byte(tok)),
 136         })
 137         return http.StatusUnauthorized
 138 }
 139
 140 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
 141         s.doVhostRequests(c, authzViaPath)
 142 }
 143 func authzViaPath(r *http.Request, tok string) int {
 144         r.URL.Path = "/t=" + tok + r.URL.Path
 145         return http.StatusNotFound
 146 }
 147
 148 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
 149         s.doVhostRequests(c, authzViaQueryString)
 150 }
 151 func authzViaQueryString(r *http.Request, tok string) int {
 152         r.URL.RawQuery = "api_token=" + tok
 153         return http.StatusUnauthorized
 154 }
 155
 156 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
 157         s.doVhostRequests(c, authzViaPOST)
 158 }
 159 func authzViaPOST(r *http.Request, tok string) int {
 160         r.Method = "POST"
 161         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 162         r.Body = ioutil.NopCloser(strings.NewReader(
 163                 url.Values{"api_token": {tok}}.Encode()))
 164         return http.StatusUnauthorized
 165 }
 166
 167 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
 168         s.doVhostRequests(c, authzViaPOST)
 169 }
 170 func authzViaXHRPOST(r *http.Request, tok string) int {
 171         r.Method = "POST"
 172         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 173         r.Header.Add("Origin", "https://origin.example")
 174         r.Body = ioutil.NopCloser(strings.NewReader(
 175                 url.Values{
 176                         "api_token":   {tok},
 177                         "disposition": {"attachment"},
 178                 }.Encode()))
 179         return http.StatusUnauthorized
 180 }
 181
 182 // Try some combinations of {url, token} using the given authorization
 183 // mechanism, and verify the result is correct.
 184 func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
 185         for _, hostPath := range []string{
 186                 arvadostest.FooCollection + ".example.com/foo",
 187                 arvadostest.FooCollection + "--collections.example.com/foo",
 188                 arvadostest.FooCollection + "--collections.example.com/_/foo",
 189                 arvadostest.FooCollectionPDH + ".example.com/foo",
 190                 strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + "--collections.example.com/foo",
 191                 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
 192         } {
 193                 c.Log("doRequests: ", hostPath)
 194                 s.doVhostRequestsWithHostPath(c, authz, hostPath)
 195         }
 196 }
 197
 198 func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
 199         for _, tok := range []string{
 200                 arvadostest.ActiveToken,
 201                 arvadostest.ActiveToken[:15],
 202                 arvadostest.SpectatorToken,
 203                 "bogus",
 204                 "",
 205         } {
 206                 u := mustParseURL("http://" + hostPath)
 207                 req := &http.Request{
 208                         Method:     "GET",
 209                         Host:       u.Host,
 210                         URL:        u,
 211                         RequestURI: u.RequestURI(),
 212                         Header:     http.Header{},
 213                 }
 214                 failCode := authz(req, tok)
 215                 req, resp := s.doReq(req)
 216                 code, body := resp.Code, resp.Body.String()
 217
 218                 // If the initial request had a (non-empty) token
 219                 // showing in the query string, we should have been
 220                 // redirected in order to hide it in a cookie.
 221                 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
 222
 223                 if tok == arvadostest.ActiveToken {
 224                         c.Check(code, check.Equals, http.StatusOK)
 225                         c.Check(body, check.Equals, "foo")
 226
 227                 } else {
 228                         c.Check(code >= 400, check.Equals, true)
 229                         c.Check(code < 500, check.Equals, true)
 230                         if tok == arvadostest.SpectatorToken {
 231                                 // Valid token never offers to retry
 232                                 // with different credentials.
 233                                 c.Check(code, check.Equals, http.StatusNotFound)
 234                         } else {
 235                                 // Invalid token can ask to retry
 236                                 // depending on the authz method.
 237                                 c.Check(code, check.Equals, failCode)
 238                         }
 239                         c.Check(body, check.Equals, "")
 240                 }
 241         }
 242 }
 243
 244 func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
 245         resp := httptest.NewRecorder()
 246         s.testServer.Handler.ServeHTTP(resp, req)
 247         if resp.Code != http.StatusSeeOther {
 248                 return req, resp
 249         }
 250         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 251         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 252         req = &http.Request{
 253                 Method:     "GET",
 254                 Host:       u.Host,
 255                 URL:        u,
 256                 RequestURI: u.RequestURI(),
 257                 Header:     http.Header{},
 258         }
 259         for _, c := range cookies {
 260                 req.AddCookie(c)
 261         }
 262         return s.doReq(req)
 263 }
 264
 265 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
 266         s.testVhostRedirectTokenToCookie(c, "GET",
 267                 arvadostest.FooCollection+".example.com/foo",
 268                 "?api_token="+arvadostest.ActiveToken,
 269                 "",
 270                 "",
 271                 http.StatusOK,
 272                 "foo",
 273         )
 274 }
 275
 276 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
 277         s.testVhostRedirectTokenToCookie(c, "GET",
 278                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
 279                 "",
 280                 "",
 281                 "",
 282                 http.StatusOK,
 283                 "foo",
 284         )
 285 }
 286
 287 // Bad token in URL is 404 Not Found because it doesn't make sense to
 288 // retry the same URL with different authorization.
 289 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
 290         s.testVhostRedirectTokenToCookie(c, "GET",
 291                 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
 292                 "",
 293                 "",
 294                 "",
 295                 http.StatusNotFound,
 296                 "",
 297         )
 298 }
 299
 300 // Bad token in a cookie (even if it got there via our own
 301 // query-string-to-cookie redirect) is, in principle, retryable at the
 302 // same URL so it's 401 Unauthorized.
 303 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
 304         s.testVhostRedirectTokenToCookie(c, "GET",
 305                 arvadostest.FooCollection+".example.com/foo",
 306                 "?api_token=thisisabogustoken",
 307                 "",
 308                 "",
 309                 http.StatusUnauthorized,
 310                 "",
 311         )
 312 }
 313
 314 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
 315         s.testVhostRedirectTokenToCookie(c, "GET",
 316                 "example.com/c="+arvadostest.FooCollection+"/foo",
 317                 "?api_token="+arvadostest.ActiveToken,
 318                 "",
 319                 "",
 320                 http.StatusBadRequest,
 321                 "",
 322         )
 323 }
 324
 325 // If client requests an attachment by putting ?disposition=attachment
 326 // in the query string, and gets redirected, the redirect target
 327 // should respond with an attachment.
 328 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
 329         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 330                 arvadostest.FooCollection+".example.com/foo",
 331                 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
 332                 "",
 333                 "",
 334                 http.StatusOK,
 335                 "foo",
 336         )
 337         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 338 }
 339
 340 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
 341         s.testServer.Config.AttachmentOnlyHost = "download.example.com"
 342         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 343                 "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
 344                 "?api_token="+arvadostest.ActiveToken,
 345                 "",
 346                 "",
 347                 http.StatusOK,
 348                 "foo",
 349         )
 350         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 351 }
 352
 353 func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
 354         s.testServer.Config.AttachmentOnlyHost = "download.example.com"
 355         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 356                 "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
 357                 "?api_token="+arvadostest.ActiveToken,
 358                 "",
 359                 "",
 360                 http.StatusOK,
 361                 "waz",
 362         )
 363         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 364         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 365                 "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz",
 366                 "?api_token="+arvadostest.ActiveToken,
 367                 "",
 368                 "",
 369                 http.StatusOK,
 370                 "waz",
 371         )
 372         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 373 }
 374
 375 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
 376         s.testServer.Config.TrustAllContent = true
 377         s.testVhostRedirectTokenToCookie(c, "GET",
 378                 "example.com/c="+arvadostest.FooCollection+"/foo",
 379                 "?api_token="+arvadostest.ActiveToken,
 380                 "",
 381                 "",
 382                 http.StatusOK,
 383                 "foo",
 384         )
 385 }
 386
 387 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
 388         s.testServer.Config.AttachmentOnlyHost = "example.com:1234"
 389
 390         s.testVhostRedirectTokenToCookie(c, "GET",
 391                 "example.com/c="+arvadostest.FooCollection+"/foo",
 392                 "?api_token="+arvadostest.ActiveToken,
 393                 "",
 394                 "",
 395                 http.StatusBadRequest,
 396                 "",
 397         )
 398
 399         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 400                 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
 401                 "?api_token="+arvadostest.ActiveToken,
 402                 "",
 403                 "",
 404                 http.StatusOK,
 405                 "foo",
 406         )
 407         c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
 408 }
 409
 410 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
 411         s.testVhostRedirectTokenToCookie(c, "POST",
 412                 arvadostest.FooCollection+".example.com/foo",
 413                 "",
 414                 "application/x-www-form-urlencoded",
 415                 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
 416                 http.StatusOK,
 417                 "foo",
 418         )
 419 }
 420
 421 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
 422         s.testVhostRedirectTokenToCookie(c, "POST",
 423                 arvadostest.FooCollection+".example.com/foo",
 424                 "",
 425                 "application/x-www-form-urlencoded",
 426                 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
 427                 http.StatusNotFound,
 428                 "",
 429         )
 430 }
 431
 432 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
 433         s.testServer.Config.AnonymousTokens = []string{arvadostest.AnonymousToken}
 434         s.testVhostRedirectTokenToCookie(c, "GET",
 435                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 436                 "",
 437                 "",
 438                 "",
 439                 http.StatusOK,
 440                 "Hello world\n",
 441         )
 442 }
 443
 444 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
 445         s.testServer.Config.AnonymousTokens = []string{"anonymousTokenConfiguredButInvalid"}
 446         s.testVhostRedirectTokenToCookie(c, "GET",
 447                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 448                 "",
 449                 "",
 450                 "",
 451                 http.StatusNotFound,
 452                 "",
 453         )
 454 }
 455
 456 func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
 457         s.testServer.Config.AttachmentOnlyHost = "download.example.com"
 458
 459         client := s.testServer.Config.Client
 460         client.AuthToken = arvadostest.ActiveToken
 461         fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
 462         c.Assert(err, check.IsNil)
 463         f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777)
 464         c.Assert(err, check.IsNil)
 465         f.Close()
 466         mtxt, err := fs.MarshalManifest(".")
 467         c.Assert(err, check.IsNil)
 468         var coll arvados.Collection
 469         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 470                 "collection": map[string]string{
 471                         "manifest_text": mtxt,
 472                 },
 473         })
 474         c.Assert(err, check.IsNil)
 475
 476         u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
 477         req := &http.Request{
 478                 Method:     "GET",
 479                 Host:       u.Host,
 480                 URL:        u,
 481                 RequestURI: u.RequestURI(),
 482                 Header: http.Header{
 483                         "Authorization": {"Bearer " + client.AuthToken},
 484                 },
 485         }
 486         resp := httptest.NewRecorder()
 487         s.testServer.Handler.ServeHTTP(resp, req)
 488         c.Check(resp.Code, check.Equals, http.StatusOK)
 489         c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\&#34;odd&#39; path chars.*`)
 490 }
 491
 492 // XHRs can't follow redirect-with-cookie so they rely on method=POST
 493 // and disposition=attachment (telling us it's acceptable to respond
 494 // with content instead of a redirect) and an Origin header that gets
 495 // added automatically by the browser (telling us it's desirable to do
 496 // so).
 497 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
 498         u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
 499         req := &http.Request{
 500                 Method:     "POST",
 501                 Host:       u.Host,
 502                 URL:        u,
 503                 RequestURI: u.RequestURI(),
 504                 Header: http.Header{
 505                         "Origin":       {"https://origin.example"},
 506                         "Content-Type": {"application/x-www-form-urlencoded"},
 507                 },
 508                 Body: ioutil.NopCloser(strings.NewReader(url.Values{
 509                         "api_token":   {arvadostest.ActiveToken},
 510                         "disposition": {"attachment"},
 511                 }.Encode())),
 512         }
 513         resp := httptest.NewRecorder()
 514         s.testServer.Handler.ServeHTTP(resp, req)
 515         c.Check(resp.Code, check.Equals, http.StatusOK)
 516         c.Check(resp.Body.String(), check.Equals, "foo")
 517         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
 518 }
 519
 520 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder {
 521         u, _ := url.Parse(`http://` + hostPath + queryString)
 522         req := &http.Request{
 523                 Method:     method,
 524                 Host:       u.Host,
 525                 URL:        u,
 526                 RequestURI: u.RequestURI(),
 527                 Header:     http.Header{"Content-Type": {contentType}},
 528                 Body:       ioutil.NopCloser(strings.NewReader(reqBody)),
 529         }
 530
 531         resp := httptest.NewRecorder()
 532         defer func() {
 533                 c.Check(resp.Code, check.Equals, expectStatus)
 534                 c.Check(resp.Body.String(), check.Equals, expectRespBody)
 535         }()
 536
 537         s.testServer.Handler.ServeHTTP(resp, req)
 538         if resp.Code != http.StatusSeeOther {
 539                 return resp
 540         }
 541         c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
 542         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 543
 544         u, _ = u.Parse(resp.Header().Get("Location"))
 545         req = &http.Request{
 546                 Method:     "GET",
 547                 Host:       u.Host,
 548                 URL:        u,
 549                 RequestURI: u.RequestURI(),
 550                 Header:     http.Header{},
 551         }
 552         for _, c := range cookies {
 553                 req.AddCookie(c)
 554         }
 555
 556         resp = httptest.NewRecorder()
 557         s.testServer.Handler.ServeHTTP(resp, req)
 558         c.Check(resp.Header().Get("Location"), check.Equals, "")
 559         return resp
 560 }
 561
 562 func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
 563         s.testServer.Config.AttachmentOnlyHost = "download.example.com"
 564         authHeader := http.Header{
 565                 "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
 566         }
 567         for _, trial := range []struct {
 568                 uri      string
 569                 header   http.Header
 570                 expect   []string
 571                 redirect string
 572                 cutDirs  int
 573         }{
 574                 {
 575                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
 576                         header:  authHeader,
 577                         expect:  []string{"dir1/foo", "dir1/bar"},
 578                         cutDirs: 0,
 579                 },
 580                 {
 581                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
 582                         header:  authHeader,
 583                         expect:  []string{"foo", "bar"},
 584                         cutDirs: 1,
 585                 },
 586                 {
 587                         uri:     "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
 588                         header:  authHeader,
 589                         expect:  []string{"dir1/foo", "dir1/bar"},
 590                         cutDirs: 2,
 591                 },
 592                 {
 593                         uri:     "download.example.com/users/active/foo_file_in_dir/",
 594                         header:  authHeader,
 595                         expect:  []string{"dir1/"},
 596                         cutDirs: 3,
 597                 },
 598                 {
 599                         uri:     "download.example.com/users/active/foo_file_in_dir/dir1/",
 600                         header:  authHeader,
 601                         expect:  []string{"bar"},
 602                         cutDirs: 4,
 603                 },
 604                 {
 605                         uri:     "download.example.com/",
 606                         header:  authHeader,
 607                         expect:  []string{"users/"},
 608                         cutDirs: 0,
 609                 },
 610                 {
 611                         uri:      "download.example.com/users",
 612                         header:   authHeader,
 613                         redirect: "/users/",
 614                         expect:   []string{"active/"},
 615                         cutDirs:  1,
 616                 },
 617                 {
 618                         uri:     "download.example.com/users/",
 619                         header:  authHeader,
 620                         expect:  []string{"active/"},
 621                         cutDirs: 1,
 622                 },
 623                 {
 624                         uri:      "download.example.com/users/active",
 625                         header:   authHeader,
 626                         redirect: "/users/active/",
 627                         expect:   []string{"foo_file_in_dir/"},
 628                         cutDirs:  2,
 629                 },
 630                 {
 631                         uri:     "download.example.com/users/active/",
 632                         header:  authHeader,
 633                         expect:  []string{"foo_file_in_dir/"},
 634                         cutDirs: 2,
 635                 },
 636                 {
 637                         uri:     "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
 638                         header:  nil,
 639                         expect:  []string{"dir1/foo", "dir1/bar"},
 640                         cutDirs: 4,
 641                 },
 642                 {
 643                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
 644                         header:  nil,
 645                         expect:  []string{"dir1/foo", "dir1/bar"},
 646                         cutDirs: 2,
 647                 },
 648                 {
 649                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken,
 650                         header:  nil,
 651                         expect:  []string{"dir1/foo", "dir1/bar"},
 652                         cutDirs: 2,
 653                 },
 654                 {
 655                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID,
 656                         header:  authHeader,
 657                         expect:  []string{"dir1/foo", "dir1/bar"},
 658                         cutDirs: 1,
 659                 },
 660                 {
 661                         uri:      "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
 662                         header:   authHeader,
 663                         redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
 664                         expect:   []string{"foo", "bar"},
 665                         cutDirs:  2,
 666                 },
 667                 {
 668                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
 669                         header:  authHeader,
 670                         expect:  []string{"foo", "bar"},
 671                         cutDirs: 3,
 672                 },
 673                 {
 674                         uri:      arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
 675                         header:   authHeader,
 676                         redirect: "/dir1/",
 677                         expect:   []string{"foo", "bar"},
 678                         cutDirs:  1,
 679                 },
 680                 {
 681                         uri:    "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
 682                         header: authHeader,
 683                         expect: nil,
 684                 },
 685                 {
 686                         uri:     "download.example.com/c=" + arvadostest.WazVersion1Collection,
 687                         header:  authHeader,
 688                         expect:  []string{"waz"},
 689                         cutDirs: 1,
 690                 },
 691                 {
 692                         uri:     "download.example.com/by_id/" + arvadostest.WazVersion1Collection,
 693                         header:  authHeader,
 694                         expect:  []string{"waz"},
 695                         cutDirs: 2,
 696                 },
 697         } {
 698                 c.Logf("HTML: %q => %q", trial.uri, trial.expect)
 699                 resp := httptest.NewRecorder()
 700                 u := mustParseURL("//" + trial.uri)
 701                 req := &http.Request{
 702                         Method:     "GET",
 703                         Host:       u.Host,
 704                         URL:        u,
 705                         RequestURI: u.RequestURI(),
 706                         Header:     copyHeader(trial.header),
 707                 }
 708                 s.testServer.Handler.ServeHTTP(resp, req)
 709                 var cookies []*http.Cookie
 710                 for resp.Code == http.StatusSeeOther {
 711                         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 712                         req = &http.Request{
 713                                 Method:     "GET",
 714                                 Host:       u.Host,
 715                                 URL:        u,
 716                                 RequestURI: u.RequestURI(),
 717                                 Header:     copyHeader(trial.header),
 718                         }
 719                         cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
 720                         for _, c := range cookies {
 721                                 req.AddCookie(c)
 722                         }
 723                         resp = httptest.NewRecorder()
 724                         s.testServer.Handler.ServeHTTP(resp, req)
 725                 }
 726                 if trial.redirect != "" {
 727                         c.Check(req.URL.Path, check.Equals, trial.redirect)
 728                 }
 729                 if trial.expect == nil {
 730                         c.Check(resp.Code, check.Equals, http.StatusNotFound)
 731                 } else {
 732                         c.Check(resp.Code, check.Equals, http.StatusOK)
 733                         for _, e := range trial.expect {
 734                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`)
 735                         }
 736                         c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`)
 737                 }
 738
 739                 c.Logf("WebDAV: %q => %q", trial.uri, trial.expect)
 740                 req = &http.Request{
 741                         Method:     "OPTIONS",
 742                         Host:       u.Host,
 743                         URL:        u,
 744                         RequestURI: u.RequestURI(),
 745                         Header:     copyHeader(trial.header),
 746                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
 747                 }
 748                 resp = httptest.NewRecorder()
 749                 s.testServer.Handler.ServeHTTP(resp, req)
 750                 if trial.expect == nil {
 751                         c.Check(resp.Code, check.Equals, http.StatusNotFound)
 752                 } else {
 753                         c.Check(resp.Code, check.Equals, http.StatusOK)
 754                 }
 755
 756                 req = &http.Request{
 757                         Method:     "PROPFIND",
 758                         Host:       u.Host,
 759                         URL:        u,
 760                         RequestURI: u.RequestURI(),
 761                         Header:     copyHeader(trial.header),
 762                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
 763                 }
 764                 resp = httptest.NewRecorder()
 765                 s.testServer.Handler.ServeHTTP(resp, req)
 766                 if trial.expect == nil {
 767                         c.Check(resp.Code, check.Equals, http.StatusNotFound)
 768                 } else {
 769                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus)
 770                         for _, e := range trial.expect {
 771                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+filepath.Join(u.Path, e)+`</D:href>.*`)
 772                         }
 773                 }
 774         }
 775 }
 776
 777 func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
 778         arv := arvados.NewClientFromEnv()
 779         var newCollection arvados.Collection
 780         err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 781                 "collection": map[string]string{
 782                         "owner_uuid":    arvadostest.ActiveUserUUID,
 783                         "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
 784                         "name":          "keep-web test collection",
 785                 },
 786                 "ensure_unique_name": true,
 787         })
 788         c.Assert(err, check.IsNil)
 789         defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
 790
 791         var updated arvados.Collection
 792         for _, fnm := range []string{"foo.txt", "bar.txt"} {
 793                 s.testServer.Config.AttachmentOnlyHost = "example.com"
 794                 u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
 795                 req := &http.Request{
 796                         Method:     "DELETE",
 797                         Host:       u.Host,
 798                         URL:        u,
 799                         RequestURI: u.RequestURI(),
 800                         Header: http.Header{
 801                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
 802                         },
 803                 }
 804                 resp := httptest.NewRecorder()
 805                 s.testServer.Handler.ServeHTTP(resp, req)
 806                 c.Check(resp.Code, check.Equals, http.StatusNoContent)
 807
 808                 updated = arvados.Collection{}
 809                 err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
 810                 c.Check(err, check.IsNil)
 811                 c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`)
 812                 c.Logf("updated manifest_text %q", updated.ManifestText)
 813         }
 814         c.Check(updated.ManifestText, check.Equals, "")
 815 }
 816
 817 func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) {
 818         s.testServer.Config.ManagementToken = arvadostest.ManagementToken
 819         authHeader := http.Header{
 820                 "Authorization": {"Bearer " + arvadostest.ManagementToken},
 821         }
 822
 823         resp := httptest.NewRecorder()
 824         u := mustParseURL("http://download.example.com/_health/ping")
 825         req := &http.Request{
 826                 Method:     "GET",
 827                 Host:       u.Host,
 828                 URL:        u,
 829                 RequestURI: u.RequestURI(),
 830                 Header:     authHeader,
 831         }
 832         s.testServer.Handler.ServeHTTP(resp, req)
 833
 834         c.Check(resp.Code, check.Equals, http.StatusOK)
 835         c.Check(resp.Body.String(), check.Matches, `{"health":"OK"}\n`)
 836 }
 837
 838 func copyHeader(h http.Header) http.Header {
 839         hc := http.Header{}
 840         for k, v := range h {
 841                 hc[k] = append([]string(nil), v...)
 842         }
 843         return hc
 844 }