services/keep-web/handler_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "fmt"
  10         "html"
  11         "io/ioutil"
  12         "net/http"
  13         "net/http/httptest"
  14         "net/url"
  15         "os"
  16         "path/filepath"
  17         "regexp"
  18         "strings"
  19
  20         "git.curoverse.com/arvados.git/sdk/go/arvados"
  21         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  22         "git.curoverse.com/arvados.git/sdk/go/auth"
  23         check "gopkg.in/check.v1"
  24 )
  25
  26 var _ = check.Suite(&UnitSuite{})
  27
  28 type UnitSuite struct{}
  29
  30 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
  31         h := handler{Config: DefaultConfig()}
  32         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
  33         req := &http.Request{
  34                 Method:     "OPTIONS",
  35                 Host:       u.Host,
  36                 URL:        u,
  37                 RequestURI: u.RequestURI(),
  38                 Header: http.Header{
  39                         "Origin":                        {"https://workbench.example"},
  40                         "Access-Control-Request-Method": {"POST"},
  41                 },
  42         }
  43
  44         // Check preflight for an allowed request
  45         resp := httptest.NewRecorder()
  46         h.ServeHTTP(resp, req)
  47         c.Check(resp.Code, check.Equals, http.StatusOK)
  48         c.Check(resp.Body.String(), check.Equals, "")
  49         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
  50         c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
  51         c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout")
  52
  53         // Check preflight for a disallowed request
  54         resp = httptest.NewRecorder()
  55         req.Header.Set("Access-Control-Request-Method", "MAKE-COFFEE")
  56         h.ServeHTTP(resp, req)
  57         c.Check(resp.Body.String(), check.Equals, "")
  58         c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
  59 }
  60
  61 func (s *UnitSuite) TestInvalidUUID(c *check.C) {
  62         bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-"
  63         token := arvadostest.ActiveToken
  64         for _, trial := range []string{
  65                 "http://keep-web/c=" + bogusID + "/foo",
  66                 "http://keep-web/c=" + bogusID + "/t=" + token + "/foo",
  67                 "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo",
  68                 "http://keep-web/collections/" + bogusID + "/foo",
  69                 "http://" + bogusID + ".keep-web/" + bogusID + "/foo",
  70                 "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
  71         } {
  72                 c.Log(trial)
  73                 u := mustParseURL(trial)
  74                 req := &http.Request{
  75                         Method:     "GET",
  76                         Host:       u.Host,
  77                         URL:        u,
  78                         RequestURI: u.RequestURI(),
  79                 }
  80                 resp := httptest.NewRecorder()
  81                 cfg := DefaultConfig()
  82                 cfg.AnonymousTokens = []string{arvadostest.AnonymousToken}
  83                 h := handler{Config: cfg}
  84                 h.ServeHTTP(resp, req)
  85                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
  86         }
  87 }
  88
  89 func mustParseURL(s string) *url.URL {
  90         r, err := url.Parse(s)
  91         if err != nil {
  92                 panic("parse URL: " + s)
  93         }
  94         return r
  95 }
  96
  97 func (s *IntegrationSuite) TestVhost404(c *check.C) {
  98         for _, testURL := range []string{
  99                 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
 100                 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
 101         } {
 102                 resp := httptest.NewRecorder()
 103                 u := mustParseURL(testURL)
 104                 req := &http.Request{
 105                         Method:     "GET",
 106                         URL:        u,
 107                         RequestURI: u.RequestURI(),
 108                 }
 109                 s.testServer.Handler.ServeHTTP(resp, req)
 110                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 111                 c.Check(resp.Body.String(), check.Equals, "")
 112         }
 113 }
 114
 115 // An authorizer modifies an HTTP request to make use of the given
 116 // token -- by adding it to a header, cookie, query param, or whatever
 117 // -- and returns the HTTP status code we should expect from keep-web if
 118 // the token is invalid.
 119 type authorizer func(*http.Request, string) int
 120
 121 func (s *IntegrationSuite) TestVhostViaAuthzHeader(c *check.C) {
 122         s.doVhostRequests(c, authzViaAuthzHeader)
 123 }
 124 func authzViaAuthzHeader(r *http.Request, tok string) int {
 125         r.Header.Add("Authorization", "OAuth2 "+tok)
 126         return http.StatusUnauthorized
 127 }
 128
 129 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
 130         s.doVhostRequests(c, authzViaCookieValue)
 131 }
 132 func authzViaCookieValue(r *http.Request, tok string) int {
 133         r.AddCookie(&http.Cookie{
 134                 Name:  "arvados_api_token",
 135                 Value: auth.EncodeTokenCookie([]byte(tok)),
 136         })
 137         return http.StatusUnauthorized
 138 }
 139
 140 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
 141         s.doVhostRequests(c, authzViaPath)
 142 }
 143 func authzViaPath(r *http.Request, tok string) int {
 144         r.URL.Path = "/t=" + tok + r.URL.Path
 145         return http.StatusNotFound
 146 }
 147
 148 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
 149         s.doVhostRequests(c, authzViaQueryString)
 150 }
 151 func authzViaQueryString(r *http.Request, tok string) int {
 152         r.URL.RawQuery = "api_token=" + tok
 153         return http.StatusUnauthorized
 154 }
 155
 156 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
 157         s.doVhostRequests(c, authzViaPOST)
 158 }
 159 func authzViaPOST(r *http.Request, tok string) int {
 160         r.Method = "POST"
 161         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 162         r.Body = ioutil.NopCloser(strings.NewReader(
 163                 url.Values{"api_token": {tok}}.Encode()))
 164         return http.StatusUnauthorized
 165 }
 166
 167 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
 168         s.doVhostRequests(c, authzViaPOST)
 169 }
 170 func authzViaXHRPOST(r *http.Request, tok string) int {
 171         r.Method = "POST"
 172         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 173         r.Header.Add("Origin", "https://origin.example")
 174         r.Body = ioutil.NopCloser(strings.NewReader(
 175                 url.Values{
 176                         "api_token":   {tok},
 177                         "disposition": {"attachment"},
 178                 }.Encode()))
 179         return http.StatusUnauthorized
 180 }
 181
 182 // Try some combinations of {url, token} using the given authorization
 183 // mechanism, and verify the result is correct.
 184 func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
 185         for _, hostPath := range []string{
 186                 arvadostest.FooCollection + ".example.com/foo",
 187                 arvadostest.FooCollection + "--collections.example.com/foo",
 188                 arvadostest.FooCollection + "--collections.example.com/_/foo",
 189                 arvadostest.FooCollectionPDH + ".example.com/foo",
 190                 strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + "--collections.example.com/foo",
 191                 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
 192         } {
 193                 c.Log("doRequests: ", hostPath)
 194                 s.doVhostRequestsWithHostPath(c, authz, hostPath)
 195         }
 196 }
 197
 198 func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
 199         for _, tok := range []string{
 200                 arvadostest.ActiveToken,
 201                 arvadostest.ActiveToken[:15],
 202                 arvadostest.SpectatorToken,
 203                 "bogus",
 204                 "",
 205         } {
 206                 u := mustParseURL("http://" + hostPath)
 207                 req := &http.Request{
 208                         Method:     "GET",
 209                         Host:       u.Host,
 210                         URL:        u,
 211                         RequestURI: u.RequestURI(),
 212                         Header:     http.Header{},
 213                 }
 214                 failCode := authz(req, tok)
 215                 req, resp := s.doReq(req)
 216                 code, body := resp.Code, resp.Body.String()
 217
 218                 // If the initial request had a (non-empty) token
 219                 // showing in the query string, we should have been
 220                 // redirected in order to hide it in a cookie.
 221                 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
 222
 223                 if tok == arvadostest.ActiveToken {
 224                         c.Check(code, check.Equals, http.StatusOK)
 225                         c.Check(body, check.Equals, "foo")
 226
 227                 } else {
 228                         c.Check(code >= 400, check.Equals, true)
 229                         c.Check(code < 500, check.Equals, true)
 230                         if tok == arvadostest.SpectatorToken {
 231                                 // Valid token never offers to retry
 232                                 // with different credentials.
 233                                 c.Check(code, check.Equals, http.StatusNotFound)
 234                         } else {
 235                                 // Invalid token can ask to retry
 236                                 // depending on the authz method.
 237                                 c.Check(code, check.Equals, failCode)
 238                         }
 239                         c.Check(body, check.Equals, "")
 240                 }
 241         }
 242 }
 243
 244 func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
 245         resp := httptest.NewRecorder()
 246         s.testServer.Handler.ServeHTTP(resp, req)
 247         if resp.Code != http.StatusSeeOther {
 248                 return req, resp
 249         }
 250         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 251         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 252         req = &http.Request{
 253                 Method:     "GET",
 254                 Host:       u.Host,
 255                 URL:        u,
 256                 RequestURI: u.RequestURI(),
 257                 Header:     http.Header{},
 258         }
 259         for _, c := range cookies {
 260                 req.AddCookie(c)
 261         }
 262         return s.doReq(req)
 263 }
 264
 265 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
 266         s.testVhostRedirectTokenToCookie(c, "GET",
 267                 arvadostest.FooCollection+".example.com/foo",
 268                 "?api_token="+arvadostest.ActiveToken,
 269                 "",
 270                 "",
 271                 http.StatusOK,
 272                 "foo",
 273         )
 274 }
 275
 276 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
 277         s.testVhostRedirectTokenToCookie(c, "GET",
 278                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
 279                 "",
 280                 "",
 281                 "",
 282                 http.StatusOK,
 283                 "foo",
 284         )
 285 }
 286
 287 // Bad token in URL is 404 Not Found because it doesn't make sense to
 288 // retry the same URL with different authorization.
 289 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
 290         s.testVhostRedirectTokenToCookie(c, "GET",
 291                 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
 292                 "",
 293                 "",
 294                 "",
 295                 http.StatusNotFound,
 296                 "",
 297         )
 298 }
 299
 300 // Bad token in a cookie (even if it got there via our own
 301 // query-string-to-cookie redirect) is, in principle, retryable at the
 302 // same URL so it's 401 Unauthorized.
 303 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
 304         s.testVhostRedirectTokenToCookie(c, "GET",
 305                 arvadostest.FooCollection+".example.com/foo",
 306                 "?api_token=thisisabogustoken",
 307                 "",
 308                 "",
 309                 http.StatusUnauthorized,
 310                 "",
 311         )
 312 }
 313
 314 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
 315         s.testVhostRedirectTokenToCookie(c, "GET",
 316                 "example.com/c="+arvadostest.FooCollection+"/foo",
 317                 "?api_token="+arvadostest.ActiveToken,
 318                 "",
 319                 "",
 320                 http.StatusBadRequest,
 321                 "",
 322         )
 323 }
 324
 325 // If client requests an attachment by putting ?disposition=attachment
 326 // in the query string, and gets redirected, the redirect target
 327 // should respond with an attachment.
 328 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
 329         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 330                 arvadostest.FooCollection+".example.com/foo",
 331                 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
 332                 "",
 333                 "",
 334                 http.StatusOK,
 335                 "foo",
 336         )
 337         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 338 }
 339
 340 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
 341         s.testServer.Config.AttachmentOnlyHost = "download.example.com"
 342         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 343                 "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
 344                 "?api_token="+arvadostest.ActiveToken,
 345                 "",
 346                 "",
 347                 http.StatusOK,
 348                 "foo",
 349         )
 350         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 351 }
 352
 353 func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
 354         s.testServer.Config.AttachmentOnlyHost = "download.example.com"
 355         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 356                 "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
 357                 "?api_token="+arvadostest.ActiveToken,
 358                 "",
 359                 "",
 360                 http.StatusOK,
 361                 "waz",
 362         )
 363         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 364         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 365                 "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz",
 366                 "?api_token="+arvadostest.ActiveToken,
 367                 "",
 368                 "",
 369                 http.StatusOK,
 370                 "waz",
 371         )
 372         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 373 }
 374
 375 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
 376         s.testServer.Config.TrustAllContent = true
 377         s.testVhostRedirectTokenToCookie(c, "GET",
 378                 "example.com/c="+arvadostest.FooCollection+"/foo",
 379                 "?api_token="+arvadostest.ActiveToken,
 380                 "",
 381                 "",
 382                 http.StatusOK,
 383                 "foo",
 384         )
 385 }
 386
 387 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
 388         s.testServer.Config.AttachmentOnlyHost = "example.com:1234"
 389
 390         s.testVhostRedirectTokenToCookie(c, "GET",
 391                 "example.com/c="+arvadostest.FooCollection+"/foo",
 392                 "?api_token="+arvadostest.ActiveToken,
 393                 "",
 394                 "",
 395                 http.StatusBadRequest,
 396                 "",
 397         )
 398
 399         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 400                 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
 401                 "?api_token="+arvadostest.ActiveToken,
 402                 "",
 403                 "",
 404                 http.StatusOK,
 405                 "foo",
 406         )
 407         c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
 408 }
 409
 410 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
 411         s.testVhostRedirectTokenToCookie(c, "POST",
 412                 arvadostest.FooCollection+".example.com/foo",
 413                 "",
 414                 "application/x-www-form-urlencoded",
 415                 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
 416                 http.StatusOK,
 417                 "foo",
 418         )
 419 }
 420
 421 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
 422         s.testVhostRedirectTokenToCookie(c, "POST",
 423                 arvadostest.FooCollection+".example.com/foo",
 424                 "",
 425                 "application/x-www-form-urlencoded",
 426                 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
 427                 http.StatusNotFound,
 428                 "",
 429         )
 430 }
 431
 432 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
 433         s.testServer.Config.AnonymousTokens = []string{arvadostest.AnonymousToken}
 434         s.testVhostRedirectTokenToCookie(c, "GET",
 435                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 436                 "",
 437                 "",
 438                 "",
 439                 http.StatusOK,
 440                 "Hello world\n",
 441         )
 442 }
 443
 444 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
 445         s.testServer.Config.AnonymousTokens = []string{"anonymousTokenConfiguredButInvalid"}
 446         s.testVhostRedirectTokenToCookie(c, "GET",
 447                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 448                 "",
 449                 "",
 450                 "",
 451                 http.StatusNotFound,
 452                 "",
 453         )
 454 }
 455
 456 func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
 457         s.testServer.Config.AttachmentOnlyHost = "download.example.com"
 458
 459         client := s.testServer.Config.Client
 460         client.AuthToken = arvadostest.ActiveToken
 461         fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
 462         c.Assert(err, check.IsNil)
 463         f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777)
 464         c.Assert(err, check.IsNil)
 465         f.Close()
 466         mtxt, err := fs.MarshalManifest(".")
 467         c.Assert(err, check.IsNil)
 468         var coll arvados.Collection
 469         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 470                 "collection": map[string]string{
 471                         "manifest_text": mtxt,
 472                 },
 473         })
 474         c.Assert(err, check.IsNil)
 475
 476         u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
 477         req := &http.Request{
 478                 Method:     "GET",
 479                 Host:       u.Host,
 480                 URL:        u,
 481                 RequestURI: u.RequestURI(),
 482                 Header: http.Header{
 483                         "Authorization": {"Bearer " + client.AuthToken},
 484                 },
 485         }
 486         resp := httptest.NewRecorder()
 487         s.testServer.Handler.ServeHTTP(resp, req)
 488         c.Check(resp.Code, check.Equals, http.StatusOK)
 489         c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\&#34;odd&#39; path chars.*`)
 490 }
 491
 492 // XHRs can't follow redirect-with-cookie so they rely on method=POST
 493 // and disposition=attachment (telling us it's acceptable to respond
 494 // with content instead of a redirect) and an Origin header that gets
 495 // added automatically by the browser (telling us it's desirable to do
 496 // so).
 497 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
 498         u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
 499         req := &http.Request{
 500                 Method:     "POST",
 501                 Host:       u.Host,
 502                 URL:        u,
 503                 RequestURI: u.RequestURI(),
 504                 Header: http.Header{
 505                         "Origin":       {"https://origin.example"},
 506                         "Content-Type": {"application/x-www-form-urlencoded"},
 507                 },
 508                 Body: ioutil.NopCloser(strings.NewReader(url.Values{
 509                         "api_token":   {arvadostest.ActiveToken},
 510                         "disposition": {"attachment"},
 511                 }.Encode())),
 512         }
 513         resp := httptest.NewRecorder()
 514         s.testServer.Handler.ServeHTTP(resp, req)
 515         c.Check(resp.Code, check.Equals, http.StatusOK)
 516         c.Check(resp.Body.String(), check.Equals, "foo")
 517         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
 518 }
 519
 520 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder {
 521         u, _ := url.Parse(`http://` + hostPath + queryString)
 522         req := &http.Request{
 523                 Method:     method,
 524                 Host:       u.Host,
 525                 URL:        u,
 526                 RequestURI: u.RequestURI(),
 527                 Header:     http.Header{"Content-Type": {contentType}},
 528                 Body:       ioutil.NopCloser(strings.NewReader(reqBody)),
 529         }
 530
 531         resp := httptest.NewRecorder()
 532         defer func() {
 533                 c.Check(resp.Code, check.Equals, expectStatus)
 534                 c.Check(resp.Body.String(), check.Equals, expectRespBody)
 535         }()
 536
 537         s.testServer.Handler.ServeHTTP(resp, req)
 538         if resp.Code != http.StatusSeeOther {
 539                 return resp
 540         }
 541         c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
 542         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 543
 544         u, _ = u.Parse(resp.Header().Get("Location"))
 545         req = &http.Request{
 546                 Method:     "GET",
 547                 Host:       u.Host,
 548                 URL:        u,
 549                 RequestURI: u.RequestURI(),
 550                 Header:     http.Header{},
 551         }
 552         for _, c := range cookies {
 553                 req.AddCookie(c)
 554         }
 555
 556         resp = httptest.NewRecorder()
 557         s.testServer.Handler.ServeHTTP(resp, req)
 558         c.Check(resp.Header().Get("Location"), check.Equals, "")
 559         return resp
 560 }
 561
 562 func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) {
 563         s.testServer.Config.AnonymousTokens = []string{arvadostest.AnonymousToken}
 564         s.testDirectoryListing(c)
 565 }
 566
 567 func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) {
 568         s.testServer.Config.AnonymousTokens = nil
 569         s.testDirectoryListing(c)
 570 }
 571
 572 func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
 573         s.testServer.Config.AttachmentOnlyHost = "download.example.com"
 574         authHeader := http.Header{
 575                 "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
 576         }
 577         for _, trial := range []struct {
 578                 uri      string
 579                 header   http.Header
 580                 expect   []string
 581                 redirect string
 582                 cutDirs  int
 583         }{
 584                 {
 585                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
 586                         header:  authHeader,
 587                         expect:  []string{"dir1/foo", "dir1/bar"},
 588                         cutDirs: 0,
 589                 },
 590                 {
 591                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
 592                         header:  authHeader,
 593                         expect:  []string{"foo", "bar"},
 594                         cutDirs: 1,
 595                 },
 596                 {
 597                         // URLs of this form ignore authHeader, and
 598                         // FooAndBarFilesInDirUUID isn't public, so
 599                         // this returns 404.
 600                         uri:    "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
 601                         header: authHeader,
 602                         expect: nil,
 603                 },
 604                 {
 605                         uri:     "download.example.com/users/active/foo_file_in_dir/",
 606                         header:  authHeader,
 607                         expect:  []string{"dir1/"},
 608                         cutDirs: 3,
 609                 },
 610                 {
 611                         uri:     "download.example.com/users/active/foo_file_in_dir/dir1/",
 612                         header:  authHeader,
 613                         expect:  []string{"bar"},
 614                         cutDirs: 4,
 615                 },
 616                 {
 617                         uri:     "download.example.com/",
 618                         header:  authHeader,
 619                         expect:  []string{"users/"},
 620                         cutDirs: 0,
 621                 },
 622                 {
 623                         uri:      "download.example.com/users",
 624                         header:   authHeader,
 625                         redirect: "/users/",
 626                         expect:   []string{"active/"},
 627                         cutDirs:  1,
 628                 },
 629                 {
 630                         uri:     "download.example.com/users/",
 631                         header:  authHeader,
 632                         expect:  []string{"active/"},
 633                         cutDirs: 1,
 634                 },
 635                 {
 636                         uri:      "download.example.com/users/active",
 637                         header:   authHeader,
 638                         redirect: "/users/active/",
 639                         expect:   []string{"foo_file_in_dir/"},
 640                         cutDirs:  2,
 641                 },
 642                 {
 643                         uri:     "download.example.com/users/active/",
 644                         header:  authHeader,
 645                         expect:  []string{"foo_file_in_dir/"},
 646                         cutDirs: 2,
 647                 },
 648                 {
 649                         uri:     "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
 650                         header:  nil,
 651                         expect:  []string{"dir1/foo", "dir1/bar"},
 652                         cutDirs: 4,
 653                 },
 654                 {
 655                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
 656                         header:  nil,
 657                         expect:  []string{"dir1/foo", "dir1/bar"},
 658                         cutDirs: 2,
 659                 },
 660                 {
 661                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken,
 662                         header:  nil,
 663                         expect:  []string{"dir1/foo", "dir1/bar"},
 664                         cutDirs: 2,
 665                 },
 666                 {
 667                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID,
 668                         header:  authHeader,
 669                         expect:  []string{"dir1/foo", "dir1/bar"},
 670                         cutDirs: 1,
 671                 },
 672                 {
 673                         uri:      "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
 674                         header:   authHeader,
 675                         redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
 676                         expect:   []string{"foo", "bar"},
 677                         cutDirs:  2,
 678                 },
 679                 {
 680                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
 681                         header:  authHeader,
 682                         expect:  []string{"foo", "bar"},
 683                         cutDirs: 3,
 684                 },
 685                 {
 686                         uri:      arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
 687                         header:   authHeader,
 688                         redirect: "/dir1/",
 689                         expect:   []string{"foo", "bar"},
 690                         cutDirs:  1,
 691                 },
 692                 {
 693                         uri:    "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
 694                         header: authHeader,
 695                         expect: nil,
 696                 },
 697                 {
 698                         uri:     "download.example.com/c=" + arvadostest.WazVersion1Collection,
 699                         header:  authHeader,
 700                         expect:  []string{"waz"},
 701                         cutDirs: 1,
 702                 },
 703                 {
 704                         uri:     "download.example.com/by_id/" + arvadostest.WazVersion1Collection,
 705                         header:  authHeader,
 706                         expect:  []string{"waz"},
 707                         cutDirs: 2,
 708                 },
 709         } {
 710                 c.Logf("HTML: %q => %q", trial.uri, trial.expect)
 711                 resp := httptest.NewRecorder()
 712                 u := mustParseURL("//" + trial.uri)
 713                 req := &http.Request{
 714                         Method:     "GET",
 715                         Host:       u.Host,
 716                         URL:        u,
 717                         RequestURI: u.RequestURI(),
 718                         Header:     copyHeader(trial.header),
 719                 }
 720                 s.testServer.Handler.ServeHTTP(resp, req)
 721                 var cookies []*http.Cookie
 722                 for resp.Code == http.StatusSeeOther {
 723                         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 724                         req = &http.Request{
 725                                 Method:     "GET",
 726                                 Host:       u.Host,
 727                                 URL:        u,
 728                                 RequestURI: u.RequestURI(),
 729                                 Header:     copyHeader(trial.header),
 730                         }
 731                         cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
 732                         for _, c := range cookies {
 733                                 req.AddCookie(c)
 734                         }
 735                         resp = httptest.NewRecorder()
 736                         s.testServer.Handler.ServeHTTP(resp, req)
 737                 }
 738                 if trial.redirect != "" {
 739                         c.Check(req.URL.Path, check.Equals, trial.redirect)
 740                 }
 741                 if trial.expect == nil {
 742                         c.Check(resp.Code, check.Equals, http.StatusNotFound)
 743                 } else {
 744                         c.Check(resp.Code, check.Equals, http.StatusOK)
 745                         for _, e := range trial.expect {
 746                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`)
 747                         }
 748                         c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`)
 749                 }
 750
 751                 c.Logf("WebDAV: %q => %q", trial.uri, trial.expect)
 752                 req = &http.Request{
 753                         Method:     "OPTIONS",
 754                         Host:       u.Host,
 755                         URL:        u,
 756                         RequestURI: u.RequestURI(),
 757                         Header:     copyHeader(trial.header),
 758                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
 759                 }
 760                 resp = httptest.NewRecorder()
 761                 s.testServer.Handler.ServeHTTP(resp, req)
 762                 if trial.expect == nil {
 763                         c.Check(resp.Code, check.Equals, http.StatusNotFound)
 764                 } else {
 765                         c.Check(resp.Code, check.Equals, http.StatusOK)
 766                 }
 767
 768                 req = &http.Request{
 769                         Method:     "PROPFIND",
 770                         Host:       u.Host,
 771                         URL:        u,
 772                         RequestURI: u.RequestURI(),
 773                         Header:     copyHeader(trial.header),
 774                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
 775                 }
 776                 resp = httptest.NewRecorder()
 777                 s.testServer.Handler.ServeHTTP(resp, req)
 778                 if trial.expect == nil {
 779                         c.Check(resp.Code, check.Equals, http.StatusNotFound)
 780                 } else {
 781                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus)
 782                         for _, e := range trial.expect {
 783                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+filepath.Join(u.Path, e)+`</D:href>.*`)
 784                         }
 785                 }
 786         }
 787 }
 788
 789 func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
 790         arv := arvados.NewClientFromEnv()
 791         var newCollection arvados.Collection
 792         err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 793                 "collection": map[string]string{
 794                         "owner_uuid":    arvadostest.ActiveUserUUID,
 795                         "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
 796                         "name":          "keep-web test collection",
 797                 },
 798                 "ensure_unique_name": true,
 799         })
 800         c.Assert(err, check.IsNil)
 801         defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
 802
 803         var updated arvados.Collection
 804         for _, fnm := range []string{"foo.txt", "bar.txt"} {
 805                 s.testServer.Config.AttachmentOnlyHost = "example.com"
 806                 u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
 807                 req := &http.Request{
 808                         Method:     "DELETE",
 809                         Host:       u.Host,
 810                         URL:        u,
 811                         RequestURI: u.RequestURI(),
 812                         Header: http.Header{
 813                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
 814                         },
 815                 }
 816                 resp := httptest.NewRecorder()
 817                 s.testServer.Handler.ServeHTTP(resp, req)
 818                 c.Check(resp.Code, check.Equals, http.StatusNoContent)
 819
 820                 updated = arvados.Collection{}
 821                 err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
 822                 c.Check(err, check.IsNil)
 823                 c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`)
 824                 c.Logf("updated manifest_text %q", updated.ManifestText)
 825         }
 826         c.Check(updated.ManifestText, check.Equals, "")
 827 }
 828
 829 func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) {
 830         s.testServer.Config.ManagementToken = arvadostest.ManagementToken
 831         authHeader := http.Header{
 832                 "Authorization": {"Bearer " + arvadostest.ManagementToken},
 833         }
 834
 835         resp := httptest.NewRecorder()
 836         u := mustParseURL("http://download.example.com/_health/ping")
 837         req := &http.Request{
 838                 Method:     "GET",
 839                 Host:       u.Host,
 840                 URL:        u,
 841                 RequestURI: u.RequestURI(),
 842                 Header:     authHeader,
 843         }
 844         s.testServer.Handler.ServeHTTP(resp, req)
 845
 846         c.Check(resp.Code, check.Equals, http.StatusOK)
 847         c.Check(resp.Body.String(), check.Matches, `{"health":"OK"}\n`)
 848 }
 849
 850 func copyHeader(h http.Header) http.Header {
 851         hc := http.Header{}
 852         for k, v := range h {
 853                 hc[k] = append([]string(nil), v...)
 854         }
 855         return hc
 856 }