services/keep-web/handler_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "fmt"
  10         "html"
  11         "io/ioutil"
  12         "net/http"
  13         "net/http/httptest"
  14         "net/url"
  15         "os"
  16         "path/filepath"
  17         "regexp"
  18         "strings"
  19
  20         "git.arvados.org/arvados.git/lib/config"
  21         "git.arvados.org/arvados.git/sdk/go/arvados"
  22         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  23         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  24         "git.arvados.org/arvados.git/sdk/go/auth"
  25         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  26         "git.arvados.org/arvados.git/sdk/go/keepclient"
  27         check "gopkg.in/check.v1"
  28 )
  29
  30 var _ = check.Suite(&UnitSuite{})
  31
  32 type UnitSuite struct {
  33         Config *arvados.Config
  34 }
  35
  36 func (s *UnitSuite) SetUpTest(c *check.C) {
  37         ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), ctxlog.TestLogger(c))
  38         ldr.Path = "-"
  39         cfg, err := ldr.Load()
  40         c.Assert(err, check.IsNil)
  41         s.Config = cfg
  42 }
  43
  44 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
  45         h := handler{Config: newConfig(s.Config)}
  46         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
  47         req := &http.Request{
  48                 Method:     "OPTIONS",
  49                 Host:       u.Host,
  50                 URL:        u,
  51                 RequestURI: u.RequestURI(),
  52                 Header: http.Header{
  53                         "Origin":                        {"https://workbench.example"},
  54                         "Access-Control-Request-Method": {"POST"},
  55                 },
  56         }
  57
  58         // Check preflight for an allowed request
  59         resp := httptest.NewRecorder()
  60         h.ServeHTTP(resp, req)
  61         c.Check(resp.Code, check.Equals, http.StatusOK)
  62         c.Check(resp.Body.String(), check.Equals, "")
  63         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
  64         c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
  65         c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout")
  66
  67         // Check preflight for a disallowed request
  68         resp = httptest.NewRecorder()
  69         req.Header.Set("Access-Control-Request-Method", "MAKE-COFFEE")
  70         h.ServeHTTP(resp, req)
  71         c.Check(resp.Body.String(), check.Equals, "")
  72         c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
  73 }
  74
  75 func (s *UnitSuite) TestInvalidUUID(c *check.C) {
  76         bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-"
  77         token := arvadostest.ActiveToken
  78         for _, trial := range []string{
  79                 "http://keep-web/c=" + bogusID + "/foo",
  80                 "http://keep-web/c=" + bogusID + "/t=" + token + "/foo",
  81                 "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo",
  82                 "http://keep-web/collections/" + bogusID + "/foo",
  83                 "http://" + bogusID + ".keep-web/" + bogusID + "/foo",
  84                 "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
  85         } {
  86                 c.Log(trial)
  87                 u := mustParseURL(trial)
  88                 req := &http.Request{
  89                         Method:     "GET",
  90                         Host:       u.Host,
  91                         URL:        u,
  92                         RequestURI: u.RequestURI(),
  93                 }
  94                 resp := httptest.NewRecorder()
  95                 cfg := newConfig(s.Config)
  96                 cfg.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
  97                 h := handler{Config: cfg}
  98                 h.ServeHTTP(resp, req)
  99                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 100         }
 101 }
 102
 103 func mustParseURL(s string) *url.URL {
 104         r, err := url.Parse(s)
 105         if err != nil {
 106                 panic("parse URL: " + s)
 107         }
 108         return r
 109 }
 110
 111 func (s *IntegrationSuite) TestVhost404(c *check.C) {
 112         for _, testURL := range []string{
 113                 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
 114                 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
 115         } {
 116                 resp := httptest.NewRecorder()
 117                 u := mustParseURL(testURL)
 118                 req := &http.Request{
 119                         Method:     "GET",
 120                         URL:        u,
 121                         RequestURI: u.RequestURI(),
 122                 }
 123                 s.testServer.Handler.ServeHTTP(resp, req)
 124                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 125                 c.Check(resp.Body.String(), check.Equals, "")
 126         }
 127 }
 128
 129 // An authorizer modifies an HTTP request to make use of the given
 130 // token -- by adding it to a header, cookie, query param, or whatever
 131 // -- and returns the HTTP status code we should expect from keep-web if
 132 // the token is invalid.
 133 type authorizer func(*http.Request, string) int
 134
 135 func (s *IntegrationSuite) TestVhostViaAuthzHeader(c *check.C) {
 136         s.doVhostRequests(c, authzViaAuthzHeader)
 137 }
 138 func authzViaAuthzHeader(r *http.Request, tok string) int {
 139         r.Header.Add("Authorization", "OAuth2 "+tok)
 140         return http.StatusUnauthorized
 141 }
 142
 143 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
 144         s.doVhostRequests(c, authzViaCookieValue)
 145 }
 146 func authzViaCookieValue(r *http.Request, tok string) int {
 147         r.AddCookie(&http.Cookie{
 148                 Name:  "arvados_api_token",
 149                 Value: auth.EncodeTokenCookie([]byte(tok)),
 150         })
 151         return http.StatusUnauthorized
 152 }
 153
 154 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
 155         s.doVhostRequests(c, authzViaPath)
 156 }
 157 func authzViaPath(r *http.Request, tok string) int {
 158         r.URL.Path = "/t=" + tok + r.URL.Path
 159         return http.StatusNotFound
 160 }
 161
 162 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
 163         s.doVhostRequests(c, authzViaQueryString)
 164 }
 165 func authzViaQueryString(r *http.Request, tok string) int {
 166         r.URL.RawQuery = "api_token=" + tok
 167         return http.StatusUnauthorized
 168 }
 169
 170 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
 171         s.doVhostRequests(c, authzViaPOST)
 172 }
 173 func authzViaPOST(r *http.Request, tok string) int {
 174         r.Method = "POST"
 175         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 176         r.Body = ioutil.NopCloser(strings.NewReader(
 177                 url.Values{"api_token": {tok}}.Encode()))
 178         return http.StatusUnauthorized
 179 }
 180
 181 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
 182         s.doVhostRequests(c, authzViaPOST)
 183 }
 184 func authzViaXHRPOST(r *http.Request, tok string) int {
 185         r.Method = "POST"
 186         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 187         r.Header.Add("Origin", "https://origin.example")
 188         r.Body = ioutil.NopCloser(strings.NewReader(
 189                 url.Values{
 190                         "api_token":   {tok},
 191                         "disposition": {"attachment"},
 192                 }.Encode()))
 193         return http.StatusUnauthorized
 194 }
 195
 196 // Try some combinations of {url, token} using the given authorization
 197 // mechanism, and verify the result is correct.
 198 func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
 199         for _, hostPath := range []string{
 200                 arvadostest.FooCollection + ".example.com/foo",
 201                 arvadostest.FooCollection + "--collections.example.com/foo",
 202                 arvadostest.FooCollection + "--collections.example.com/_/foo",
 203                 arvadostest.FooCollectionPDH + ".example.com/foo",
 204                 strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + "--collections.example.com/foo",
 205                 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
 206         } {
 207                 c.Log("doRequests: ", hostPath)
 208                 s.doVhostRequestsWithHostPath(c, authz, hostPath)
 209         }
 210 }
 211
 212 func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
 213         for _, tok := range []string{
 214                 arvadostest.ActiveToken,
 215                 arvadostest.ActiveToken[:15],
 216                 arvadostest.SpectatorToken,
 217                 "bogus",
 218                 "",
 219         } {
 220                 u := mustParseURL("http://" + hostPath)
 221                 req := &http.Request{
 222                         Method:     "GET",
 223                         Host:       u.Host,
 224                         URL:        u,
 225                         RequestURI: u.RequestURI(),
 226                         Header:     http.Header{},
 227                 }
 228                 failCode := authz(req, tok)
 229                 req, resp := s.doReq(req)
 230                 code, body := resp.Code, resp.Body.String()
 231
 232                 // If the initial request had a (non-empty) token
 233                 // showing in the query string, we should have been
 234                 // redirected in order to hide it in a cookie.
 235                 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
 236
 237                 if tok == arvadostest.ActiveToken {
 238                         c.Check(code, check.Equals, http.StatusOK)
 239                         c.Check(body, check.Equals, "foo")
 240
 241                 } else {
 242                         c.Check(code >= 400, check.Equals, true)
 243                         c.Check(code < 500, check.Equals, true)
 244                         if tok == arvadostest.SpectatorToken {
 245                                 // Valid token never offers to retry
 246                                 // with different credentials.
 247                                 c.Check(code, check.Equals, http.StatusNotFound)
 248                         } else {
 249                                 // Invalid token can ask to retry
 250                                 // depending on the authz method.
 251                                 c.Check(code, check.Equals, failCode)
 252                         }
 253                         c.Check(body, check.Equals, "")
 254                 }
 255         }
 256 }
 257
 258 func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
 259         resp := httptest.NewRecorder()
 260         s.testServer.Handler.ServeHTTP(resp, req)
 261         if resp.Code != http.StatusSeeOther {
 262                 return req, resp
 263         }
 264         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 265         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 266         req = &http.Request{
 267                 Method:     "GET",
 268                 Host:       u.Host,
 269                 URL:        u,
 270                 RequestURI: u.RequestURI(),
 271                 Header:     http.Header{},
 272         }
 273         for _, c := range cookies {
 274                 req.AddCookie(c)
 275         }
 276         return s.doReq(req)
 277 }
 278
 279 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
 280         s.testVhostRedirectTokenToCookie(c, "GET",
 281                 arvadostest.FooCollection+".example.com/foo",
 282                 "?api_token="+arvadostest.ActiveToken,
 283                 "",
 284                 "",
 285                 http.StatusOK,
 286                 "foo",
 287         )
 288 }
 289
 290 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
 291         s.testVhostRedirectTokenToCookie(c, "GET",
 292                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
 293                 "",
 294                 "",
 295                 "",
 296                 http.StatusOK,
 297                 "foo",
 298         )
 299 }
 300
 301 // Bad token in URL is 404 Not Found because it doesn't make sense to
 302 // retry the same URL with different authorization.
 303 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
 304         s.testVhostRedirectTokenToCookie(c, "GET",
 305                 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
 306                 "",
 307                 "",
 308                 "",
 309                 http.StatusNotFound,
 310                 "",
 311         )
 312 }
 313
 314 // Bad token in a cookie (even if it got there via our own
 315 // query-string-to-cookie redirect) is, in principle, retryable at the
 316 // same URL so it's 401 Unauthorized.
 317 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
 318         s.testVhostRedirectTokenToCookie(c, "GET",
 319                 arvadostest.FooCollection+".example.com/foo",
 320                 "?api_token=thisisabogustoken",
 321                 "",
 322                 "",
 323                 http.StatusUnauthorized,
 324                 "",
 325         )
 326 }
 327
 328 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
 329         s.testVhostRedirectTokenToCookie(c, "GET",
 330                 "example.com/c="+arvadostest.FooCollection+"/foo",
 331                 "?api_token="+arvadostest.ActiveToken,
 332                 "",
 333                 "",
 334                 http.StatusBadRequest,
 335                 "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n",
 336         )
 337 }
 338
 339 // If client requests an attachment by putting ?disposition=attachment
 340 // in the query string, and gets redirected, the redirect target
 341 // should respond with an attachment.
 342 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
 343         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 344                 arvadostest.FooCollection+".example.com/foo",
 345                 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
 346                 "",
 347                 "",
 348                 http.StatusOK,
 349                 "foo",
 350         )
 351         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 352 }
 353
 354 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
 355         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 356         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 357                 "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
 358                 "?api_token="+arvadostest.ActiveToken,
 359                 "",
 360                 "",
 361                 http.StatusOK,
 362                 "foo",
 363         )
 364         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 365 }
 366
 367 func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
 368         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 369         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 370                 "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
 371                 "?api_token="+arvadostest.ActiveToken,
 372                 "",
 373                 "",
 374                 http.StatusOK,
 375                 "waz",
 376         )
 377         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 378         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 379                 "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz",
 380                 "?api_token="+arvadostest.ActiveToken,
 381                 "",
 382                 "",
 383                 http.StatusOK,
 384                 "waz",
 385         )
 386         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 387 }
 388
 389 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
 390         s.testServer.Config.cluster.Collections.TrustAllContent = true
 391         s.testVhostRedirectTokenToCookie(c, "GET",
 392                 "example.com/c="+arvadostest.FooCollection+"/foo",
 393                 "?api_token="+arvadostest.ActiveToken,
 394                 "",
 395                 "",
 396                 http.StatusOK,
 397                 "foo",
 398         )
 399 }
 400
 401 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
 402         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
 403
 404         s.testVhostRedirectTokenToCookie(c, "GET",
 405                 "example.com/c="+arvadostest.FooCollection+"/foo",
 406                 "?api_token="+arvadostest.ActiveToken,
 407                 "",
 408                 "",
 409                 http.StatusBadRequest,
 410                 "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n",
 411         )
 412
 413         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 414                 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
 415                 "?api_token="+arvadostest.ActiveToken,
 416                 "",
 417                 "",
 418                 http.StatusOK,
 419                 "foo",
 420         )
 421         c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
 422 }
 423
 424 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
 425         s.testVhostRedirectTokenToCookie(c, "POST",
 426                 arvadostest.FooCollection+".example.com/foo",
 427                 "",
 428                 "application/x-www-form-urlencoded",
 429                 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
 430                 http.StatusOK,
 431                 "foo",
 432         )
 433 }
 434
 435 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
 436         s.testVhostRedirectTokenToCookie(c, "POST",
 437                 arvadostest.FooCollection+".example.com/foo",
 438                 "",
 439                 "application/x-www-form-urlencoded",
 440                 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
 441                 http.StatusNotFound,
 442                 "",
 443         )
 444 }
 445
 446 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
 447         s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 448         s.testVhostRedirectTokenToCookie(c, "GET",
 449                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 450                 "",
 451                 "",
 452                 "",
 453                 http.StatusOK,
 454                 "Hello world\n",
 455         )
 456 }
 457
 458 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
 459         s.testServer.Config.cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
 460         s.testVhostRedirectTokenToCookie(c, "GET",
 461                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 462                 "",
 463                 "",
 464                 "",
 465                 http.StatusNotFound,
 466                 "",
 467         )
 468 }
 469
 470 func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
 471         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 472
 473         client := s.testServer.Config.Client
 474         client.AuthToken = arvadostest.ActiveToken
 475         fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
 476         c.Assert(err, check.IsNil)
 477         f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777)
 478         c.Assert(err, check.IsNil)
 479         f.Close()
 480         mtxt, err := fs.MarshalManifest(".")
 481         c.Assert(err, check.IsNil)
 482         var coll arvados.Collection
 483         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 484                 "collection": map[string]string{
 485                         "manifest_text": mtxt,
 486                 },
 487         })
 488         c.Assert(err, check.IsNil)
 489
 490         u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
 491         req := &http.Request{
 492                 Method:     "GET",
 493                 Host:       u.Host,
 494                 URL:        u,
 495                 RequestURI: u.RequestURI(),
 496                 Header: http.Header{
 497                         "Authorization": {"Bearer " + client.AuthToken},
 498                 },
 499         }
 500         resp := httptest.NewRecorder()
 501         s.testServer.Handler.ServeHTTP(resp, req)
 502         c.Check(resp.Code, check.Equals, http.StatusOK)
 503         c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\&#34;odd&#39; path chars.*`)
 504 }
 505
 506 func (s *IntegrationSuite) TestForwardSlashSubstitution(c *check.C) {
 507         arv := arvados.NewClientFromEnv()
 508         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 509         s.testServer.Config.cluster.Collections.ForwardSlashNameSubstitution = "{SOLIDUS}"
 510         name := "foo/bar/baz"
 511         nameShown := strings.Replace(name, "/", "{SOLIDUS}", -1)
 512         nameShownEscaped := strings.Replace(name, "/", "%7bSOLIDUS%7d", -1)
 513
 514         client := s.testServer.Config.Client
 515         client.AuthToken = arvadostest.ActiveToken
 516         fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
 517         c.Assert(err, check.IsNil)
 518         f, err := fs.OpenFile("filename", os.O_CREATE, 0777)
 519         c.Assert(err, check.IsNil)
 520         f.Close()
 521         mtxt, err := fs.MarshalManifest(".")
 522         c.Assert(err, check.IsNil)
 523         var coll arvados.Collection
 524         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 525                 "collection": map[string]string{
 526                         "manifest_text": mtxt,
 527                         "name":          name,
 528                         "owner_uuid":    arvadostest.AProjectUUID,
 529                 },
 530         })
 531         c.Assert(err, check.IsNil)
 532         defer arv.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
 533
 534         base := "http://download.example.com/by_id/" + coll.OwnerUUID + "/"
 535         for tryURL, expectRegexp := range map[string]string{
 536                 base:                          `(?ms).*href="./` + nameShownEscaped + `/"\S+` + nameShown + `.*`,
 537                 base + nameShownEscaped + "/": `(?ms).*href="./filename"\S+filename.*`,
 538         } {
 539                 u, _ := url.Parse(tryURL)
 540                 req := &http.Request{
 541                         Method:     "GET",
 542                         Host:       u.Host,
 543                         URL:        u,
 544                         RequestURI: u.RequestURI(),
 545                         Header: http.Header{
 546                                 "Authorization": {"Bearer " + client.AuthToken},
 547                         },
 548                 }
 549                 resp := httptest.NewRecorder()
 550                 s.testServer.Handler.ServeHTTP(resp, req)
 551                 c.Check(resp.Code, check.Equals, http.StatusOK)
 552                 c.Check(resp.Body.String(), check.Matches, expectRegexp)
 553         }
 554 }
 555
 556 // XHRs can't follow redirect-with-cookie so they rely on method=POST
 557 // and disposition=attachment (telling us it's acceptable to respond
 558 // with content instead of a redirect) and an Origin header that gets
 559 // added automatically by the browser (telling us it's desirable to do
 560 // so).
 561 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
 562         u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
 563         req := &http.Request{
 564                 Method:     "POST",
 565                 Host:       u.Host,
 566                 URL:        u,
 567                 RequestURI: u.RequestURI(),
 568                 Header: http.Header{
 569                         "Origin":       {"https://origin.example"},
 570                         "Content-Type": {"application/x-www-form-urlencoded"},
 571                 },
 572                 Body: ioutil.NopCloser(strings.NewReader(url.Values{
 573                         "api_token":   {arvadostest.ActiveToken},
 574                         "disposition": {"attachment"},
 575                 }.Encode())),
 576         }
 577         resp := httptest.NewRecorder()
 578         s.testServer.Handler.ServeHTTP(resp, req)
 579         c.Check(resp.Code, check.Equals, http.StatusOK)
 580         c.Check(resp.Body.String(), check.Equals, "foo")
 581         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
 582 }
 583
 584 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder {
 585         u, _ := url.Parse(`http://` + hostPath + queryString)
 586         req := &http.Request{
 587                 Method:     method,
 588                 Host:       u.Host,
 589                 URL:        u,
 590                 RequestURI: u.RequestURI(),
 591                 Header:     http.Header{"Content-Type": {contentType}},
 592                 Body:       ioutil.NopCloser(strings.NewReader(reqBody)),
 593         }
 594
 595         resp := httptest.NewRecorder()
 596         defer func() {
 597                 c.Check(resp.Code, check.Equals, expectStatus)
 598                 c.Check(resp.Body.String(), check.Equals, expectRespBody)
 599         }()
 600
 601         s.testServer.Handler.ServeHTTP(resp, req)
 602         if resp.Code != http.StatusSeeOther {
 603                 return resp
 604         }
 605         c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
 606         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 607
 608         u, _ = u.Parse(resp.Header().Get("Location"))
 609         req = &http.Request{
 610                 Method:     "GET",
 611                 Host:       u.Host,
 612                 URL:        u,
 613                 RequestURI: u.RequestURI(),
 614                 Header:     http.Header{},
 615         }
 616         for _, c := range cookies {
 617                 req.AddCookie(c)
 618         }
 619
 620         resp = httptest.NewRecorder()
 621         s.testServer.Handler.ServeHTTP(resp, req)
 622         c.Check(resp.Header().Get("Location"), check.Equals, "")
 623         return resp
 624 }
 625
 626 func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) {
 627         s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 628         s.testDirectoryListing(c)
 629 }
 630
 631 func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) {
 632         s.testServer.Config.cluster.Users.AnonymousUserToken = ""
 633         s.testDirectoryListing(c)
 634 }
 635
 636 func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
 637         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 638         authHeader := http.Header{
 639                 "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
 640         }
 641         for _, trial := range []struct {
 642                 uri      string
 643                 header   http.Header
 644                 expect   []string
 645                 redirect string
 646                 cutDirs  int
 647         }{
 648                 {
 649                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
 650                         header:  authHeader,
 651                         expect:  []string{"dir1/foo", "dir1/bar"},
 652                         cutDirs: 0,
 653                 },
 654                 {
 655                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
 656                         header:  authHeader,
 657                         expect:  []string{"foo", "bar"},
 658                         cutDirs: 1,
 659                 },
 660                 {
 661                         // URLs of this form ignore authHeader, and
 662                         // FooAndBarFilesInDirUUID isn't public, so
 663                         // this returns 404.
 664                         uri:    "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
 665                         header: authHeader,
 666                         expect: nil,
 667                 },
 668                 {
 669                         uri:     "download.example.com/users/active/foo_file_in_dir/",
 670                         header:  authHeader,
 671                         expect:  []string{"dir1/"},
 672                         cutDirs: 3,
 673                 },
 674                 {
 675                         uri:     "download.example.com/users/active/foo_file_in_dir/dir1/",
 676                         header:  authHeader,
 677                         expect:  []string{"bar"},
 678                         cutDirs: 4,
 679                 },
 680                 {
 681                         uri:     "download.example.com/",
 682                         header:  authHeader,
 683                         expect:  []string{"users/"},
 684                         cutDirs: 0,
 685                 },
 686                 {
 687                         uri:      "download.example.com/users",
 688                         header:   authHeader,
 689                         redirect: "/users/",
 690                         expect:   []string{"active/"},
 691                         cutDirs:  1,
 692                 },
 693                 {
 694                         uri:     "download.example.com/users/",
 695                         header:  authHeader,
 696                         expect:  []string{"active/"},
 697                         cutDirs: 1,
 698                 },
 699                 {
 700                         uri:      "download.example.com/users/active",
 701                         header:   authHeader,
 702                         redirect: "/users/active/",
 703                         expect:   []string{"foo_file_in_dir/"},
 704                         cutDirs:  2,
 705                 },
 706                 {
 707                         uri:     "download.example.com/users/active/",
 708                         header:  authHeader,
 709                         expect:  []string{"foo_file_in_dir/"},
 710                         cutDirs: 2,
 711                 },
 712                 {
 713                         uri:     "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
 714                         header:  nil,
 715                         expect:  []string{"dir1/foo", "dir1/bar"},
 716                         cutDirs: 4,
 717                 },
 718                 {
 719                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
 720                         header:  nil,
 721                         expect:  []string{"dir1/foo", "dir1/bar"},
 722                         cutDirs: 2,
 723                 },
 724                 {
 725                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken,
 726                         header:  nil,
 727                         expect:  []string{"dir1/foo", "dir1/bar"},
 728                         cutDirs: 2,
 729                 },
 730                 {
 731                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID,
 732                         header:  authHeader,
 733                         expect:  []string{"dir1/foo", "dir1/bar"},
 734                         cutDirs: 1,
 735                 },
 736                 {
 737                         uri:      "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
 738                         header:   authHeader,
 739                         redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
 740                         expect:   []string{"foo", "bar"},
 741                         cutDirs:  2,
 742                 },
 743                 {
 744                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
 745                         header:  authHeader,
 746                         expect:  []string{"foo", "bar"},
 747                         cutDirs: 3,
 748                 },
 749                 {
 750                         uri:      arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
 751                         header:   authHeader,
 752                         redirect: "/dir1/",
 753                         expect:   []string{"foo", "bar"},
 754                         cutDirs:  1,
 755                 },
 756                 {
 757                         uri:    "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
 758                         header: authHeader,
 759                         expect: nil,
 760                 },
 761                 {
 762                         uri:     "download.example.com/c=" + arvadostest.WazVersion1Collection,
 763                         header:  authHeader,
 764                         expect:  []string{"waz"},
 765                         cutDirs: 1,
 766                 },
 767                 {
 768                         uri:     "download.example.com/by_id/" + arvadostest.WazVersion1Collection,
 769                         header:  authHeader,
 770                         expect:  []string{"waz"},
 771                         cutDirs: 2,
 772                 },
 773         } {
 774                 comment := check.Commentf("HTML: %q => %q", trial.uri, trial.expect)
 775                 resp := httptest.NewRecorder()
 776                 u := mustParseURL("//" + trial.uri)
 777                 req := &http.Request{
 778                         Method:     "GET",
 779                         Host:       u.Host,
 780                         URL:        u,
 781                         RequestURI: u.RequestURI(),
 782                         Header:     copyHeader(trial.header),
 783                 }
 784                 s.testServer.Handler.ServeHTTP(resp, req)
 785                 var cookies []*http.Cookie
 786                 for resp.Code == http.StatusSeeOther {
 787                         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 788                         req = &http.Request{
 789                                 Method:     "GET",
 790                                 Host:       u.Host,
 791                                 URL:        u,
 792                                 RequestURI: u.RequestURI(),
 793                                 Header:     copyHeader(trial.header),
 794                         }
 795                         cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
 796                         for _, c := range cookies {
 797                                 req.AddCookie(c)
 798                         }
 799                         resp = httptest.NewRecorder()
 800                         s.testServer.Handler.ServeHTTP(resp, req)
 801                 }
 802                 if trial.redirect != "" {
 803                         c.Check(req.URL.Path, check.Equals, trial.redirect, comment)
 804                 }
 805                 if trial.expect == nil {
 806                         c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
 807                 } else {
 808                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
 809                         for _, e := range trial.expect {
 810                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`, comment)
 811                         }
 812                         c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`, comment)
 813                 }
 814
 815                 comment = check.Commentf("WebDAV: %q => %q", trial.uri, trial.expect)
 816                 req = &http.Request{
 817                         Method:     "OPTIONS",
 818                         Host:       u.Host,
 819                         URL:        u,
 820                         RequestURI: u.RequestURI(),
 821                         Header:     copyHeader(trial.header),
 822                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
 823                 }
 824                 resp = httptest.NewRecorder()
 825                 s.testServer.Handler.ServeHTTP(resp, req)
 826                 if trial.expect == nil {
 827                         c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
 828                 } else {
 829                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
 830                 }
 831
 832                 req = &http.Request{
 833                         Method:     "PROPFIND",
 834                         Host:       u.Host,
 835                         URL:        u,
 836                         RequestURI: u.RequestURI(),
 837                         Header:     copyHeader(trial.header),
 838                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
 839                 }
 840                 resp = httptest.NewRecorder()
 841                 s.testServer.Handler.ServeHTTP(resp, req)
 842                 if trial.expect == nil {
 843                         c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
 844                 } else {
 845                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment)
 846                         for _, e := range trial.expect {
 847                                 if strings.HasSuffix(e, "/") {
 848                                         e = filepath.Join(u.Path, e) + "/"
 849                                 } else {
 850                                         e = filepath.Join(u.Path, e)
 851                                 }
 852                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+e+`</D:href>.*`, comment)
 853                         }
 854                 }
 855         }
 856 }
 857
 858 func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
 859         arv := arvados.NewClientFromEnv()
 860         var newCollection arvados.Collection
 861         err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 862                 "collection": map[string]string{
 863                         "owner_uuid":    arvadostest.ActiveUserUUID,
 864                         "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
 865                         "name":          "keep-web test collection",
 866                 },
 867                 "ensure_unique_name": true,
 868         })
 869         c.Assert(err, check.IsNil)
 870         defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
 871
 872         var updated arvados.Collection
 873         for _, fnm := range []string{"foo.txt", "bar.txt"} {
 874                 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
 875                 u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
 876                 req := &http.Request{
 877                         Method:     "DELETE",
 878                         Host:       u.Host,
 879                         URL:        u,
 880                         RequestURI: u.RequestURI(),
 881                         Header: http.Header{
 882                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
 883                         },
 884                 }
 885                 resp := httptest.NewRecorder()
 886                 s.testServer.Handler.ServeHTTP(resp, req)
 887                 c.Check(resp.Code, check.Equals, http.StatusNoContent)
 888
 889                 updated = arvados.Collection{}
 890                 err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
 891                 c.Check(err, check.IsNil)
 892                 c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`)
 893                 c.Logf("updated manifest_text %q", updated.ManifestText)
 894         }
 895         c.Check(updated.ManifestText, check.Equals, "")
 896 }
 897
 898 func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) {
 899         s.testServer.Config.cluster.ManagementToken = arvadostest.ManagementToken
 900         authHeader := http.Header{
 901                 "Authorization": {"Bearer " + arvadostest.ManagementToken},
 902         }
 903
 904         resp := httptest.NewRecorder()
 905         u := mustParseURL("http://download.example.com/_health/ping")
 906         req := &http.Request{
 907                 Method:     "GET",
 908                 Host:       u.Host,
 909                 URL:        u,
 910                 RequestURI: u.RequestURI(),
 911                 Header:     authHeader,
 912         }
 913         s.testServer.Handler.ServeHTTP(resp, req)
 914
 915         c.Check(resp.Code, check.Equals, http.StatusOK)
 916         c.Check(resp.Body.String(), check.Matches, `{"health":"OK"}\n`)
 917 }
 918
 919 func (s *IntegrationSuite) TestFileContentType(c *check.C) {
 920         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 921
 922         client := s.testServer.Config.Client
 923         client.AuthToken = arvadostest.ActiveToken
 924         arv, err := arvadosclient.New(&client)
 925         c.Assert(err, check.Equals, nil)
 926         kc, err := keepclient.MakeKeepClient(arv)
 927         c.Assert(err, check.Equals, nil)
 928
 929         fs, err := (&arvados.Collection{}).FileSystem(&client, kc)
 930         c.Assert(err, check.IsNil)
 931
 932         trials := []struct {
 933                 filename    string
 934                 content     string
 935                 contentType string
 936         }{
 937                 {"picture.txt", "BMX bikes are small this year\n", "text/plain; charset=utf-8"},
 938                 {"picture.bmp", "BMX bikes are small this year\n", "image/x-ms-bmp"},
 939                 {"picture.jpg", "BMX bikes are small this year\n", "image/jpeg"},
 940                 {"picture1", "BMX bikes are small this year\n", "image/bmp"},            // content sniff; "BM" is the magic signature for .bmp
 941                 {"picture2", "Cars are small this year\n", "text/plain; charset=utf-8"}, // content sniff
 942         }
 943         for _, trial := range trials {
 944                 f, err := fs.OpenFile(trial.filename, os.O_CREATE|os.O_WRONLY, 0777)
 945                 c.Assert(err, check.IsNil)
 946                 _, err = f.Write([]byte(trial.content))
 947                 c.Assert(err, check.IsNil)
 948                 c.Assert(f.Close(), check.IsNil)
 949         }
 950         mtxt, err := fs.MarshalManifest(".")
 951         c.Assert(err, check.IsNil)
 952         var coll arvados.Collection
 953         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 954                 "collection": map[string]string{
 955                         "manifest_text": mtxt,
 956                 },
 957         })
 958         c.Assert(err, check.IsNil)
 959
 960         for _, trial := range trials {
 961                 u, _ := url.Parse("http://download.example.com/by_id/" + coll.UUID + "/" + trial.filename)
 962                 req := &http.Request{
 963                         Method:     "GET",
 964                         Host:       u.Host,
 965                         URL:        u,
 966                         RequestURI: u.RequestURI(),
 967                         Header: http.Header{
 968                                 "Authorization": {"Bearer " + client.AuthToken},
 969                         },
 970                 }
 971                 resp := httptest.NewRecorder()
 972                 s.testServer.Handler.ServeHTTP(resp, req)
 973                 c.Check(resp.Code, check.Equals, http.StatusOK)
 974                 c.Check(resp.Header().Get("Content-Type"), check.Equals, trial.contentType)
 975                 c.Check(resp.Body.String(), check.Equals, trial.content)
 976         }
 977 }
 978
 979 func (s *IntegrationSuite) TestKeepClientBlockCache(c *check.C) {
 980         s.testServer.Config.cluster.Collections.WebDAVCache.MaxBlockEntries = 42
 981         c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Not(check.Equals), 42)
 982         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/t=" + arvadostest.ActiveToken + "/foo")
 983         req := &http.Request{
 984                 Method:     "GET",
 985                 Host:       u.Host,
 986                 URL:        u,
 987                 RequestURI: u.RequestURI(),
 988         }
 989         resp := httptest.NewRecorder()
 990         s.testServer.Handler.ServeHTTP(resp, req)
 991         c.Check(resp.Code, check.Equals, http.StatusOK)
 992         c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Equals, 42)
 993 }
 994
 995 func copyHeader(h http.Header) http.Header {
 996         hc := http.Header{}
 997         for k, v := range h {
 998                 hc[k] = append([]string(nil), v...)
 999         }
1000         return hc
1001 }