services/keep-web/handler_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "fmt"
  10         "html"
  11         "io/ioutil"
  12         "net/http"
  13         "net/http/httptest"
  14         "net/url"
  15         "os"
  16         "path/filepath"
  17         "regexp"
  18         "strings"
  19
  20         "git.arvados.org/arvados.git/lib/config"
  21         "git.arvados.org/arvados.git/sdk/go/arvados"
  22         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  23         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  24         "git.arvados.org/arvados.git/sdk/go/auth"
  25         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  26         "git.arvados.org/arvados.git/sdk/go/keepclient"
  27         check "gopkg.in/check.v1"
  28 )
  29
  30 var _ = check.Suite(&UnitSuite{})
  31
  32 type UnitSuite struct {
  33         Config *arvados.Config
  34 }
  35
  36 func (s *UnitSuite) SetUpTest(c *check.C) {
  37         ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), ctxlog.TestLogger(c))
  38         ldr.Path = "-"
  39         cfg, err := ldr.Load()
  40         c.Assert(err, check.IsNil)
  41         s.Config = cfg
  42 }
  43
  44 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
  45         h := handler{Config: newConfig(s.Config)}
  46         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
  47         req := &http.Request{
  48                 Method:     "OPTIONS",
  49                 Host:       u.Host,
  50                 URL:        u,
  51                 RequestURI: u.RequestURI(),
  52                 Header: http.Header{
  53                         "Origin":                        {"https://workbench.example"},
  54                         "Access-Control-Request-Method": {"POST"},
  55                 },
  56         }
  57
  58         // Check preflight for an allowed request
  59         resp := httptest.NewRecorder()
  60         h.ServeHTTP(resp, req)
  61         c.Check(resp.Code, check.Equals, http.StatusOK)
  62         c.Check(resp.Body.String(), check.Equals, "")
  63         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
  64         c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
  65         c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout")
  66
  67         // Check preflight for a disallowed request
  68         resp = httptest.NewRecorder()
  69         req.Header.Set("Access-Control-Request-Method", "MAKE-COFFEE")
  70         h.ServeHTTP(resp, req)
  71         c.Check(resp.Body.String(), check.Equals, "")
  72         c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
  73 }
  74
  75 func (s *UnitSuite) TestInvalidUUID(c *check.C) {
  76         bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-"
  77         token := arvadostest.ActiveToken
  78         for _, trial := range []string{
  79                 "http://keep-web/c=" + bogusID + "/foo",
  80                 "http://keep-web/c=" + bogusID + "/t=" + token + "/foo",
  81                 "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo",
  82                 "http://keep-web/collections/" + bogusID + "/foo",
  83                 "http://" + bogusID + ".keep-web/" + bogusID + "/foo",
  84                 "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
  85         } {
  86                 c.Log(trial)
  87                 u := mustParseURL(trial)
  88                 req := &http.Request{
  89                         Method:     "GET",
  90                         Host:       u.Host,
  91                         URL:        u,
  92                         RequestURI: u.RequestURI(),
  93                 }
  94                 resp := httptest.NewRecorder()
  95                 cfg := newConfig(s.Config)
  96                 cfg.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
  97                 h := handler{Config: cfg}
  98                 h.ServeHTTP(resp, req)
  99                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 100         }
 101 }
 102
 103 func mustParseURL(s string) *url.URL {
 104         r, err := url.Parse(s)
 105         if err != nil {
 106                 panic("parse URL: " + s)
 107         }
 108         return r
 109 }
 110
 111 func (s *IntegrationSuite) TestVhost404(c *check.C) {
 112         for _, testURL := range []string{
 113                 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
 114                 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
 115         } {
 116                 resp := httptest.NewRecorder()
 117                 u := mustParseURL(testURL)
 118                 req := &http.Request{
 119                         Method:     "GET",
 120                         URL:        u,
 121                         RequestURI: u.RequestURI(),
 122                 }
 123                 s.testServer.Handler.ServeHTTP(resp, req)
 124                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 125                 c.Check(resp.Body.String(), check.Equals, notFoundMessage+"\n")
 126         }
 127 }
 128
 129 // An authorizer modifies an HTTP request to make use of the given
 130 // token -- by adding it to a header, cookie, query param, or whatever
 131 // -- and returns the HTTP status code we should expect from keep-web if
 132 // the token is invalid.
 133 type authorizer func(*http.Request, string) int
 134
 135 func (s *IntegrationSuite) TestVhostViaAuthzHeader(c *check.C) {
 136         s.doVhostRequests(c, authzViaAuthzHeader)
 137 }
 138 func authzViaAuthzHeader(r *http.Request, tok string) int {
 139         r.Header.Add("Authorization", "OAuth2 "+tok)
 140         return http.StatusUnauthorized
 141 }
 142
 143 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
 144         s.doVhostRequests(c, authzViaCookieValue)
 145 }
 146 func authzViaCookieValue(r *http.Request, tok string) int {
 147         r.AddCookie(&http.Cookie{
 148                 Name:  "arvados_api_token",
 149                 Value: auth.EncodeTokenCookie([]byte(tok)),
 150         })
 151         return http.StatusUnauthorized
 152 }
 153
 154 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
 155         s.doVhostRequests(c, authzViaPath)
 156 }
 157 func authzViaPath(r *http.Request, tok string) int {
 158         r.URL.Path = "/t=" + tok + r.URL.Path
 159         return http.StatusNotFound
 160 }
 161
 162 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
 163         s.doVhostRequests(c, authzViaQueryString)
 164 }
 165 func authzViaQueryString(r *http.Request, tok string) int {
 166         r.URL.RawQuery = "api_token=" + tok
 167         return http.StatusUnauthorized
 168 }
 169
 170 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
 171         s.doVhostRequests(c, authzViaPOST)
 172 }
 173 func authzViaPOST(r *http.Request, tok string) int {
 174         r.Method = "POST"
 175         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 176         r.Body = ioutil.NopCloser(strings.NewReader(
 177                 url.Values{"api_token": {tok}}.Encode()))
 178         return http.StatusUnauthorized
 179 }
 180
 181 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
 182         s.doVhostRequests(c, authzViaPOST)
 183 }
 184 func authzViaXHRPOST(r *http.Request, tok string) int {
 185         r.Method = "POST"
 186         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 187         r.Header.Add("Origin", "https://origin.example")
 188         r.Body = ioutil.NopCloser(strings.NewReader(
 189                 url.Values{
 190                         "api_token":   {tok},
 191                         "disposition": {"attachment"},
 192                 }.Encode()))
 193         return http.StatusUnauthorized
 194 }
 195
 196 // Try some combinations of {url, token} using the given authorization
 197 // mechanism, and verify the result is correct.
 198 func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
 199         for _, hostPath := range []string{
 200                 arvadostest.FooCollection + ".example.com/foo",
 201                 arvadostest.FooCollection + "--collections.example.com/foo",
 202                 arvadostest.FooCollection + "--collections.example.com/_/foo",
 203                 arvadostest.FooCollectionPDH + ".example.com/foo",
 204                 strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + "--collections.example.com/foo",
 205                 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
 206         } {
 207                 c.Log("doRequests: ", hostPath)
 208                 s.doVhostRequestsWithHostPath(c, authz, hostPath)
 209         }
 210 }
 211
 212 func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
 213         for _, tok := range []string{
 214                 arvadostest.ActiveToken,
 215                 arvadostest.ActiveToken[:15],
 216                 arvadostest.SpectatorToken,
 217                 "bogus",
 218                 "",
 219         } {
 220                 u := mustParseURL("http://" + hostPath)
 221                 req := &http.Request{
 222                         Method:     "GET",
 223                         Host:       u.Host,
 224                         URL:        u,
 225                         RequestURI: u.RequestURI(),
 226                         Header:     http.Header{},
 227                 }
 228                 failCode := authz(req, tok)
 229                 req, resp := s.doReq(req)
 230                 code, body := resp.Code, resp.Body.String()
 231
 232                 // If the initial request had a (non-empty) token
 233                 // showing in the query string, we should have been
 234                 // redirected in order to hide it in a cookie.
 235                 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
 236
 237                 if tok == arvadostest.ActiveToken {
 238                         c.Check(code, check.Equals, http.StatusOK)
 239                         c.Check(body, check.Equals, "foo")
 240
 241                 } else {
 242                         c.Check(code >= 400, check.Equals, true)
 243                         c.Check(code < 500, check.Equals, true)
 244                         if tok == arvadostest.SpectatorToken {
 245                                 // Valid token never offers to retry
 246                                 // with different credentials.
 247                                 c.Check(code, check.Equals, http.StatusNotFound)
 248                         } else {
 249                                 // Invalid token can ask to retry
 250                                 // depending on the authz method.
 251                                 c.Check(code, check.Equals, failCode)
 252                         }
 253                         if code == 404 {
 254                                 c.Check(body, check.Equals, notFoundMessage+"\n")
 255                         } else {
 256                                 c.Check(body, check.Equals, unauthorizedMessage+"\n")
 257                         }
 258                 }
 259         }
 260 }
 261
 262 func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
 263         resp := httptest.NewRecorder()
 264         s.testServer.Handler.ServeHTTP(resp, req)
 265         if resp.Code != http.StatusSeeOther {
 266                 return req, resp
 267         }
 268         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 269         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 270         req = &http.Request{
 271                 Method:     "GET",
 272                 Host:       u.Host,
 273                 URL:        u,
 274                 RequestURI: u.RequestURI(),
 275                 Header:     http.Header{},
 276         }
 277         for _, c := range cookies {
 278                 req.AddCookie(c)
 279         }
 280         return s.doReq(req)
 281 }
 282
 283 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
 284         s.testVhostRedirectTokenToCookie(c, "GET",
 285                 arvadostest.FooCollection+".example.com/foo",
 286                 "?api_token="+arvadostest.ActiveToken,
 287                 "",
 288                 "",
 289                 http.StatusOK,
 290                 "foo",
 291         )
 292 }
 293
 294 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
 295         s.testVhostRedirectTokenToCookie(c, "GET",
 296                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
 297                 "",
 298                 "",
 299                 "",
 300                 http.StatusOK,
 301                 "foo",
 302         )
 303 }
 304
 305 // Bad token in URL is 404 Not Found because it doesn't make sense to
 306 // retry the same URL with different authorization.
 307 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
 308         s.testVhostRedirectTokenToCookie(c, "GET",
 309                 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
 310                 "",
 311                 "",
 312                 "",
 313                 http.StatusNotFound,
 314                 notFoundMessage+"\n",
 315         )
 316 }
 317
 318 // Bad token in a cookie (even if it got there via our own
 319 // query-string-to-cookie redirect) is, in principle, retryable at the
 320 // same URL so it's 401 Unauthorized.
 321 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
 322         s.testVhostRedirectTokenToCookie(c, "GET",
 323                 arvadostest.FooCollection+".example.com/foo",
 324                 "?api_token=thisisabogustoken",
 325                 "",
 326                 "",
 327                 http.StatusUnauthorized,
 328                 unauthorizedMessage+"\n",
 329         )
 330 }
 331
 332 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
 333         s.testVhostRedirectTokenToCookie(c, "GET",
 334                 "example.com/c="+arvadostest.FooCollection+"/foo",
 335                 "?api_token="+arvadostest.ActiveToken,
 336                 "",
 337                 "",
 338                 http.StatusBadRequest,
 339                 "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n",
 340         )
 341 }
 342
 343 // If client requests an attachment by putting ?disposition=attachment
 344 // in the query string, and gets redirected, the redirect target
 345 // should respond with an attachment.
 346 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
 347         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 348                 arvadostest.FooCollection+".example.com/foo",
 349                 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
 350                 "",
 351                 "",
 352                 http.StatusOK,
 353                 "foo",
 354         )
 355         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 356 }
 357
 358 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
 359         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 360         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 361                 "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
 362                 "?api_token="+arvadostest.ActiveToken,
 363                 "",
 364                 "",
 365                 http.StatusOK,
 366                 "foo",
 367         )
 368         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 369 }
 370
 371 func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
 372         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 373         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 374                 "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
 375                 "?api_token="+arvadostest.ActiveToken,
 376                 "",
 377                 "",
 378                 http.StatusOK,
 379                 "waz",
 380         )
 381         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 382         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 383                 "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz",
 384                 "?api_token="+arvadostest.ActiveToken,
 385                 "",
 386                 "",
 387                 http.StatusOK,
 388                 "waz",
 389         )
 390         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 391 }
 392
 393 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
 394         s.testServer.Config.cluster.Collections.TrustAllContent = true
 395         s.testVhostRedirectTokenToCookie(c, "GET",
 396                 "example.com/c="+arvadostest.FooCollection+"/foo",
 397                 "?api_token="+arvadostest.ActiveToken,
 398                 "",
 399                 "",
 400                 http.StatusOK,
 401                 "foo",
 402         )
 403 }
 404
 405 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
 406         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
 407
 408         s.testVhostRedirectTokenToCookie(c, "GET",
 409                 "example.com/c="+arvadostest.FooCollection+"/foo",
 410                 "?api_token="+arvadostest.ActiveToken,
 411                 "",
 412                 "",
 413                 http.StatusBadRequest,
 414                 "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n",
 415         )
 416
 417         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 418                 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
 419                 "?api_token="+arvadostest.ActiveToken,
 420                 "",
 421                 "",
 422                 http.StatusOK,
 423                 "foo",
 424         )
 425         c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
 426 }
 427
 428 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
 429         s.testVhostRedirectTokenToCookie(c, "POST",
 430                 arvadostest.FooCollection+".example.com/foo",
 431                 "",
 432                 "application/x-www-form-urlencoded",
 433                 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
 434                 http.StatusOK,
 435                 "foo",
 436         )
 437 }
 438
 439 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
 440         s.testVhostRedirectTokenToCookie(c, "POST",
 441                 arvadostest.FooCollection+".example.com/foo",
 442                 "",
 443                 "application/x-www-form-urlencoded",
 444                 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
 445                 http.StatusNotFound,
 446                 notFoundMessage+"\n",
 447         )
 448 }
 449
 450 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
 451         s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 452         s.testVhostRedirectTokenToCookie(c, "GET",
 453                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 454                 "",
 455                 "",
 456                 "",
 457                 http.StatusOK,
 458                 "Hello world\n",
 459         )
 460 }
 461
 462 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
 463         s.testServer.Config.cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
 464         s.testVhostRedirectTokenToCookie(c, "GET",
 465                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 466                 "",
 467                 "",
 468                 "",
 469                 http.StatusNotFound,
 470                 notFoundMessage+"\n",
 471         )
 472 }
 473
 474 func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
 475         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 476
 477         client := s.testServer.Config.Client
 478         client.AuthToken = arvadostest.ActiveToken
 479         fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
 480         c.Assert(err, check.IsNil)
 481         f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777)
 482         c.Assert(err, check.IsNil)
 483         f.Close()
 484         mtxt, err := fs.MarshalManifest(".")
 485         c.Assert(err, check.IsNil)
 486         var coll arvados.Collection
 487         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 488                 "collection": map[string]string{
 489                         "manifest_text": mtxt,
 490                 },
 491         })
 492         c.Assert(err, check.IsNil)
 493
 494         u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
 495         req := &http.Request{
 496                 Method:     "GET",
 497                 Host:       u.Host,
 498                 URL:        u,
 499                 RequestURI: u.RequestURI(),
 500                 Header: http.Header{
 501                         "Authorization": {"Bearer " + client.AuthToken},
 502                 },
 503         }
 504         resp := httptest.NewRecorder()
 505         s.testServer.Handler.ServeHTTP(resp, req)
 506         c.Check(resp.Code, check.Equals, http.StatusOK)
 507         c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\&#34;odd&#39; path chars.*`)
 508 }
 509
 510 func (s *IntegrationSuite) TestForwardSlashSubstitution(c *check.C) {
 511         arv := arvados.NewClientFromEnv()
 512         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 513         s.testServer.Config.cluster.Collections.ForwardSlashNameSubstitution = "{SOLIDUS}"
 514         name := "foo/bar/baz"
 515         nameShown := strings.Replace(name, "/", "{SOLIDUS}", -1)
 516         nameShownEscaped := strings.Replace(name, "/", "%7bSOLIDUS%7d", -1)
 517
 518         client := s.testServer.Config.Client
 519         client.AuthToken = arvadostest.ActiveToken
 520         fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
 521         c.Assert(err, check.IsNil)
 522         f, err := fs.OpenFile("filename", os.O_CREATE, 0777)
 523         c.Assert(err, check.IsNil)
 524         f.Close()
 525         mtxt, err := fs.MarshalManifest(".")
 526         c.Assert(err, check.IsNil)
 527         var coll arvados.Collection
 528         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 529                 "collection": map[string]string{
 530                         "manifest_text": mtxt,
 531                         "name":          name,
 532                         "owner_uuid":    arvadostest.AProjectUUID,
 533                 },
 534         })
 535         c.Assert(err, check.IsNil)
 536         defer arv.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
 537
 538         base := "http://download.example.com/by_id/" + coll.OwnerUUID + "/"
 539         for tryURL, expectRegexp := range map[string]string{
 540                 base:                          `(?ms).*href="./` + nameShownEscaped + `/"\S+` + nameShown + `.*`,
 541                 base + nameShownEscaped + "/": `(?ms).*href="./filename"\S+filename.*`,
 542         } {
 543                 u, _ := url.Parse(tryURL)
 544                 req := &http.Request{
 545                         Method:     "GET",
 546                         Host:       u.Host,
 547                         URL:        u,
 548                         RequestURI: u.RequestURI(),
 549                         Header: http.Header{
 550                                 "Authorization": {"Bearer " + client.AuthToken},
 551                         },
 552                 }
 553                 resp := httptest.NewRecorder()
 554                 s.testServer.Handler.ServeHTTP(resp, req)
 555                 c.Check(resp.Code, check.Equals, http.StatusOK)
 556                 c.Check(resp.Body.String(), check.Matches, expectRegexp)
 557         }
 558 }
 559
 560 // XHRs can't follow redirect-with-cookie so they rely on method=POST
 561 // and disposition=attachment (telling us it's acceptable to respond
 562 // with content instead of a redirect) and an Origin header that gets
 563 // added automatically by the browser (telling us it's desirable to do
 564 // so).
 565 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
 566         u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
 567         req := &http.Request{
 568                 Method:     "POST",
 569                 Host:       u.Host,
 570                 URL:        u,
 571                 RequestURI: u.RequestURI(),
 572                 Header: http.Header{
 573                         "Origin":       {"https://origin.example"},
 574                         "Content-Type": {"application/x-www-form-urlencoded"},
 575                 },
 576                 Body: ioutil.NopCloser(strings.NewReader(url.Values{
 577                         "api_token":   {arvadostest.ActiveToken},
 578                         "disposition": {"attachment"},
 579                 }.Encode())),
 580         }
 581         resp := httptest.NewRecorder()
 582         s.testServer.Handler.ServeHTTP(resp, req)
 583         c.Check(resp.Code, check.Equals, http.StatusOK)
 584         c.Check(resp.Body.String(), check.Equals, "foo")
 585         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
 586 }
 587
 588 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder {
 589         u, _ := url.Parse(`http://` + hostPath + queryString)
 590         req := &http.Request{
 591                 Method:     method,
 592                 Host:       u.Host,
 593                 URL:        u,
 594                 RequestURI: u.RequestURI(),
 595                 Header:     http.Header{"Content-Type": {contentType}},
 596                 Body:       ioutil.NopCloser(strings.NewReader(reqBody)),
 597         }
 598
 599         resp := httptest.NewRecorder()
 600         defer func() {
 601                 c.Check(resp.Code, check.Equals, expectStatus)
 602                 c.Check(resp.Body.String(), check.Equals, expectRespBody)
 603         }()
 604
 605         s.testServer.Handler.ServeHTTP(resp, req)
 606         if resp.Code != http.StatusSeeOther {
 607                 return resp
 608         }
 609         c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
 610         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 611
 612         u, _ = u.Parse(resp.Header().Get("Location"))
 613         req = &http.Request{
 614                 Method:     "GET",
 615                 Host:       u.Host,
 616                 URL:        u,
 617                 RequestURI: u.RequestURI(),
 618                 Header:     http.Header{},
 619         }
 620         for _, c := range cookies {
 621                 req.AddCookie(c)
 622         }
 623
 624         resp = httptest.NewRecorder()
 625         s.testServer.Handler.ServeHTTP(resp, req)
 626         c.Check(resp.Header().Get("Location"), check.Equals, "")
 627         return resp
 628 }
 629
 630 func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) {
 631         s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 632         s.testDirectoryListing(c)
 633 }
 634
 635 func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) {
 636         s.testServer.Config.cluster.Users.AnonymousUserToken = ""
 637         s.testDirectoryListing(c)
 638 }
 639
 640 func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
 641         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 642         authHeader := http.Header{
 643                 "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
 644         }
 645         for _, trial := range []struct {
 646                 uri      string
 647                 header   http.Header
 648                 expect   []string
 649                 redirect string
 650                 cutDirs  int
 651         }{
 652                 {
 653                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
 654                         header:  authHeader,
 655                         expect:  []string{"dir1/foo", "dir1/bar"},
 656                         cutDirs: 0,
 657                 },
 658                 {
 659                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
 660                         header:  authHeader,
 661                         expect:  []string{"foo", "bar"},
 662                         cutDirs: 1,
 663                 },
 664                 {
 665                         // URLs of this form ignore authHeader, and
 666                         // FooAndBarFilesInDirUUID isn't public, so
 667                         // this returns 404.
 668                         uri:    "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
 669                         header: authHeader,
 670                         expect: nil,
 671                 },
 672                 {
 673                         uri:     "download.example.com/users/active/foo_file_in_dir/",
 674                         header:  authHeader,
 675                         expect:  []string{"dir1/"},
 676                         cutDirs: 3,
 677                 },
 678                 {
 679                         uri:     "download.example.com/users/active/foo_file_in_dir/dir1/",
 680                         header:  authHeader,
 681                         expect:  []string{"bar"},
 682                         cutDirs: 4,
 683                 },
 684                 {
 685                         uri:     "download.example.com/",
 686                         header:  authHeader,
 687                         expect:  []string{"users/"},
 688                         cutDirs: 0,
 689                 },
 690                 {
 691                         uri:      "download.example.com/users",
 692                         header:   authHeader,
 693                         redirect: "/users/",
 694                         expect:   []string{"active/"},
 695                         cutDirs:  1,
 696                 },
 697                 {
 698                         uri:     "download.example.com/users/",
 699                         header:  authHeader,
 700                         expect:  []string{"active/"},
 701                         cutDirs: 1,
 702                 },
 703                 {
 704                         uri:      "download.example.com/users/active",
 705                         header:   authHeader,
 706                         redirect: "/users/active/",
 707                         expect:   []string{"foo_file_in_dir/"},
 708                         cutDirs:  2,
 709                 },
 710                 {
 711                         uri:     "download.example.com/users/active/",
 712                         header:  authHeader,
 713                         expect:  []string{"foo_file_in_dir/"},
 714                         cutDirs: 2,
 715                 },
 716                 {
 717                         uri:     "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
 718                         header:  nil,
 719                         expect:  []string{"dir1/foo", "dir1/bar"},
 720                         cutDirs: 4,
 721                 },
 722                 {
 723                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
 724                         header:  nil,
 725                         expect:  []string{"dir1/foo", "dir1/bar"},
 726                         cutDirs: 2,
 727                 },
 728                 {
 729                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken,
 730                         header:  nil,
 731                         expect:  []string{"dir1/foo", "dir1/bar"},
 732                         cutDirs: 2,
 733                 },
 734                 {
 735                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID,
 736                         header:  authHeader,
 737                         expect:  []string{"dir1/foo", "dir1/bar"},
 738                         cutDirs: 1,
 739                 },
 740                 {
 741                         uri:      "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
 742                         header:   authHeader,
 743                         redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
 744                         expect:   []string{"foo", "bar"},
 745                         cutDirs:  2,
 746                 },
 747                 {
 748                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
 749                         header:  authHeader,
 750                         expect:  []string{"foo", "bar"},
 751                         cutDirs: 3,
 752                 },
 753                 {
 754                         uri:      arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
 755                         header:   authHeader,
 756                         redirect: "/dir1/",
 757                         expect:   []string{"foo", "bar"},
 758                         cutDirs:  1,
 759                 },
 760                 {
 761                         uri:    "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
 762                         header: authHeader,
 763                         expect: nil,
 764                 },
 765                 {
 766                         uri:     "download.example.com/c=" + arvadostest.WazVersion1Collection,
 767                         header:  authHeader,
 768                         expect:  []string{"waz"},
 769                         cutDirs: 1,
 770                 },
 771                 {
 772                         uri:     "download.example.com/by_id/" + arvadostest.WazVersion1Collection,
 773                         header:  authHeader,
 774                         expect:  []string{"waz"},
 775                         cutDirs: 2,
 776                 },
 777         } {
 778                 comment := check.Commentf("HTML: %q => %q", trial.uri, trial.expect)
 779                 resp := httptest.NewRecorder()
 780                 u := mustParseURL("//" + trial.uri)
 781                 req := &http.Request{
 782                         Method:     "GET",
 783                         Host:       u.Host,
 784                         URL:        u,
 785                         RequestURI: u.RequestURI(),
 786                         Header:     copyHeader(trial.header),
 787                 }
 788                 s.testServer.Handler.ServeHTTP(resp, req)
 789                 var cookies []*http.Cookie
 790                 for resp.Code == http.StatusSeeOther {
 791                         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 792                         req = &http.Request{
 793                                 Method:     "GET",
 794                                 Host:       u.Host,
 795                                 URL:        u,
 796                                 RequestURI: u.RequestURI(),
 797                                 Header:     copyHeader(trial.header),
 798                         }
 799                         cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
 800                         for _, c := range cookies {
 801                                 req.AddCookie(c)
 802                         }
 803                         resp = httptest.NewRecorder()
 804                         s.testServer.Handler.ServeHTTP(resp, req)
 805                 }
 806                 if trial.redirect != "" {
 807                         c.Check(req.URL.Path, check.Equals, trial.redirect, comment)
 808                 }
 809                 if trial.expect == nil {
 810                         c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
 811                 } else {
 812                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
 813                         for _, e := range trial.expect {
 814                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`, comment)
 815                         }
 816                         c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`, comment)
 817                 }
 818
 819                 comment = check.Commentf("WebDAV: %q => %q", trial.uri, trial.expect)
 820                 req = &http.Request{
 821                         Method:     "OPTIONS",
 822                         Host:       u.Host,
 823                         URL:        u,
 824                         RequestURI: u.RequestURI(),
 825                         Header:     copyHeader(trial.header),
 826                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
 827                 }
 828                 resp = httptest.NewRecorder()
 829                 s.testServer.Handler.ServeHTTP(resp, req)
 830                 if trial.expect == nil {
 831                         c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
 832                 } else {
 833                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
 834                 }
 835
 836                 req = &http.Request{
 837                         Method:     "PROPFIND",
 838                         Host:       u.Host,
 839                         URL:        u,
 840                         RequestURI: u.RequestURI(),
 841                         Header:     copyHeader(trial.header),
 842                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
 843                 }
 844                 resp = httptest.NewRecorder()
 845                 s.testServer.Handler.ServeHTTP(resp, req)
 846                 if trial.expect == nil {
 847                         c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
 848                 } else {
 849                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment)
 850                         for _, e := range trial.expect {
 851                                 if strings.HasSuffix(e, "/") {
 852                                         e = filepath.Join(u.Path, e) + "/"
 853                                 } else {
 854                                         e = filepath.Join(u.Path, e)
 855                                 }
 856                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+e+`</D:href>.*`, comment)
 857                         }
 858                 }
 859         }
 860 }
 861
 862 func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
 863         arv := arvados.NewClientFromEnv()
 864         var newCollection arvados.Collection
 865         err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 866                 "collection": map[string]string{
 867                         "owner_uuid":    arvadostest.ActiveUserUUID,
 868                         "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
 869                         "name":          "keep-web test collection",
 870                 },
 871                 "ensure_unique_name": true,
 872         })
 873         c.Assert(err, check.IsNil)
 874         defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
 875
 876         var updated arvados.Collection
 877         for _, fnm := range []string{"foo.txt", "bar.txt"} {
 878                 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
 879                 u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
 880                 req := &http.Request{
 881                         Method:     "DELETE",
 882                         Host:       u.Host,
 883                         URL:        u,
 884                         RequestURI: u.RequestURI(),
 885                         Header: http.Header{
 886                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
 887                         },
 888                 }
 889                 resp := httptest.NewRecorder()
 890                 s.testServer.Handler.ServeHTTP(resp, req)
 891                 c.Check(resp.Code, check.Equals, http.StatusNoContent)
 892
 893                 updated = arvados.Collection{}
 894                 err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
 895                 c.Check(err, check.IsNil)
 896                 c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`)
 897                 c.Logf("updated manifest_text %q", updated.ManifestText)
 898         }
 899         c.Check(updated.ManifestText, check.Equals, "")
 900 }
 901
 902 func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) {
 903         s.testServer.Config.cluster.ManagementToken = arvadostest.ManagementToken
 904         authHeader := http.Header{
 905                 "Authorization": {"Bearer " + arvadostest.ManagementToken},
 906         }
 907
 908         resp := httptest.NewRecorder()
 909         u := mustParseURL("http://download.example.com/_health/ping")
 910         req := &http.Request{
 911                 Method:     "GET",
 912                 Host:       u.Host,
 913                 URL:        u,
 914                 RequestURI: u.RequestURI(),
 915                 Header:     authHeader,
 916         }
 917         s.testServer.Handler.ServeHTTP(resp, req)
 918
 919         c.Check(resp.Code, check.Equals, http.StatusOK)
 920         c.Check(resp.Body.String(), check.Matches, `{"health":"OK"}\n`)
 921 }
 922
 923 func (s *IntegrationSuite) TestFileContentType(c *check.C) {
 924         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 925
 926         client := s.testServer.Config.Client
 927         client.AuthToken = arvadostest.ActiveToken
 928         arv, err := arvadosclient.New(&client)
 929         c.Assert(err, check.Equals, nil)
 930         kc, err := keepclient.MakeKeepClient(arv)
 931         c.Assert(err, check.Equals, nil)
 932
 933         fs, err := (&arvados.Collection{}).FileSystem(&client, kc)
 934         c.Assert(err, check.IsNil)
 935
 936         trials := []struct {
 937                 filename    string
 938                 content     string
 939                 contentType string
 940         }{
 941                 {"picture.txt", "BMX bikes are small this year\n", "text/plain; charset=utf-8"},
 942                 {"picture.bmp", "BMX bikes are small this year\n", "image/x-ms-bmp"},
 943                 {"picture.jpg", "BMX bikes are small this year\n", "image/jpeg"},
 944                 {"picture1", "BMX bikes are small this year\n", "image/bmp"},            // content sniff; "BM" is the magic signature for .bmp
 945                 {"picture2", "Cars are small this year\n", "text/plain; charset=utf-8"}, // content sniff
 946         }
 947         for _, trial := range trials {
 948                 f, err := fs.OpenFile(trial.filename, os.O_CREATE|os.O_WRONLY, 0777)
 949                 c.Assert(err, check.IsNil)
 950                 _, err = f.Write([]byte(trial.content))
 951                 c.Assert(err, check.IsNil)
 952                 c.Assert(f.Close(), check.IsNil)
 953         }
 954         mtxt, err := fs.MarshalManifest(".")
 955         c.Assert(err, check.IsNil)
 956         var coll arvados.Collection
 957         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 958                 "collection": map[string]string{
 959                         "manifest_text": mtxt,
 960                 },
 961         })
 962         c.Assert(err, check.IsNil)
 963
 964         for _, trial := range trials {
 965                 u, _ := url.Parse("http://download.example.com/by_id/" + coll.UUID + "/" + trial.filename)
 966                 req := &http.Request{
 967                         Method:     "GET",
 968                         Host:       u.Host,
 969                         URL:        u,
 970                         RequestURI: u.RequestURI(),
 971                         Header: http.Header{
 972                                 "Authorization": {"Bearer " + client.AuthToken},
 973                         },
 974                 }
 975                 resp := httptest.NewRecorder()
 976                 s.testServer.Handler.ServeHTTP(resp, req)
 977                 c.Check(resp.Code, check.Equals, http.StatusOK)
 978                 c.Check(resp.Header().Get("Content-Type"), check.Equals, trial.contentType)
 979                 c.Check(resp.Body.String(), check.Equals, trial.content)
 980         }
 981 }
 982
 983 func (s *IntegrationSuite) TestKeepClientBlockCache(c *check.C) {
 984         s.testServer.Config.cluster.Collections.WebDAVCache.MaxBlockEntries = 42
 985         c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Not(check.Equals), 42)
 986         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/t=" + arvadostest.ActiveToken + "/foo")
 987         req := &http.Request{
 988                 Method:     "GET",
 989                 Host:       u.Host,
 990                 URL:        u,
 991                 RequestURI: u.RequestURI(),
 992         }
 993         resp := httptest.NewRecorder()
 994         s.testServer.Handler.ServeHTTP(resp, req)
 995         c.Check(resp.Code, check.Equals, http.StatusOK)
 996         c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Equals, 42)
 997 }
 998
 999 func copyHeader(h http.Header) http.Header {
1000         hc := http.Header{}
1001         for k, v := range h {
1002                 hc[k] = append([]string(nil), v...)
1003         }
1004         return hc
1005 }