services/keep-web/handler_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "fmt"
  10         "html"
  11         "io/ioutil"
  12         "net/http"
  13         "net/http/httptest"
  14         "net/url"
  15         "os"
  16         "path/filepath"
  17         "regexp"
  18         "strings"
  19
  20         "git.arvados.org/arvados.git/lib/config"
  21         "git.arvados.org/arvados.git/sdk/go/arvados"
  22         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  23         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  24         "git.arvados.org/arvados.git/sdk/go/auth"
  25         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  26         "git.arvados.org/arvados.git/sdk/go/keepclient"
  27         check "gopkg.in/check.v1"
  28 )
  29
  30 var _ = check.Suite(&UnitSuite{})
  31
  32 type UnitSuite struct {
  33         Config *arvados.Config
  34 }
  35
  36 func (s *UnitSuite) SetUpTest(c *check.C) {
  37         ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), ctxlog.TestLogger(c))
  38         ldr.Path = "-"
  39         cfg, err := ldr.Load()
  40         c.Assert(err, check.IsNil)
  41         s.Config = cfg
  42 }
  43
  44 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
  45         h := handler{Config: newConfig(s.Config)}
  46         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
  47         req := &http.Request{
  48                 Method:     "OPTIONS",
  49                 Host:       u.Host,
  50                 URL:        u,
  51                 RequestURI: u.RequestURI(),
  52                 Header: http.Header{
  53                         "Origin":                        {"https://workbench.example"},
  54                         "Access-Control-Request-Method": {"POST"},
  55                 },
  56         }
  57
  58         // Check preflight for an allowed request
  59         resp := httptest.NewRecorder()
  60         h.ServeHTTP(resp, req)
  61         c.Check(resp.Code, check.Equals, http.StatusOK)
  62         c.Check(resp.Body.String(), check.Equals, "")
  63         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
  64         c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
  65         c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout")
  66
  67         // Check preflight for a disallowed request
  68         resp = httptest.NewRecorder()
  69         req.Header.Set("Access-Control-Request-Method", "MAKE-COFFEE")
  70         h.ServeHTTP(resp, req)
  71         c.Check(resp.Body.String(), check.Equals, "")
  72         c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
  73 }
  74
  75 func (s *UnitSuite) TestInvalidUUID(c *check.C) {
  76         bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-"
  77         token := arvadostest.ActiveToken
  78         for _, trial := range []string{
  79                 "http://keep-web/c=" + bogusID + "/foo",
  80                 "http://keep-web/c=" + bogusID + "/t=" + token + "/foo",
  81                 "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo",
  82                 "http://keep-web/collections/" + bogusID + "/foo",
  83                 "http://" + bogusID + ".keep-web/" + bogusID + "/foo",
  84                 "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
  85         } {
  86                 c.Log(trial)
  87                 u := mustParseURL(trial)
  88                 req := &http.Request{
  89                         Method:     "GET",
  90                         Host:       u.Host,
  91                         URL:        u,
  92                         RequestURI: u.RequestURI(),
  93                 }
  94                 resp := httptest.NewRecorder()
  95                 cfg := newConfig(s.Config)
  96                 cfg.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
  97                 h := handler{Config: cfg}
  98                 h.ServeHTTP(resp, req)
  99                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 100         }
 101 }
 102
 103 func mustParseURL(s string) *url.URL {
 104         r, err := url.Parse(s)
 105         if err != nil {
 106                 panic("parse URL: " + s)
 107         }
 108         return r
 109 }
 110
 111 func (s *IntegrationSuite) TestVhost404(c *check.C) {
 112         for _, testURL := range []string{
 113                 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
 114                 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
 115         } {
 116                 resp := httptest.NewRecorder()
 117                 u := mustParseURL(testURL)
 118                 req := &http.Request{
 119                         Method:     "GET",
 120                         URL:        u,
 121                         RequestURI: u.RequestURI(),
 122                 }
 123                 s.testServer.Handler.ServeHTTP(resp, req)
 124                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 125                 c.Check(resp.Body.String(), check.Equals, notFoundMessage+"\n")
 126         }
 127 }
 128
 129 // An authorizer modifies an HTTP request to make use of the given
 130 // token -- by adding it to a header, cookie, query param, or whatever
 131 // -- and returns the HTTP status code we should expect from keep-web if
 132 // the token is invalid.
 133 type authorizer func(*http.Request, string) int
 134
 135 func (s *IntegrationSuite) TestVhostViaAuthzHeader(c *check.C) {
 136         s.doVhostRequests(c, authzViaAuthzHeader)
 137 }
 138 func authzViaAuthzHeader(r *http.Request, tok string) int {
 139         r.Header.Add("Authorization", "OAuth2 "+tok)
 140         return http.StatusUnauthorized
 141 }
 142
 143 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
 144         s.doVhostRequests(c, authzViaCookieValue)
 145 }
 146 func authzViaCookieValue(r *http.Request, tok string) int {
 147         r.AddCookie(&http.Cookie{
 148                 Name:  "arvados_api_token",
 149                 Value: auth.EncodeTokenCookie([]byte(tok)),
 150         })
 151         return http.StatusUnauthorized
 152 }
 153
 154 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
 155         s.doVhostRequests(c, authzViaPath)
 156 }
 157 func authzViaPath(r *http.Request, tok string) int {
 158         r.URL.Path = "/t=" + tok + r.URL.Path
 159         return http.StatusNotFound
 160 }
 161
 162 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
 163         s.doVhostRequests(c, authzViaQueryString)
 164 }
 165 func authzViaQueryString(r *http.Request, tok string) int {
 166         r.URL.RawQuery = "api_token=" + tok
 167         return http.StatusUnauthorized
 168 }
 169
 170 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
 171         s.doVhostRequests(c, authzViaPOST)
 172 }
 173 func authzViaPOST(r *http.Request, tok string) int {
 174         r.Method = "POST"
 175         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 176         r.Body = ioutil.NopCloser(strings.NewReader(
 177                 url.Values{"api_token": {tok}}.Encode()))
 178         return http.StatusUnauthorized
 179 }
 180
 181 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
 182         s.doVhostRequests(c, authzViaPOST)
 183 }
 184 func authzViaXHRPOST(r *http.Request, tok string) int {
 185         r.Method = "POST"
 186         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 187         r.Header.Add("Origin", "https://origin.example")
 188         r.Body = ioutil.NopCloser(strings.NewReader(
 189                 url.Values{
 190                         "api_token":   {tok},
 191                         "disposition": {"attachment"},
 192                 }.Encode()))
 193         return http.StatusUnauthorized
 194 }
 195
 196 // Try some combinations of {url, token} using the given authorization
 197 // mechanism, and verify the result is correct.
 198 func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
 199         for _, hostPath := range []string{
 200                 arvadostest.FooCollection + ".example.com/foo",
 201                 arvadostest.FooCollection + "--collections.example.com/foo",
 202                 arvadostest.FooCollection + "--collections.example.com/_/foo",
 203                 arvadostest.FooCollectionPDH + ".example.com/foo",
 204                 strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + "--collections.example.com/foo",
 205                 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
 206         } {
 207                 c.Log("doRequests: ", hostPath)
 208                 s.doVhostRequestsWithHostPath(c, authz, hostPath)
 209         }
 210 }
 211
 212 func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
 213         for _, tok := range []string{
 214                 arvadostest.ActiveToken,
 215                 arvadostest.ActiveToken[:15],
 216                 arvadostest.SpectatorToken,
 217                 "bogus",
 218                 "",
 219         } {
 220                 u := mustParseURL("http://" + hostPath)
 221                 req := &http.Request{
 222                         Method:     "GET",
 223                         Host:       u.Host,
 224                         URL:        u,
 225                         RequestURI: u.RequestURI(),
 226                         Header:     http.Header{},
 227                 }
 228                 failCode := authz(req, tok)
 229                 req, resp := s.doReq(req)
 230                 code, body := resp.Code, resp.Body.String()
 231
 232                 // If the initial request had a (non-empty) token
 233                 // showing in the query string, we should have been
 234                 // redirected in order to hide it in a cookie.
 235                 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
 236
 237                 if tok == arvadostest.ActiveToken {
 238                         c.Check(code, check.Equals, http.StatusOK)
 239                         c.Check(body, check.Equals, "foo")
 240
 241                 } else {
 242                         c.Check(code >= 400, check.Equals, true)
 243                         c.Check(code < 500, check.Equals, true)
 244                         if tok == arvadostest.SpectatorToken {
 245                                 // Valid token never offers to retry
 246                                 // with different credentials.
 247                                 c.Check(code, check.Equals, http.StatusNotFound)
 248                         } else {
 249                                 // Invalid token can ask to retry
 250                                 // depending on the authz method.
 251                                 c.Check(code, check.Equals, failCode)
 252                         }
 253                         if code == 404 {
 254                                 c.Check(body, check.Equals, notFoundMessage+"\n")
 255                         } else {
 256                                 c.Check(body, check.Equals, unauthorizedMessage+"\n")
 257                         }
 258                 }
 259         }
 260 }
 261
 262 func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
 263         resp := httptest.NewRecorder()
 264         s.testServer.Handler.ServeHTTP(resp, req)
 265         if resp.Code != http.StatusSeeOther {
 266                 return req, resp
 267         }
 268         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 269         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 270         req = &http.Request{
 271                 Method:     "GET",
 272                 Host:       u.Host,
 273                 URL:        u,
 274                 RequestURI: u.RequestURI(),
 275                 Header:     http.Header{},
 276         }
 277         for _, c := range cookies {
 278                 req.AddCookie(c)
 279         }
 280         return s.doReq(req)
 281 }
 282
 283 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
 284         s.testVhostRedirectTokenToCookie(c, "GET",
 285                 arvadostest.FooCollection+".example.com/foo",
 286                 "?api_token="+arvadostest.ActiveToken,
 287                 "",
 288                 "",
 289                 http.StatusOK,
 290                 "foo",
 291         )
 292 }
 293
 294 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
 295         s.testVhostRedirectTokenToCookie(c, "GET",
 296                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
 297                 "",
 298                 "",
 299                 "",
 300                 http.StatusOK,
 301                 "foo",
 302         )
 303 }
 304
 305 // Bad token in URL is 404 Not Found because it doesn't make sense to
 306 // retry the same URL with different authorization.
 307 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
 308         s.testVhostRedirectTokenToCookie(c, "GET",
 309                 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
 310                 "",
 311                 "",
 312                 "",
 313                 http.StatusNotFound,
 314                 notFoundMessage+"\n",
 315         )
 316 }
 317
 318 // Bad token in a cookie (even if it got there via our own
 319 // query-string-to-cookie redirect) is, in principle, retryable at the
 320 // same URL so it's 401 Unauthorized.
 321 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
 322         s.testVhostRedirectTokenToCookie(c, "GET",
 323                 arvadostest.FooCollection+".example.com/foo",
 324                 "?api_token=thisisabogustoken",
 325                 "",
 326                 "",
 327                 http.StatusUnauthorized,
 328                 unauthorizedMessage+"\n",
 329         )
 330 }
 331
 332 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
 333         s.testVhostRedirectTokenToCookie(c, "GET",
 334                 "example.com/c="+arvadostest.FooCollection+"/foo",
 335                 "?api_token="+arvadostest.ActiveToken,
 336                 "",
 337                 "",
 338                 http.StatusBadRequest,
 339                 "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n",
 340         )
 341 }
 342
 343 // If client requests an attachment by putting ?disposition=attachment
 344 // in the query string, and gets redirected, the redirect target
 345 // should respond with an attachment.
 346 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
 347         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 348                 arvadostest.FooCollection+".example.com/foo",
 349                 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
 350                 "",
 351                 "",
 352                 http.StatusOK,
 353                 "foo",
 354         )
 355         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 356 }
 357
 358 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
 359         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 360         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 361                 "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
 362                 "?api_token="+arvadostest.ActiveToken,
 363                 "",
 364                 "",
 365                 http.StatusOK,
 366                 "foo",
 367         )
 368         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 369 }
 370
 371 func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
 372         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 373         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 374                 "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
 375                 "?api_token="+arvadostest.ActiveToken,
 376                 "",
 377                 "",
 378                 http.StatusOK,
 379                 "waz",
 380         )
 381         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 382         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 383                 "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz",
 384                 "?api_token="+arvadostest.ActiveToken,
 385                 "",
 386                 "",
 387                 http.StatusOK,
 388                 "waz",
 389         )
 390         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 391 }
 392
 393 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
 394         s.testServer.Config.cluster.Collections.TrustAllContent = true
 395         s.testVhostRedirectTokenToCookie(c, "GET",
 396                 "example.com/c="+arvadostest.FooCollection+"/foo",
 397                 "?api_token="+arvadostest.ActiveToken,
 398                 "",
 399                 "",
 400                 http.StatusOK,
 401                 "foo",
 402         )
 403 }
 404
 405 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
 406         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
 407
 408         s.testVhostRedirectTokenToCookie(c, "GET",
 409                 "example.com/c="+arvadostest.FooCollection+"/foo",
 410                 "?api_token="+arvadostest.ActiveToken,
 411                 "",
 412                 "",
 413                 http.StatusBadRequest,
 414                 "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n",
 415         )
 416
 417         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 418                 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
 419                 "?api_token="+arvadostest.ActiveToken,
 420                 "",
 421                 "",
 422                 http.StatusOK,
 423                 "foo",
 424         )
 425         c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
 426 }
 427
 428 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
 429         s.testVhostRedirectTokenToCookie(c, "POST",
 430                 arvadostest.FooCollection+".example.com/foo",
 431                 "",
 432                 "application/x-www-form-urlencoded",
 433                 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
 434                 http.StatusOK,
 435                 "foo",
 436         )
 437 }
 438
 439 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
 440         s.testVhostRedirectTokenToCookie(c, "POST",
 441                 arvadostest.FooCollection+".example.com/foo",
 442                 "",
 443                 "application/x-www-form-urlencoded",
 444                 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
 445                 http.StatusNotFound,
 446                 notFoundMessage+"\n",
 447         )
 448 }
 449
 450 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
 451         s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 452         s.testVhostRedirectTokenToCookie(c, "GET",
 453                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 454                 "",
 455                 "",
 456                 "",
 457                 http.StatusOK,
 458                 "Hello world\n",
 459         )
 460 }
 461
 462 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
 463         s.testServer.Config.cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
 464         s.testVhostRedirectTokenToCookie(c, "GET",
 465                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 466                 "",
 467                 "",
 468                 "",
 469                 http.StatusNotFound,
 470                 notFoundMessage+"\n",
 471         )
 472 }
 473
 474 func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
 475         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 476
 477         client := s.testServer.Config.Client
 478         client.AuthToken = arvadostest.ActiveToken
 479         fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
 480         c.Assert(err, check.IsNil)
 481         f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777)
 482         c.Assert(err, check.IsNil)
 483         f.Close()
 484         mtxt, err := fs.MarshalManifest(".")
 485         c.Assert(err, check.IsNil)
 486         var coll arvados.Collection
 487         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 488                 "collection": map[string]string{
 489                         "manifest_text": mtxt,
 490                 },
 491         })
 492         c.Assert(err, check.IsNil)
 493
 494         u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
 495         req := &http.Request{
 496                 Method:     "GET",
 497                 Host:       u.Host,
 498                 URL:        u,
 499                 RequestURI: u.RequestURI(),
 500                 Header: http.Header{
 501                         "Authorization": {"Bearer " + client.AuthToken},
 502                 },
 503         }
 504         resp := httptest.NewRecorder()
 505         s.testServer.Handler.ServeHTTP(resp, req)
 506         c.Check(resp.Code, check.Equals, http.StatusOK)
 507         c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\&#34;odd&#39; path chars.*`)
 508 }
 509
 510 func (s *IntegrationSuite) TestForwardSlashSubstitution(c *check.C) {
 511         arv := arvados.NewClientFromEnv()
 512         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 513         s.testServer.Config.cluster.Collections.ForwardSlashNameSubstitution = "{SOLIDUS}"
 514         name := "foo/bar/baz"
 515         nameShown := strings.Replace(name, "/", "{SOLIDUS}", -1)
 516         nameShownEscaped := strings.Replace(name, "/", "%7bSOLIDUS%7d", -1)
 517
 518         client := s.testServer.Config.Client
 519         client.AuthToken = arvadostest.ActiveToken
 520         fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
 521         c.Assert(err, check.IsNil)
 522         f, err := fs.OpenFile("filename", os.O_CREATE, 0777)
 523         c.Assert(err, check.IsNil)
 524         f.Close()
 525         mtxt, err := fs.MarshalManifest(".")
 526         c.Assert(err, check.IsNil)
 527         var coll arvados.Collection
 528         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 529                 "collection": map[string]string{
 530                         "manifest_text": mtxt,
 531                         "name":          name,
 532                         "owner_uuid":    arvadostest.AProjectUUID,
 533                 },
 534         })
 535         c.Assert(err, check.IsNil)
 536         defer arv.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
 537
 538         base := "http://download.example.com/by_id/" + coll.OwnerUUID + "/"
 539         for tryURL, expectRegexp := range map[string]string{
 540                 base:                          `(?ms).*href="./` + nameShownEscaped + `/"\S+` + nameShown + `.*`,
 541                 base + nameShownEscaped + "/": `(?ms).*href="./filename"\S+filename.*`,
 542         } {
 543                 u, _ := url.Parse(tryURL)
 544                 req := &http.Request{
 545                         Method:     "GET",
 546                         Host:       u.Host,
 547                         URL:        u,
 548                         RequestURI: u.RequestURI(),
 549                         Header: http.Header{
 550                                 "Authorization": {"Bearer " + client.AuthToken},
 551                         },
 552                 }
 553                 resp := httptest.NewRecorder()
 554                 s.testServer.Handler.ServeHTTP(resp, req)
 555                 c.Check(resp.Code, check.Equals, http.StatusOK)
 556                 c.Check(resp.Body.String(), check.Matches, expectRegexp)
 557         }
 558 }
 559
 560 // XHRs can't follow redirect-with-cookie so they rely on method=POST
 561 // and disposition=attachment (telling us it's acceptable to respond
 562 // with content instead of a redirect) and an Origin header that gets
 563 // added automatically by the browser (telling us it's desirable to do
 564 // so).
 565 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
 566         u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
 567         req := &http.Request{
 568                 Method:     "POST",
 569                 Host:       u.Host,
 570                 URL:        u,
 571                 RequestURI: u.RequestURI(),
 572                 Header: http.Header{
 573                         "Origin":       {"https://origin.example"},
 574                         "Content-Type": {"application/x-www-form-urlencoded"},
 575                 },
 576                 Body: ioutil.NopCloser(strings.NewReader(url.Values{
 577                         "api_token":   {arvadostest.ActiveToken},
 578                         "disposition": {"attachment"},
 579                 }.Encode())),
 580         }
 581         resp := httptest.NewRecorder()
 582         s.testServer.Handler.ServeHTTP(resp, req)
 583         c.Check(resp.Code, check.Equals, http.StatusOK)
 584         c.Check(resp.Body.String(), check.Equals, "foo")
 585         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
 586
 587         // GET + Origin header is representative of both AJAX GET
 588         // requests and inline images via <IMG crossorigin="anonymous"
 589         // src="...">.
 590         u.RawQuery = "api_token=" + url.QueryEscape(arvadostest.ActiveTokenV2)
 591         req = &http.Request{
 592                 Method:     "GET",
 593                 Host:       u.Host,
 594                 URL:        u,
 595                 RequestURI: u.RequestURI(),
 596                 Header: http.Header{
 597                         "Origin": {"https://origin.example"},
 598                 },
 599         }
 600         resp = httptest.NewRecorder()
 601         s.testServer.Handler.ServeHTTP(resp, req)
 602         c.Check(resp.Code, check.Equals, http.StatusOK)
 603         c.Check(resp.Body.String(), check.Equals, "foo")
 604         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
 605 }
 606
 607 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder {
 608         u, _ := url.Parse(`http://` + hostPath + queryString)
 609         req := &http.Request{
 610                 Method:     method,
 611                 Host:       u.Host,
 612                 URL:        u,
 613                 RequestURI: u.RequestURI(),
 614                 Header:     http.Header{"Content-Type": {contentType}},
 615                 Body:       ioutil.NopCloser(strings.NewReader(reqBody)),
 616         }
 617
 618         resp := httptest.NewRecorder()
 619         defer func() {
 620                 c.Check(resp.Code, check.Equals, expectStatus)
 621                 c.Check(resp.Body.String(), check.Equals, expectRespBody)
 622         }()
 623
 624         s.testServer.Handler.ServeHTTP(resp, req)
 625         if resp.Code != http.StatusSeeOther {
 626                 return resp
 627         }
 628         c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
 629         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 630
 631         u, _ = u.Parse(resp.Header().Get("Location"))
 632         req = &http.Request{
 633                 Method:     "GET",
 634                 Host:       u.Host,
 635                 URL:        u,
 636                 RequestURI: u.RequestURI(),
 637                 Header:     http.Header{},
 638         }
 639         for _, c := range cookies {
 640                 req.AddCookie(c)
 641         }
 642
 643         resp = httptest.NewRecorder()
 644         s.testServer.Handler.ServeHTTP(resp, req)
 645         c.Check(resp.Header().Get("Location"), check.Equals, "")
 646         return resp
 647 }
 648
 649 func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) {
 650         s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 651         s.testDirectoryListing(c)
 652 }
 653
 654 func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) {
 655         s.testServer.Config.cluster.Users.AnonymousUserToken = ""
 656         s.testDirectoryListing(c)
 657 }
 658
 659 func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
 660         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 661         authHeader := http.Header{
 662                 "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
 663         }
 664         for _, trial := range []struct {
 665                 uri      string
 666                 header   http.Header
 667                 expect   []string
 668                 redirect string
 669                 cutDirs  int
 670         }{
 671                 {
 672                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
 673                         header:  authHeader,
 674                         expect:  []string{"dir1/foo", "dir1/bar"},
 675                         cutDirs: 0,
 676                 },
 677                 {
 678                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
 679                         header:  authHeader,
 680                         expect:  []string{"foo", "bar"},
 681                         cutDirs: 1,
 682                 },
 683                 {
 684                         // URLs of this form ignore authHeader, and
 685                         // FooAndBarFilesInDirUUID isn't public, so
 686                         // this returns 404.
 687                         uri:    "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
 688                         header: authHeader,
 689                         expect: nil,
 690                 },
 691                 {
 692                         uri:     "download.example.com/users/active/foo_file_in_dir/",
 693                         header:  authHeader,
 694                         expect:  []string{"dir1/"},
 695                         cutDirs: 3,
 696                 },
 697                 {
 698                         uri:     "download.example.com/users/active/foo_file_in_dir/dir1/",
 699                         header:  authHeader,
 700                         expect:  []string{"bar"},
 701                         cutDirs: 4,
 702                 },
 703                 {
 704                         uri:     "download.example.com/",
 705                         header:  authHeader,
 706                         expect:  []string{"users/"},
 707                         cutDirs: 0,
 708                 },
 709                 {
 710                         uri:      "download.example.com/users",
 711                         header:   authHeader,
 712                         redirect: "/users/",
 713                         expect:   []string{"active/"},
 714                         cutDirs:  1,
 715                 },
 716                 {
 717                         uri:     "download.example.com/users/",
 718                         header:  authHeader,
 719                         expect:  []string{"active/"},
 720                         cutDirs: 1,
 721                 },
 722                 {
 723                         uri:      "download.example.com/users/active",
 724                         header:   authHeader,
 725                         redirect: "/users/active/",
 726                         expect:   []string{"foo_file_in_dir/"},
 727                         cutDirs:  2,
 728                 },
 729                 {
 730                         uri:     "download.example.com/users/active/",
 731                         header:  authHeader,
 732                         expect:  []string{"foo_file_in_dir/"},
 733                         cutDirs: 2,
 734                 },
 735                 {
 736                         uri:     "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
 737                         header:  nil,
 738                         expect:  []string{"dir1/foo", "dir1/bar"},
 739                         cutDirs: 4,
 740                 },
 741                 {
 742                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
 743                         header:  nil,
 744                         expect:  []string{"dir1/foo", "dir1/bar"},
 745                         cutDirs: 2,
 746                 },
 747                 {
 748                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken,
 749                         header:  nil,
 750                         expect:  []string{"dir1/foo", "dir1/bar"},
 751                         cutDirs: 2,
 752                 },
 753                 {
 754                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID,
 755                         header:  authHeader,
 756                         expect:  []string{"dir1/foo", "dir1/bar"},
 757                         cutDirs: 1,
 758                 },
 759                 {
 760                         uri:      "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
 761                         header:   authHeader,
 762                         redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
 763                         expect:   []string{"foo", "bar"},
 764                         cutDirs:  2,
 765                 },
 766                 {
 767                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
 768                         header:  authHeader,
 769                         expect:  []string{"foo", "bar"},
 770                         cutDirs: 3,
 771                 },
 772                 {
 773                         uri:      arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
 774                         header:   authHeader,
 775                         redirect: "/dir1/",
 776                         expect:   []string{"foo", "bar"},
 777                         cutDirs:  1,
 778                 },
 779                 {
 780                         uri:    "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
 781                         header: authHeader,
 782                         expect: nil,
 783                 },
 784                 {
 785                         uri:     "download.example.com/c=" + arvadostest.WazVersion1Collection,
 786                         header:  authHeader,
 787                         expect:  []string{"waz"},
 788                         cutDirs: 1,
 789                 },
 790                 {
 791                         uri:     "download.example.com/by_id/" + arvadostest.WazVersion1Collection,
 792                         header:  authHeader,
 793                         expect:  []string{"waz"},
 794                         cutDirs: 2,
 795                 },
 796         } {
 797                 comment := check.Commentf("HTML: %q => %q", trial.uri, trial.expect)
 798                 resp := httptest.NewRecorder()
 799                 u := mustParseURL("//" + trial.uri)
 800                 req := &http.Request{
 801                         Method:     "GET",
 802                         Host:       u.Host,
 803                         URL:        u,
 804                         RequestURI: u.RequestURI(),
 805                         Header:     copyHeader(trial.header),
 806                 }
 807                 s.testServer.Handler.ServeHTTP(resp, req)
 808                 var cookies []*http.Cookie
 809                 for resp.Code == http.StatusSeeOther {
 810                         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 811                         req = &http.Request{
 812                                 Method:     "GET",
 813                                 Host:       u.Host,
 814                                 URL:        u,
 815                                 RequestURI: u.RequestURI(),
 816                                 Header:     copyHeader(trial.header),
 817                         }
 818                         cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
 819                         for _, c := range cookies {
 820                                 req.AddCookie(c)
 821                         }
 822                         resp = httptest.NewRecorder()
 823                         s.testServer.Handler.ServeHTTP(resp, req)
 824                 }
 825                 if trial.redirect != "" {
 826                         c.Check(req.URL.Path, check.Equals, trial.redirect, comment)
 827                 }
 828                 if trial.expect == nil {
 829                         c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
 830                 } else {
 831                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
 832                         for _, e := range trial.expect {
 833                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`, comment)
 834                         }
 835                         c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`, comment)
 836                 }
 837
 838                 comment = check.Commentf("WebDAV: %q => %q", trial.uri, trial.expect)
 839                 req = &http.Request{
 840                         Method:     "OPTIONS",
 841                         Host:       u.Host,
 842                         URL:        u,
 843                         RequestURI: u.RequestURI(),
 844                         Header:     copyHeader(trial.header),
 845                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
 846                 }
 847                 resp = httptest.NewRecorder()
 848                 s.testServer.Handler.ServeHTTP(resp, req)
 849                 if trial.expect == nil {
 850                         c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
 851                 } else {
 852                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
 853                 }
 854
 855                 req = &http.Request{
 856                         Method:     "PROPFIND",
 857                         Host:       u.Host,
 858                         URL:        u,
 859                         RequestURI: u.RequestURI(),
 860                         Header:     copyHeader(trial.header),
 861                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
 862                 }
 863                 resp = httptest.NewRecorder()
 864                 s.testServer.Handler.ServeHTTP(resp, req)
 865                 if trial.expect == nil {
 866                         c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
 867                 } else {
 868                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment)
 869                         for _, e := range trial.expect {
 870                                 if strings.HasSuffix(e, "/") {
 871                                         e = filepath.Join(u.Path, e) + "/"
 872                                 } else {
 873                                         e = filepath.Join(u.Path, e)
 874                                 }
 875                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+e+`</D:href>.*`, comment)
 876                         }
 877                 }
 878         }
 879 }
 880
 881 func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
 882         arv := arvados.NewClientFromEnv()
 883         var newCollection arvados.Collection
 884         err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 885                 "collection": map[string]string{
 886                         "owner_uuid":    arvadostest.ActiveUserUUID,
 887                         "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
 888                         "name":          "keep-web test collection",
 889                 },
 890                 "ensure_unique_name": true,
 891         })
 892         c.Assert(err, check.IsNil)
 893         defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
 894
 895         var updated arvados.Collection
 896         for _, fnm := range []string{"foo.txt", "bar.txt"} {
 897                 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
 898                 u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
 899                 req := &http.Request{
 900                         Method:     "DELETE",
 901                         Host:       u.Host,
 902                         URL:        u,
 903                         RequestURI: u.RequestURI(),
 904                         Header: http.Header{
 905                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
 906                         },
 907                 }
 908                 resp := httptest.NewRecorder()
 909                 s.testServer.Handler.ServeHTTP(resp, req)
 910                 c.Check(resp.Code, check.Equals, http.StatusNoContent)
 911
 912                 updated = arvados.Collection{}
 913                 err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
 914                 c.Check(err, check.IsNil)
 915                 c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`)
 916                 c.Logf("updated manifest_text %q", updated.ManifestText)
 917         }
 918         c.Check(updated.ManifestText, check.Equals, "")
 919 }
 920
 921 func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) {
 922         s.testServer.Config.cluster.ManagementToken = arvadostest.ManagementToken
 923         authHeader := http.Header{
 924                 "Authorization": {"Bearer " + arvadostest.ManagementToken},
 925         }
 926
 927         resp := httptest.NewRecorder()
 928         u := mustParseURL("http://download.example.com/_health/ping")
 929         req := &http.Request{
 930                 Method:     "GET",
 931                 Host:       u.Host,
 932                 URL:        u,
 933                 RequestURI: u.RequestURI(),
 934                 Header:     authHeader,
 935         }
 936         s.testServer.Handler.ServeHTTP(resp, req)
 937
 938         c.Check(resp.Code, check.Equals, http.StatusOK)
 939         c.Check(resp.Body.String(), check.Matches, `{"health":"OK"}\n`)
 940 }
 941
 942 func (s *IntegrationSuite) TestFileContentType(c *check.C) {
 943         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 944
 945         client := s.testServer.Config.Client
 946         client.AuthToken = arvadostest.ActiveToken
 947         arv, err := arvadosclient.New(&client)
 948         c.Assert(err, check.Equals, nil)
 949         kc, err := keepclient.MakeKeepClient(arv)
 950         c.Assert(err, check.Equals, nil)
 951
 952         fs, err := (&arvados.Collection{}).FileSystem(&client, kc)
 953         c.Assert(err, check.IsNil)
 954
 955         trials := []struct {
 956                 filename    string
 957                 content     string
 958                 contentType string
 959         }{
 960                 {"picture.txt", "BMX bikes are small this year\n", "text/plain; charset=utf-8"},
 961                 {"picture.bmp", "BMX bikes are small this year\n", "image/x-ms-bmp"},
 962                 {"picture.jpg", "BMX bikes are small this year\n", "image/jpeg"},
 963                 {"picture1", "BMX bikes are small this year\n", "image/bmp"},            // content sniff; "BM" is the magic signature for .bmp
 964                 {"picture2", "Cars are small this year\n", "text/plain; charset=utf-8"}, // content sniff
 965         }
 966         for _, trial := range trials {
 967                 f, err := fs.OpenFile(trial.filename, os.O_CREATE|os.O_WRONLY, 0777)
 968                 c.Assert(err, check.IsNil)
 969                 _, err = f.Write([]byte(trial.content))
 970                 c.Assert(err, check.IsNil)
 971                 c.Assert(f.Close(), check.IsNil)
 972         }
 973         mtxt, err := fs.MarshalManifest(".")
 974         c.Assert(err, check.IsNil)
 975         var coll arvados.Collection
 976         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 977                 "collection": map[string]string{
 978                         "manifest_text": mtxt,
 979                 },
 980         })
 981         c.Assert(err, check.IsNil)
 982
 983         for _, trial := range trials {
 984                 u, _ := url.Parse("http://download.example.com/by_id/" + coll.UUID + "/" + trial.filename)
 985                 req := &http.Request{
 986                         Method:     "GET",
 987                         Host:       u.Host,
 988                         URL:        u,
 989                         RequestURI: u.RequestURI(),
 990                         Header: http.Header{
 991                                 "Authorization": {"Bearer " + client.AuthToken},
 992                         },
 993                 }
 994                 resp := httptest.NewRecorder()
 995                 s.testServer.Handler.ServeHTTP(resp, req)
 996                 c.Check(resp.Code, check.Equals, http.StatusOK)
 997                 c.Check(resp.Header().Get("Content-Type"), check.Equals, trial.contentType)
 998                 c.Check(resp.Body.String(), check.Equals, trial.content)
 999         }
1000 }
1001
1002 func (s *IntegrationSuite) TestKeepClientBlockCache(c *check.C) {
1003         s.testServer.Config.cluster.Collections.WebDAVCache.MaxBlockEntries = 42
1004         c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Not(check.Equals), 42)
1005         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/t=" + arvadostest.ActiveToken + "/foo")
1006         req := &http.Request{
1007                 Method:     "GET",
1008                 Host:       u.Host,
1009                 URL:        u,
1010                 RequestURI: u.RequestURI(),
1011         }
1012         resp := httptest.NewRecorder()
1013         s.testServer.Handler.ServeHTTP(resp, req)
1014         c.Check(resp.Code, check.Equals, http.StatusOK)
1015         c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Equals, 42)
1016 }
1017
1018 func copyHeader(h http.Header) http.Header {
1019         hc := http.Header{}
1020         for k, v := range h {
1021                 hc[k] = append([]string(nil), v...)
1022         }
1023         return hc
1024 }