services/keep-web/handler_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "fmt"
  10         "html"
  11         "io/ioutil"
  12         "net/http"
  13         "net/http/httptest"
  14         "net/url"
  15         "os"
  16         "path/filepath"
  17         "regexp"
  18         "strings"
  19
  20         "git.arvados.org/arvados.git/lib/config"
  21         "git.arvados.org/arvados.git/sdk/go/arvados"
  22         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  23         "git.arvados.org/arvados.git/sdk/go/auth"
  24         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  25         "git.arvados.org/arvados.git/sdk/go/keepclient"
  26         check "gopkg.in/check.v1"
  27 )
  28
  29 var _ = check.Suite(&UnitSuite{})
  30
  31 type UnitSuite struct {
  32         Config *arvados.Config
  33 }
  34
  35 func (s *UnitSuite) SetUpTest(c *check.C) {
  36         ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), ctxlog.TestLogger(c))
  37         ldr.Path = "-"
  38         cfg, err := ldr.Load()
  39         c.Assert(err, check.IsNil)
  40         s.Config = cfg
  41 }
  42
  43 func (s *UnitSuite) TestKeepClientBlockCache(c *check.C) {
  44         cfg := newConfig(s.Config)
  45         cfg.cluster.Collections.WebDAVCache.MaxBlockEntries = 42
  46         h := handler{Config: cfg}
  47         c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Not(check.Equals), cfg.cluster.Collections.WebDAVCache.MaxBlockEntries)
  48         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/t=" + arvadostest.ActiveToken + "/foo")
  49         req := &http.Request{
  50                 Method:     "GET",
  51                 Host:       u.Host,
  52                 URL:        u,
  53                 RequestURI: u.RequestURI(),
  54         }
  55         resp := httptest.NewRecorder()
  56         h.ServeHTTP(resp, req)
  57         c.Check(resp.Code, check.Equals, http.StatusOK)
  58         c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Equals, cfg.cluster.Collections.WebDAVCache.MaxBlockEntries)
  59 }
  60
  61 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
  62         h := handler{Config: newConfig(s.Config)}
  63         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
  64         req := &http.Request{
  65                 Method:     "OPTIONS",
  66                 Host:       u.Host,
  67                 URL:        u,
  68                 RequestURI: u.RequestURI(),
  69                 Header: http.Header{
  70                         "Origin":                        {"https://workbench.example"},
  71                         "Access-Control-Request-Method": {"POST"},
  72                 },
  73         }
  74
  75         // Check preflight for an allowed request
  76         resp := httptest.NewRecorder()
  77         h.ServeHTTP(resp, req)
  78         c.Check(resp.Code, check.Equals, http.StatusOK)
  79         c.Check(resp.Body.String(), check.Equals, "")
  80         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
  81         c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
  82         c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout")
  83
  84         // Check preflight for a disallowed request
  85         resp = httptest.NewRecorder()
  86         req.Header.Set("Access-Control-Request-Method", "MAKE-COFFEE")
  87         h.ServeHTTP(resp, req)
  88         c.Check(resp.Body.String(), check.Equals, "")
  89         c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
  90 }
  91
  92 func (s *UnitSuite) TestInvalidUUID(c *check.C) {
  93         bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-"
  94         token := arvadostest.ActiveToken
  95         for _, trial := range []string{
  96                 "http://keep-web/c=" + bogusID + "/foo",
  97                 "http://keep-web/c=" + bogusID + "/t=" + token + "/foo",
  98                 "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo",
  99                 "http://keep-web/collections/" + bogusID + "/foo",
 100                 "http://" + bogusID + ".keep-web/" + bogusID + "/foo",
 101                 "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
 102         } {
 103                 c.Log(trial)
 104                 u := mustParseURL(trial)
 105                 req := &http.Request{
 106                         Method:     "GET",
 107                         Host:       u.Host,
 108                         URL:        u,
 109                         RequestURI: u.RequestURI(),
 110                 }
 111                 resp := httptest.NewRecorder()
 112                 cfg := newConfig(s.Config)
 113                 cfg.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 114                 h := handler{Config: cfg}
 115                 h.ServeHTTP(resp, req)
 116                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 117         }
 118 }
 119
 120 func mustParseURL(s string) *url.URL {
 121         r, err := url.Parse(s)
 122         if err != nil {
 123                 panic("parse URL: " + s)
 124         }
 125         return r
 126 }
 127
 128 func (s *IntegrationSuite) TestVhost404(c *check.C) {
 129         for _, testURL := range []string{
 130                 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
 131                 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
 132         } {
 133                 resp := httptest.NewRecorder()
 134                 u := mustParseURL(testURL)
 135                 req := &http.Request{
 136                         Method:     "GET",
 137                         URL:        u,
 138                         RequestURI: u.RequestURI(),
 139                 }
 140                 s.testServer.Handler.ServeHTTP(resp, req)
 141                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 142                 c.Check(resp.Body.String(), check.Equals, "")
 143         }
 144 }
 145
 146 // An authorizer modifies an HTTP request to make use of the given
 147 // token -- by adding it to a header, cookie, query param, or whatever
 148 // -- and returns the HTTP status code we should expect from keep-web if
 149 // the token is invalid.
 150 type authorizer func(*http.Request, string) int
 151
 152 func (s *IntegrationSuite) TestVhostViaAuthzHeader(c *check.C) {
 153         s.doVhostRequests(c, authzViaAuthzHeader)
 154 }
 155 func authzViaAuthzHeader(r *http.Request, tok string) int {
 156         r.Header.Add("Authorization", "OAuth2 "+tok)
 157         return http.StatusUnauthorized
 158 }
 159
 160 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
 161         s.doVhostRequests(c, authzViaCookieValue)
 162 }
 163 func authzViaCookieValue(r *http.Request, tok string) int {
 164         r.AddCookie(&http.Cookie{
 165                 Name:  "arvados_api_token",
 166                 Value: auth.EncodeTokenCookie([]byte(tok)),
 167         })
 168         return http.StatusUnauthorized
 169 }
 170
 171 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
 172         s.doVhostRequests(c, authzViaPath)
 173 }
 174 func authzViaPath(r *http.Request, tok string) int {
 175         r.URL.Path = "/t=" + tok + r.URL.Path
 176         return http.StatusNotFound
 177 }
 178
 179 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
 180         s.doVhostRequests(c, authzViaQueryString)
 181 }
 182 func authzViaQueryString(r *http.Request, tok string) int {
 183         r.URL.RawQuery = "api_token=" + tok
 184         return http.StatusUnauthorized
 185 }
 186
 187 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
 188         s.doVhostRequests(c, authzViaPOST)
 189 }
 190 func authzViaPOST(r *http.Request, tok string) int {
 191         r.Method = "POST"
 192         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 193         r.Body = ioutil.NopCloser(strings.NewReader(
 194                 url.Values{"api_token": {tok}}.Encode()))
 195         return http.StatusUnauthorized
 196 }
 197
 198 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
 199         s.doVhostRequests(c, authzViaPOST)
 200 }
 201 func authzViaXHRPOST(r *http.Request, tok string) int {
 202         r.Method = "POST"
 203         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 204         r.Header.Add("Origin", "https://origin.example")
 205         r.Body = ioutil.NopCloser(strings.NewReader(
 206                 url.Values{
 207                         "api_token":   {tok},
 208                         "disposition": {"attachment"},
 209                 }.Encode()))
 210         return http.StatusUnauthorized
 211 }
 212
 213 // Try some combinations of {url, token} using the given authorization
 214 // mechanism, and verify the result is correct.
 215 func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
 216         for _, hostPath := range []string{
 217                 arvadostest.FooCollection + ".example.com/foo",
 218                 arvadostest.FooCollection + "--collections.example.com/foo",
 219                 arvadostest.FooCollection + "--collections.example.com/_/foo",
 220                 arvadostest.FooCollectionPDH + ".example.com/foo",
 221                 strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + "--collections.example.com/foo",
 222                 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
 223         } {
 224                 c.Log("doRequests: ", hostPath)
 225                 s.doVhostRequestsWithHostPath(c, authz, hostPath)
 226         }
 227 }
 228
 229 func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
 230         for _, tok := range []string{
 231                 arvadostest.ActiveToken,
 232                 arvadostest.ActiveToken[:15],
 233                 arvadostest.SpectatorToken,
 234                 "bogus",
 235                 "",
 236         } {
 237                 u := mustParseURL("http://" + hostPath)
 238                 req := &http.Request{
 239                         Method:     "GET",
 240                         Host:       u.Host,
 241                         URL:        u,
 242                         RequestURI: u.RequestURI(),
 243                         Header:     http.Header{},
 244                 }
 245                 failCode := authz(req, tok)
 246                 req, resp := s.doReq(req)
 247                 code, body := resp.Code, resp.Body.String()
 248
 249                 // If the initial request had a (non-empty) token
 250                 // showing in the query string, we should have been
 251                 // redirected in order to hide it in a cookie.
 252                 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
 253
 254                 if tok == arvadostest.ActiveToken {
 255                         c.Check(code, check.Equals, http.StatusOK)
 256                         c.Check(body, check.Equals, "foo")
 257
 258                 } else {
 259                         c.Check(code >= 400, check.Equals, true)
 260                         c.Check(code < 500, check.Equals, true)
 261                         if tok == arvadostest.SpectatorToken {
 262                                 // Valid token never offers to retry
 263                                 // with different credentials.
 264                                 c.Check(code, check.Equals, http.StatusNotFound)
 265                         } else {
 266                                 // Invalid token can ask to retry
 267                                 // depending on the authz method.
 268                                 c.Check(code, check.Equals, failCode)
 269                         }
 270                         c.Check(body, check.Equals, "")
 271                 }
 272         }
 273 }
 274
 275 func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
 276         resp := httptest.NewRecorder()
 277         s.testServer.Handler.ServeHTTP(resp, req)
 278         if resp.Code != http.StatusSeeOther {
 279                 return req, resp
 280         }
 281         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 282         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 283         req = &http.Request{
 284                 Method:     "GET",
 285                 Host:       u.Host,
 286                 URL:        u,
 287                 RequestURI: u.RequestURI(),
 288                 Header:     http.Header{},
 289         }
 290         for _, c := range cookies {
 291                 req.AddCookie(c)
 292         }
 293         return s.doReq(req)
 294 }
 295
 296 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
 297         s.testVhostRedirectTokenToCookie(c, "GET",
 298                 arvadostest.FooCollection+".example.com/foo",
 299                 "?api_token="+arvadostest.ActiveToken,
 300                 "",
 301                 "",
 302                 http.StatusOK,
 303                 "foo",
 304         )
 305 }
 306
 307 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
 308         s.testVhostRedirectTokenToCookie(c, "GET",
 309                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
 310                 "",
 311                 "",
 312                 "",
 313                 http.StatusOK,
 314                 "foo",
 315         )
 316 }
 317
 318 // Bad token in URL is 404 Not Found because it doesn't make sense to
 319 // retry the same URL with different authorization.
 320 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
 321         s.testVhostRedirectTokenToCookie(c, "GET",
 322                 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
 323                 "",
 324                 "",
 325                 "",
 326                 http.StatusNotFound,
 327                 "",
 328         )
 329 }
 330
 331 // Bad token in a cookie (even if it got there via our own
 332 // query-string-to-cookie redirect) is, in principle, retryable at the
 333 // same URL so it's 401 Unauthorized.
 334 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
 335         s.testVhostRedirectTokenToCookie(c, "GET",
 336                 arvadostest.FooCollection+".example.com/foo",
 337                 "?api_token=thisisabogustoken",
 338                 "",
 339                 "",
 340                 http.StatusUnauthorized,
 341                 "",
 342         )
 343 }
 344
 345 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
 346         s.testVhostRedirectTokenToCookie(c, "GET",
 347                 "example.com/c="+arvadostest.FooCollection+"/foo",
 348                 "?api_token="+arvadostest.ActiveToken,
 349                 "",
 350                 "",
 351                 http.StatusBadRequest,
 352                 "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n",
 353         )
 354 }
 355
 356 // If client requests an attachment by putting ?disposition=attachment
 357 // in the query string, and gets redirected, the redirect target
 358 // should respond with an attachment.
 359 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
 360         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 361                 arvadostest.FooCollection+".example.com/foo",
 362                 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
 363                 "",
 364                 "",
 365                 http.StatusOK,
 366                 "foo",
 367         )
 368         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 369 }
 370
 371 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
 372         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 373         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 374                 "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
 375                 "?api_token="+arvadostest.ActiveToken,
 376                 "",
 377                 "",
 378                 http.StatusOK,
 379                 "foo",
 380         )
 381         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 382 }
 383
 384 func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
 385         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 386         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 387                 "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
 388                 "?api_token="+arvadostest.ActiveToken,
 389                 "",
 390                 "",
 391                 http.StatusOK,
 392                 "waz",
 393         )
 394         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 395         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 396                 "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz",
 397                 "?api_token="+arvadostest.ActiveToken,
 398                 "",
 399                 "",
 400                 http.StatusOK,
 401                 "waz",
 402         )
 403         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 404 }
 405
 406 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
 407         s.testServer.Config.cluster.Collections.TrustAllContent = true
 408         s.testVhostRedirectTokenToCookie(c, "GET",
 409                 "example.com/c="+arvadostest.FooCollection+"/foo",
 410                 "?api_token="+arvadostest.ActiveToken,
 411                 "",
 412                 "",
 413                 http.StatusOK,
 414                 "foo",
 415         )
 416 }
 417
 418 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
 419         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
 420
 421         s.testVhostRedirectTokenToCookie(c, "GET",
 422                 "example.com/c="+arvadostest.FooCollection+"/foo",
 423                 "?api_token="+arvadostest.ActiveToken,
 424                 "",
 425                 "",
 426                 http.StatusBadRequest,
 427                 "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n",
 428         )
 429
 430         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 431                 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
 432                 "?api_token="+arvadostest.ActiveToken,
 433                 "",
 434                 "",
 435                 http.StatusOK,
 436                 "foo",
 437         )
 438         c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
 439 }
 440
 441 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
 442         s.testVhostRedirectTokenToCookie(c, "POST",
 443                 arvadostest.FooCollection+".example.com/foo",
 444                 "",
 445                 "application/x-www-form-urlencoded",
 446                 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
 447                 http.StatusOK,
 448                 "foo",
 449         )
 450 }
 451
 452 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
 453         s.testVhostRedirectTokenToCookie(c, "POST",
 454                 arvadostest.FooCollection+".example.com/foo",
 455                 "",
 456                 "application/x-www-form-urlencoded",
 457                 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
 458                 http.StatusNotFound,
 459                 "",
 460         )
 461 }
 462
 463 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
 464         s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 465         s.testVhostRedirectTokenToCookie(c, "GET",
 466                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 467                 "",
 468                 "",
 469                 "",
 470                 http.StatusOK,
 471                 "Hello world\n",
 472         )
 473 }
 474
 475 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
 476         s.testServer.Config.cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
 477         s.testVhostRedirectTokenToCookie(c, "GET",
 478                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 479                 "",
 480                 "",
 481                 "",
 482                 http.StatusNotFound,
 483                 "",
 484         )
 485 }
 486
 487 func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
 488         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 489
 490         client := s.testServer.Config.Client
 491         client.AuthToken = arvadostest.ActiveToken
 492         fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
 493         c.Assert(err, check.IsNil)
 494         f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777)
 495         c.Assert(err, check.IsNil)
 496         f.Close()
 497         mtxt, err := fs.MarshalManifest(".")
 498         c.Assert(err, check.IsNil)
 499         var coll arvados.Collection
 500         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 501                 "collection": map[string]string{
 502                         "manifest_text": mtxt,
 503                 },
 504         })
 505         c.Assert(err, check.IsNil)
 506
 507         u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
 508         req := &http.Request{
 509                 Method:     "GET",
 510                 Host:       u.Host,
 511                 URL:        u,
 512                 RequestURI: u.RequestURI(),
 513                 Header: http.Header{
 514                         "Authorization": {"Bearer " + client.AuthToken},
 515                 },
 516         }
 517         resp := httptest.NewRecorder()
 518         s.testServer.Handler.ServeHTTP(resp, req)
 519         c.Check(resp.Code, check.Equals, http.StatusOK)
 520         c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\&#34;odd&#39; path chars.*`)
 521 }
 522
 523 func (s *IntegrationSuite) TestForwardSlashSubstitution(c *check.C) {
 524         arv := arvados.NewClientFromEnv()
 525         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 526         s.testServer.Config.cluster.Collections.ForwardSlashNameSubstitution = "{SOLIDUS}"
 527         name := "foo/bar/baz"
 528         nameShown := strings.Replace(name, "/", "{SOLIDUS}", -1)
 529         nameShownEscaped := strings.Replace(name, "/", "%7bSOLIDUS%7d", -1)
 530
 531         client := s.testServer.Config.Client
 532         client.AuthToken = arvadostest.ActiveToken
 533         fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
 534         c.Assert(err, check.IsNil)
 535         f, err := fs.OpenFile("filename", os.O_CREATE, 0777)
 536         c.Assert(err, check.IsNil)
 537         f.Close()
 538         mtxt, err := fs.MarshalManifest(".")
 539         c.Assert(err, check.IsNil)
 540         var coll arvados.Collection
 541         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 542                 "collection": map[string]string{
 543                         "manifest_text": mtxt,
 544                         "name":          name,
 545                         "owner_uuid":    arvadostest.AProjectUUID,
 546                 },
 547         })
 548         c.Assert(err, check.IsNil)
 549         defer arv.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
 550
 551         base := "http://download.example.com/by_id/" + coll.OwnerUUID + "/"
 552         for tryURL, expectRegexp := range map[string]string{
 553                 base:                   `(?ms).*href="./` + nameShownEscaped + `/"\S+` + nameShown + `.*`,
 554                 base + nameShown + "/": `(?ms).*href="./filename"\S+filename.*`,
 555         } {
 556                 u, _ := url.Parse(tryURL)
 557                 req := &http.Request{
 558                         Method:     "GET",
 559                         Host:       u.Host,
 560                         URL:        u,
 561                         RequestURI: u.RequestURI(),
 562                         Header: http.Header{
 563                                 "Authorization": {"Bearer " + client.AuthToken},
 564                         },
 565                 }
 566                 resp := httptest.NewRecorder()
 567                 s.testServer.Handler.ServeHTTP(resp, req)
 568                 c.Check(resp.Code, check.Equals, http.StatusOK)
 569                 c.Check(resp.Body.String(), check.Matches, expectRegexp)
 570         }
 571 }
 572
 573 // XHRs can't follow redirect-with-cookie so they rely on method=POST
 574 // and disposition=attachment (telling us it's acceptable to respond
 575 // with content instead of a redirect) and an Origin header that gets
 576 // added automatically by the browser (telling us it's desirable to do
 577 // so).
 578 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
 579         u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
 580         req := &http.Request{
 581                 Method:     "POST",
 582                 Host:       u.Host,
 583                 URL:        u,
 584                 RequestURI: u.RequestURI(),
 585                 Header: http.Header{
 586                         "Origin":       {"https://origin.example"},
 587                         "Content-Type": {"application/x-www-form-urlencoded"},
 588                 },
 589                 Body: ioutil.NopCloser(strings.NewReader(url.Values{
 590                         "api_token":   {arvadostest.ActiveToken},
 591                         "disposition": {"attachment"},
 592                 }.Encode())),
 593         }
 594         resp := httptest.NewRecorder()
 595         s.testServer.Handler.ServeHTTP(resp, req)
 596         c.Check(resp.Code, check.Equals, http.StatusOK)
 597         c.Check(resp.Body.String(), check.Equals, "foo")
 598         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
 599 }
 600
 601 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder {
 602         u, _ := url.Parse(`http://` + hostPath + queryString)
 603         req := &http.Request{
 604                 Method:     method,
 605                 Host:       u.Host,
 606                 URL:        u,
 607                 RequestURI: u.RequestURI(),
 608                 Header:     http.Header{"Content-Type": {contentType}},
 609                 Body:       ioutil.NopCloser(strings.NewReader(reqBody)),
 610         }
 611
 612         resp := httptest.NewRecorder()
 613         defer func() {
 614                 c.Check(resp.Code, check.Equals, expectStatus)
 615                 c.Check(resp.Body.String(), check.Equals, expectRespBody)
 616         }()
 617
 618         s.testServer.Handler.ServeHTTP(resp, req)
 619         if resp.Code != http.StatusSeeOther {
 620                 return resp
 621         }
 622         c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
 623         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 624
 625         u, _ = u.Parse(resp.Header().Get("Location"))
 626         req = &http.Request{
 627                 Method:     "GET",
 628                 Host:       u.Host,
 629                 URL:        u,
 630                 RequestURI: u.RequestURI(),
 631                 Header:     http.Header{},
 632         }
 633         for _, c := range cookies {
 634                 req.AddCookie(c)
 635         }
 636
 637         resp = httptest.NewRecorder()
 638         s.testServer.Handler.ServeHTTP(resp, req)
 639         c.Check(resp.Header().Get("Location"), check.Equals, "")
 640         return resp
 641 }
 642
 643 func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) {
 644         s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 645         s.testDirectoryListing(c)
 646 }
 647
 648 func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) {
 649         s.testServer.Config.cluster.Users.AnonymousUserToken = ""
 650         s.testDirectoryListing(c)
 651 }
 652
 653 func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
 654         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 655         authHeader := http.Header{
 656                 "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
 657         }
 658         for _, trial := range []struct {
 659                 uri      string
 660                 header   http.Header
 661                 expect   []string
 662                 redirect string
 663                 cutDirs  int
 664         }{
 665                 {
 666                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
 667                         header:  authHeader,
 668                         expect:  []string{"dir1/foo", "dir1/bar"},
 669                         cutDirs: 0,
 670                 },
 671                 {
 672                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
 673                         header:  authHeader,
 674                         expect:  []string{"foo", "bar"},
 675                         cutDirs: 1,
 676                 },
 677                 {
 678                         // URLs of this form ignore authHeader, and
 679                         // FooAndBarFilesInDirUUID isn't public, so
 680                         // this returns 404.
 681                         uri:    "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
 682                         header: authHeader,
 683                         expect: nil,
 684                 },
 685                 {
 686                         uri:     "download.example.com/users/active/foo_file_in_dir/",
 687                         header:  authHeader,
 688                         expect:  []string{"dir1/"},
 689                         cutDirs: 3,
 690                 },
 691                 {
 692                         uri:     "download.example.com/users/active/foo_file_in_dir/dir1/",
 693                         header:  authHeader,
 694                         expect:  []string{"bar"},
 695                         cutDirs: 4,
 696                 },
 697                 {
 698                         uri:     "download.example.com/",
 699                         header:  authHeader,
 700                         expect:  []string{"users/"},
 701                         cutDirs: 0,
 702                 },
 703                 {
 704                         uri:      "download.example.com/users",
 705                         header:   authHeader,
 706                         redirect: "/users/",
 707                         expect:   []string{"active/"},
 708                         cutDirs:  1,
 709                 },
 710                 {
 711                         uri:     "download.example.com/users/",
 712                         header:  authHeader,
 713                         expect:  []string{"active/"},
 714                         cutDirs: 1,
 715                 },
 716                 {
 717                         uri:      "download.example.com/users/active",
 718                         header:   authHeader,
 719                         redirect: "/users/active/",
 720                         expect:   []string{"foo_file_in_dir/"},
 721                         cutDirs:  2,
 722                 },
 723                 {
 724                         uri:     "download.example.com/users/active/",
 725                         header:  authHeader,
 726                         expect:  []string{"foo_file_in_dir/"},
 727                         cutDirs: 2,
 728                 },
 729                 {
 730                         uri:     "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
 731                         header:  nil,
 732                         expect:  []string{"dir1/foo", "dir1/bar"},
 733                         cutDirs: 4,
 734                 },
 735                 {
 736                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
 737                         header:  nil,
 738                         expect:  []string{"dir1/foo", "dir1/bar"},
 739                         cutDirs: 2,
 740                 },
 741                 {
 742                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken,
 743                         header:  nil,
 744                         expect:  []string{"dir1/foo", "dir1/bar"},
 745                         cutDirs: 2,
 746                 },
 747                 {
 748                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID,
 749                         header:  authHeader,
 750                         expect:  []string{"dir1/foo", "dir1/bar"},
 751                         cutDirs: 1,
 752                 },
 753                 {
 754                         uri:      "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
 755                         header:   authHeader,
 756                         redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
 757                         expect:   []string{"foo", "bar"},
 758                         cutDirs:  2,
 759                 },
 760                 {
 761                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
 762                         header:  authHeader,
 763                         expect:  []string{"foo", "bar"},
 764                         cutDirs: 3,
 765                 },
 766                 {
 767                         uri:      arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
 768                         header:   authHeader,
 769                         redirect: "/dir1/",
 770                         expect:   []string{"foo", "bar"},
 771                         cutDirs:  1,
 772                 },
 773                 {
 774                         uri:    "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
 775                         header: authHeader,
 776                         expect: nil,
 777                 },
 778                 {
 779                         uri:     "download.example.com/c=" + arvadostest.WazVersion1Collection,
 780                         header:  authHeader,
 781                         expect:  []string{"waz"},
 782                         cutDirs: 1,
 783                 },
 784                 {
 785                         uri:     "download.example.com/by_id/" + arvadostest.WazVersion1Collection,
 786                         header:  authHeader,
 787                         expect:  []string{"waz"},
 788                         cutDirs: 2,
 789                 },
 790         } {
 791                 comment := check.Commentf("HTML: %q => %q", trial.uri, trial.expect)
 792                 resp := httptest.NewRecorder()
 793                 u := mustParseURL("//" + trial.uri)
 794                 req := &http.Request{
 795                         Method:     "GET",
 796                         Host:       u.Host,
 797                         URL:        u,
 798                         RequestURI: u.RequestURI(),
 799                         Header:     copyHeader(trial.header),
 800                 }
 801                 s.testServer.Handler.ServeHTTP(resp, req)
 802                 var cookies []*http.Cookie
 803                 for resp.Code == http.StatusSeeOther {
 804                         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 805                         req = &http.Request{
 806                                 Method:     "GET",
 807                                 Host:       u.Host,
 808                                 URL:        u,
 809                                 RequestURI: u.RequestURI(),
 810                                 Header:     copyHeader(trial.header),
 811                         }
 812                         cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
 813                         for _, c := range cookies {
 814                                 req.AddCookie(c)
 815                         }
 816                         resp = httptest.NewRecorder()
 817                         s.testServer.Handler.ServeHTTP(resp, req)
 818                 }
 819                 if trial.redirect != "" {
 820                         c.Check(req.URL.Path, check.Equals, trial.redirect, comment)
 821                 }
 822                 if trial.expect == nil {
 823                         c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
 824                 } else {
 825                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
 826                         for _, e := range trial.expect {
 827                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`, comment)
 828                         }
 829                         c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`, comment)
 830                 }
 831
 832                 comment = check.Commentf("WebDAV: %q => %q", trial.uri, trial.expect)
 833                 req = &http.Request{
 834                         Method:     "OPTIONS",
 835                         Host:       u.Host,
 836                         URL:        u,
 837                         RequestURI: u.RequestURI(),
 838                         Header:     copyHeader(trial.header),
 839                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
 840                 }
 841                 resp = httptest.NewRecorder()
 842                 s.testServer.Handler.ServeHTTP(resp, req)
 843                 if trial.expect == nil {
 844                         c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
 845                 } else {
 846                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
 847                 }
 848
 849                 req = &http.Request{
 850                         Method:     "PROPFIND",
 851                         Host:       u.Host,
 852                         URL:        u,
 853                         RequestURI: u.RequestURI(),
 854                         Header:     copyHeader(trial.header),
 855                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
 856                 }
 857                 resp = httptest.NewRecorder()
 858                 s.testServer.Handler.ServeHTTP(resp, req)
 859                 if trial.expect == nil {
 860                         c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
 861                 } else {
 862                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment)
 863                         for _, e := range trial.expect {
 864                                 if strings.HasSuffix(e, "/") {
 865                                         e = filepath.Join(u.Path, e) + "/"
 866                                 } else {
 867                                         e = filepath.Join(u.Path, e)
 868                                 }
 869                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+e+`</D:href>.*`, comment)
 870                         }
 871                 }
 872         }
 873 }
 874
 875 func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
 876         arv := arvados.NewClientFromEnv()
 877         var newCollection arvados.Collection
 878         err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 879                 "collection": map[string]string{
 880                         "owner_uuid":    arvadostest.ActiveUserUUID,
 881                         "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
 882                         "name":          "keep-web test collection",
 883                 },
 884                 "ensure_unique_name": true,
 885         })
 886         c.Assert(err, check.IsNil)
 887         defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
 888
 889         var updated arvados.Collection
 890         for _, fnm := range []string{"foo.txt", "bar.txt"} {
 891                 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
 892                 u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
 893                 req := &http.Request{
 894                         Method:     "DELETE",
 895                         Host:       u.Host,
 896                         URL:        u,
 897                         RequestURI: u.RequestURI(),
 898                         Header: http.Header{
 899                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
 900                         },
 901                 }
 902                 resp := httptest.NewRecorder()
 903                 s.testServer.Handler.ServeHTTP(resp, req)
 904                 c.Check(resp.Code, check.Equals, http.StatusNoContent)
 905
 906                 updated = arvados.Collection{}
 907                 err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
 908                 c.Check(err, check.IsNil)
 909                 c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`)
 910                 c.Logf("updated manifest_text %q", updated.ManifestText)
 911         }
 912         c.Check(updated.ManifestText, check.Equals, "")
 913 }
 914
 915 func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) {
 916         s.testServer.Config.cluster.ManagementToken = arvadostest.ManagementToken
 917         authHeader := http.Header{
 918                 "Authorization": {"Bearer " + arvadostest.ManagementToken},
 919         }
 920
 921         resp := httptest.NewRecorder()
 922         u := mustParseURL("http://download.example.com/_health/ping")
 923         req := &http.Request{
 924                 Method:     "GET",
 925                 Host:       u.Host,
 926                 URL:        u,
 927                 RequestURI: u.RequestURI(),
 928                 Header:     authHeader,
 929         }
 930         s.testServer.Handler.ServeHTTP(resp, req)
 931
 932         c.Check(resp.Code, check.Equals, http.StatusOK)
 933         c.Check(resp.Body.String(), check.Matches, `{"health":"OK"}\n`)
 934 }
 935
 936 func copyHeader(h http.Header) http.Header {
 937         hc := http.Header{}
 938         for k, v := range h {
 939                 hc[k] = append([]string(nil), v...)
 940         }
 941         return hc
 942 }