services/keep-web/handler_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "fmt"
  10         "html"
  11         "io/ioutil"
  12         "net/http"
  13         "net/http/httptest"
  14         "net/url"
  15         "os"
  16         "path/filepath"
  17         "regexp"
  18         "strings"
  19
  20         "git.arvados.org/arvados.git/lib/config"
  21         "git.arvados.org/arvados.git/sdk/go/arvados"
  22         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  23         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  24         "git.arvados.org/arvados.git/sdk/go/auth"
  25         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  26         "git.arvados.org/arvados.git/sdk/go/keepclient"
  27         check "gopkg.in/check.v1"
  28 )
  29
  30 var _ = check.Suite(&UnitSuite{})
  31
  32 type UnitSuite struct {
  33         Config *arvados.Config
  34 }
  35
  36 func (s *UnitSuite) SetUpTest(c *check.C) {
  37         ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), ctxlog.TestLogger(c))
  38         ldr.Path = "-"
  39         cfg, err := ldr.Load()
  40         c.Assert(err, check.IsNil)
  41         s.Config = cfg
  42 }
  43
  44 func (s *UnitSuite) TestKeepClientBlockCache(c *check.C) {
  45         cfg := newConfig(s.Config)
  46         cfg.cluster.Collections.WebDAVCache.MaxBlockEntries = 42
  47         h := handler{Config: cfg}
  48         c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Not(check.Equals), cfg.cluster.Collections.WebDAVCache.MaxBlockEntries)
  49         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/t=" + arvadostest.ActiveToken + "/foo")
  50         req := &http.Request{
  51                 Method:     "GET",
  52                 Host:       u.Host,
  53                 URL:        u,
  54                 RequestURI: u.RequestURI(),
  55         }
  56         resp := httptest.NewRecorder()
  57         h.ServeHTTP(resp, req)
  58         c.Check(resp.Code, check.Equals, http.StatusOK)
  59         c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Equals, cfg.cluster.Collections.WebDAVCache.MaxBlockEntries)
  60 }
  61
  62 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
  63         h := handler{Config: newConfig(s.Config)}
  64         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
  65         req := &http.Request{
  66                 Method:     "OPTIONS",
  67                 Host:       u.Host,
  68                 URL:        u,
  69                 RequestURI: u.RequestURI(),
  70                 Header: http.Header{
  71                         "Origin":                        {"https://workbench.example"},
  72                         "Access-Control-Request-Method": {"POST"},
  73                 },
  74         }
  75
  76         // Check preflight for an allowed request
  77         resp := httptest.NewRecorder()
  78         h.ServeHTTP(resp, req)
  79         c.Check(resp.Code, check.Equals, http.StatusOK)
  80         c.Check(resp.Body.String(), check.Equals, "")
  81         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
  82         c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
  83         c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout")
  84
  85         // Check preflight for a disallowed request
  86         resp = httptest.NewRecorder()
  87         req.Header.Set("Access-Control-Request-Method", "MAKE-COFFEE")
  88         h.ServeHTTP(resp, req)
  89         c.Check(resp.Body.String(), check.Equals, "")
  90         c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
  91 }
  92
  93 func (s *UnitSuite) TestInvalidUUID(c *check.C) {
  94         bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-"
  95         token := arvadostest.ActiveToken
  96         for _, trial := range []string{
  97                 "http://keep-web/c=" + bogusID + "/foo",
  98                 "http://keep-web/c=" + bogusID + "/t=" + token + "/foo",
  99                 "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo",
 100                 "http://keep-web/collections/" + bogusID + "/foo",
 101                 "http://" + bogusID + ".keep-web/" + bogusID + "/foo",
 102                 "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
 103         } {
 104                 c.Log(trial)
 105                 u := mustParseURL(trial)
 106                 req := &http.Request{
 107                         Method:     "GET",
 108                         Host:       u.Host,
 109                         URL:        u,
 110                         RequestURI: u.RequestURI(),
 111                 }
 112                 resp := httptest.NewRecorder()
 113                 cfg := newConfig(s.Config)
 114                 cfg.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 115                 h := handler{Config: cfg}
 116                 h.ServeHTTP(resp, req)
 117                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 118         }
 119 }
 120
 121 func mustParseURL(s string) *url.URL {
 122         r, err := url.Parse(s)
 123         if err != nil {
 124                 panic("parse URL: " + s)
 125         }
 126         return r
 127 }
 128
 129 func (s *IntegrationSuite) TestVhost404(c *check.C) {
 130         for _, testURL := range []string{
 131                 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
 132                 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
 133         } {
 134                 resp := httptest.NewRecorder()
 135                 u := mustParseURL(testURL)
 136                 req := &http.Request{
 137                         Method:     "GET",
 138                         URL:        u,
 139                         RequestURI: u.RequestURI(),
 140                 }
 141                 s.testServer.Handler.ServeHTTP(resp, req)
 142                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 143                 c.Check(resp.Body.String(), check.Equals, "")
 144         }
 145 }
 146
 147 // An authorizer modifies an HTTP request to make use of the given
 148 // token -- by adding it to a header, cookie, query param, or whatever
 149 // -- and returns the HTTP status code we should expect from keep-web if
 150 // the token is invalid.
 151 type authorizer func(*http.Request, string) int
 152
 153 func (s *IntegrationSuite) TestVhostViaAuthzHeader(c *check.C) {
 154         s.doVhostRequests(c, authzViaAuthzHeader)
 155 }
 156 func authzViaAuthzHeader(r *http.Request, tok string) int {
 157         r.Header.Add("Authorization", "OAuth2 "+tok)
 158         return http.StatusUnauthorized
 159 }
 160
 161 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
 162         s.doVhostRequests(c, authzViaCookieValue)
 163 }
 164 func authzViaCookieValue(r *http.Request, tok string) int {
 165         r.AddCookie(&http.Cookie{
 166                 Name:  "arvados_api_token",
 167                 Value: auth.EncodeTokenCookie([]byte(tok)),
 168         })
 169         return http.StatusUnauthorized
 170 }
 171
 172 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
 173         s.doVhostRequests(c, authzViaPath)
 174 }
 175 func authzViaPath(r *http.Request, tok string) int {
 176         r.URL.Path = "/t=" + tok + r.URL.Path
 177         return http.StatusNotFound
 178 }
 179
 180 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
 181         s.doVhostRequests(c, authzViaQueryString)
 182 }
 183 func authzViaQueryString(r *http.Request, tok string) int {
 184         r.URL.RawQuery = "api_token=" + tok
 185         return http.StatusUnauthorized
 186 }
 187
 188 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
 189         s.doVhostRequests(c, authzViaPOST)
 190 }
 191 func authzViaPOST(r *http.Request, tok string) int {
 192         r.Method = "POST"
 193         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 194         r.Body = ioutil.NopCloser(strings.NewReader(
 195                 url.Values{"api_token": {tok}}.Encode()))
 196         return http.StatusUnauthorized
 197 }
 198
 199 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
 200         s.doVhostRequests(c, authzViaPOST)
 201 }
 202 func authzViaXHRPOST(r *http.Request, tok string) int {
 203         r.Method = "POST"
 204         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 205         r.Header.Add("Origin", "https://origin.example")
 206         r.Body = ioutil.NopCloser(strings.NewReader(
 207                 url.Values{
 208                         "api_token":   {tok},
 209                         "disposition": {"attachment"},
 210                 }.Encode()))
 211         return http.StatusUnauthorized
 212 }
 213
 214 // Try some combinations of {url, token} using the given authorization
 215 // mechanism, and verify the result is correct.
 216 func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
 217         for _, hostPath := range []string{
 218                 arvadostest.FooCollection + ".example.com/foo",
 219                 arvadostest.FooCollection + "--collections.example.com/foo",
 220                 arvadostest.FooCollection + "--collections.example.com/_/foo",
 221                 arvadostest.FooCollectionPDH + ".example.com/foo",
 222                 strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + "--collections.example.com/foo",
 223                 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
 224         } {
 225                 c.Log("doRequests: ", hostPath)
 226                 s.doVhostRequestsWithHostPath(c, authz, hostPath)
 227         }
 228 }
 229
 230 func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
 231         for _, tok := range []string{
 232                 arvadostest.ActiveToken,
 233                 arvadostest.ActiveToken[:15],
 234                 arvadostest.SpectatorToken,
 235                 "bogus",
 236                 "",
 237         } {
 238                 u := mustParseURL("http://" + hostPath)
 239                 req := &http.Request{
 240                         Method:     "GET",
 241                         Host:       u.Host,
 242                         URL:        u,
 243                         RequestURI: u.RequestURI(),
 244                         Header:     http.Header{},
 245                 }
 246                 failCode := authz(req, tok)
 247                 req, resp := s.doReq(req)
 248                 code, body := resp.Code, resp.Body.String()
 249
 250                 // If the initial request had a (non-empty) token
 251                 // showing in the query string, we should have been
 252                 // redirected in order to hide it in a cookie.
 253                 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
 254
 255                 if tok == arvadostest.ActiveToken {
 256                         c.Check(code, check.Equals, http.StatusOK)
 257                         c.Check(body, check.Equals, "foo")
 258
 259                 } else {
 260                         c.Check(code >= 400, check.Equals, true)
 261                         c.Check(code < 500, check.Equals, true)
 262                         if tok == arvadostest.SpectatorToken {
 263                                 // Valid token never offers to retry
 264                                 // with different credentials.
 265                                 c.Check(code, check.Equals, http.StatusNotFound)
 266                         } else {
 267                                 // Invalid token can ask to retry
 268                                 // depending on the authz method.
 269                                 c.Check(code, check.Equals, failCode)
 270                         }
 271                         c.Check(body, check.Equals, "")
 272                 }
 273         }
 274 }
 275
 276 func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
 277         resp := httptest.NewRecorder()
 278         s.testServer.Handler.ServeHTTP(resp, req)
 279         if resp.Code != http.StatusSeeOther {
 280                 return req, resp
 281         }
 282         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 283         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 284         req = &http.Request{
 285                 Method:     "GET",
 286                 Host:       u.Host,
 287                 URL:        u,
 288                 RequestURI: u.RequestURI(),
 289                 Header:     http.Header{},
 290         }
 291         for _, c := range cookies {
 292                 req.AddCookie(c)
 293         }
 294         return s.doReq(req)
 295 }
 296
 297 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
 298         s.testVhostRedirectTokenToCookie(c, "GET",
 299                 arvadostest.FooCollection+".example.com/foo",
 300                 "?api_token="+arvadostest.ActiveToken,
 301                 "",
 302                 "",
 303                 http.StatusOK,
 304                 "foo",
 305         )
 306 }
 307
 308 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
 309         s.testVhostRedirectTokenToCookie(c, "GET",
 310                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
 311                 "",
 312                 "",
 313                 "",
 314                 http.StatusOK,
 315                 "foo",
 316         )
 317 }
 318
 319 // Bad token in URL is 404 Not Found because it doesn't make sense to
 320 // retry the same URL with different authorization.
 321 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
 322         s.testVhostRedirectTokenToCookie(c, "GET",
 323                 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
 324                 "",
 325                 "",
 326                 "",
 327                 http.StatusNotFound,
 328                 "",
 329         )
 330 }
 331
 332 // Bad token in a cookie (even if it got there via our own
 333 // query-string-to-cookie redirect) is, in principle, retryable at the
 334 // same URL so it's 401 Unauthorized.
 335 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
 336         s.testVhostRedirectTokenToCookie(c, "GET",
 337                 arvadostest.FooCollection+".example.com/foo",
 338                 "?api_token=thisisabogustoken",
 339                 "",
 340                 "",
 341                 http.StatusUnauthorized,
 342                 "",
 343         )
 344 }
 345
 346 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
 347         s.testVhostRedirectTokenToCookie(c, "GET",
 348                 "example.com/c="+arvadostest.FooCollection+"/foo",
 349                 "?api_token="+arvadostest.ActiveToken,
 350                 "",
 351                 "",
 352                 http.StatusBadRequest,
 353                 "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n",
 354         )
 355 }
 356
 357 // If client requests an attachment by putting ?disposition=attachment
 358 // in the query string, and gets redirected, the redirect target
 359 // should respond with an attachment.
 360 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
 361         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 362                 arvadostest.FooCollection+".example.com/foo",
 363                 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
 364                 "",
 365                 "",
 366                 http.StatusOK,
 367                 "foo",
 368         )
 369         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 370 }
 371
 372 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
 373         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 374         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 375                 "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
 376                 "?api_token="+arvadostest.ActiveToken,
 377                 "",
 378                 "",
 379                 http.StatusOK,
 380                 "foo",
 381         )
 382         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 383 }
 384
 385 func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
 386         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 387         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 388                 "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
 389                 "?api_token="+arvadostest.ActiveToken,
 390                 "",
 391                 "",
 392                 http.StatusOK,
 393                 "waz",
 394         )
 395         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 396         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 397                 "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz",
 398                 "?api_token="+arvadostest.ActiveToken,
 399                 "",
 400                 "",
 401                 http.StatusOK,
 402                 "waz",
 403         )
 404         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 405 }
 406
 407 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
 408         s.testServer.Config.cluster.Collections.TrustAllContent = true
 409         s.testVhostRedirectTokenToCookie(c, "GET",
 410                 "example.com/c="+arvadostest.FooCollection+"/foo",
 411                 "?api_token="+arvadostest.ActiveToken,
 412                 "",
 413                 "",
 414                 http.StatusOK,
 415                 "foo",
 416         )
 417 }
 418
 419 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
 420         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
 421
 422         s.testVhostRedirectTokenToCookie(c, "GET",
 423                 "example.com/c="+arvadostest.FooCollection+"/foo",
 424                 "?api_token="+arvadostest.ActiveToken,
 425                 "",
 426                 "",
 427                 http.StatusBadRequest,
 428                 "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n",
 429         )
 430
 431         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 432                 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
 433                 "?api_token="+arvadostest.ActiveToken,
 434                 "",
 435                 "",
 436                 http.StatusOK,
 437                 "foo",
 438         )
 439         c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
 440 }
 441
 442 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
 443         s.testVhostRedirectTokenToCookie(c, "POST",
 444                 arvadostest.FooCollection+".example.com/foo",
 445                 "",
 446                 "application/x-www-form-urlencoded",
 447                 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
 448                 http.StatusOK,
 449                 "foo",
 450         )
 451 }
 452
 453 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
 454         s.testVhostRedirectTokenToCookie(c, "POST",
 455                 arvadostest.FooCollection+".example.com/foo",
 456                 "",
 457                 "application/x-www-form-urlencoded",
 458                 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
 459                 http.StatusNotFound,
 460                 "",
 461         )
 462 }
 463
 464 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
 465         s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 466         s.testVhostRedirectTokenToCookie(c, "GET",
 467                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 468                 "",
 469                 "",
 470                 "",
 471                 http.StatusOK,
 472                 "Hello world\n",
 473         )
 474 }
 475
 476 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
 477         s.testServer.Config.cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
 478         s.testVhostRedirectTokenToCookie(c, "GET",
 479                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 480                 "",
 481                 "",
 482                 "",
 483                 http.StatusNotFound,
 484                 "",
 485         )
 486 }
 487
 488 func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
 489         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 490
 491         client := s.testServer.Config.Client
 492         client.AuthToken = arvadostest.ActiveToken
 493         fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
 494         c.Assert(err, check.IsNil)
 495         f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777)
 496         c.Assert(err, check.IsNil)
 497         f.Close()
 498         mtxt, err := fs.MarshalManifest(".")
 499         c.Assert(err, check.IsNil)
 500         var coll arvados.Collection
 501         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 502                 "collection": map[string]string{
 503                         "manifest_text": mtxt,
 504                 },
 505         })
 506         c.Assert(err, check.IsNil)
 507
 508         u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
 509         req := &http.Request{
 510                 Method:     "GET",
 511                 Host:       u.Host,
 512                 URL:        u,
 513                 RequestURI: u.RequestURI(),
 514                 Header: http.Header{
 515                         "Authorization": {"Bearer " + client.AuthToken},
 516                 },
 517         }
 518         resp := httptest.NewRecorder()
 519         s.testServer.Handler.ServeHTTP(resp, req)
 520         c.Check(resp.Code, check.Equals, http.StatusOK)
 521         c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\&#34;odd&#39; path chars.*`)
 522 }
 523
 524 func (s *IntegrationSuite) TestForwardSlashSubstitution(c *check.C) {
 525         arv := arvados.NewClientFromEnv()
 526         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 527         s.testServer.Config.cluster.Collections.ForwardSlashNameSubstitution = "{SOLIDUS}"
 528         name := "foo/bar/baz"
 529         nameShown := strings.Replace(name, "/", "{SOLIDUS}", -1)
 530         nameShownEscaped := strings.Replace(name, "/", "%7bSOLIDUS%7d", -1)
 531
 532         client := s.testServer.Config.Client
 533         client.AuthToken = arvadostest.ActiveToken
 534         fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
 535         c.Assert(err, check.IsNil)
 536         f, err := fs.OpenFile("filename", os.O_CREATE, 0777)
 537         c.Assert(err, check.IsNil)
 538         f.Close()
 539         mtxt, err := fs.MarshalManifest(".")
 540         c.Assert(err, check.IsNil)
 541         var coll arvados.Collection
 542         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 543                 "collection": map[string]string{
 544                         "manifest_text": mtxt,
 545                         "name":          name,
 546                         "owner_uuid":    arvadostest.AProjectUUID,
 547                 },
 548         })
 549         c.Assert(err, check.IsNil)
 550         defer arv.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
 551
 552         base := "http://download.example.com/by_id/" + coll.OwnerUUID + "/"
 553         for tryURL, expectRegexp := range map[string]string{
 554                 base:                          `(?ms).*href="./` + nameShownEscaped + `/"\S+` + nameShown + `.*`,
 555                 base + nameShownEscaped + "/": `(?ms).*href="./filename"\S+filename.*`,
 556         } {
 557                 u, _ := url.Parse(tryURL)
 558                 req := &http.Request{
 559                         Method:     "GET",
 560                         Host:       u.Host,
 561                         URL:        u,
 562                         RequestURI: u.RequestURI(),
 563                         Header: http.Header{
 564                                 "Authorization": {"Bearer " + client.AuthToken},
 565                         },
 566                 }
 567                 resp := httptest.NewRecorder()
 568                 s.testServer.Handler.ServeHTTP(resp, req)
 569                 c.Check(resp.Code, check.Equals, http.StatusOK)
 570                 c.Check(resp.Body.String(), check.Matches, expectRegexp)
 571         }
 572 }
 573
 574 // XHRs can't follow redirect-with-cookie so they rely on method=POST
 575 // and disposition=attachment (telling us it's acceptable to respond
 576 // with content instead of a redirect) and an Origin header that gets
 577 // added automatically by the browser (telling us it's desirable to do
 578 // so).
 579 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
 580         u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
 581         req := &http.Request{
 582                 Method:     "POST",
 583                 Host:       u.Host,
 584                 URL:        u,
 585                 RequestURI: u.RequestURI(),
 586                 Header: http.Header{
 587                         "Origin":       {"https://origin.example"},
 588                         "Content-Type": {"application/x-www-form-urlencoded"},
 589                 },
 590                 Body: ioutil.NopCloser(strings.NewReader(url.Values{
 591                         "api_token":   {arvadostest.ActiveToken},
 592                         "disposition": {"attachment"},
 593                 }.Encode())),
 594         }
 595         resp := httptest.NewRecorder()
 596         s.testServer.Handler.ServeHTTP(resp, req)
 597         c.Check(resp.Code, check.Equals, http.StatusOK)
 598         c.Check(resp.Body.String(), check.Equals, "foo")
 599         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
 600 }
 601
 602 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder {
 603         u, _ := url.Parse(`http://` + hostPath + queryString)
 604         req := &http.Request{
 605                 Method:     method,
 606                 Host:       u.Host,
 607                 URL:        u,
 608                 RequestURI: u.RequestURI(),
 609                 Header:     http.Header{"Content-Type": {contentType}},
 610                 Body:       ioutil.NopCloser(strings.NewReader(reqBody)),
 611         }
 612
 613         resp := httptest.NewRecorder()
 614         defer func() {
 615                 c.Check(resp.Code, check.Equals, expectStatus)
 616                 c.Check(resp.Body.String(), check.Equals, expectRespBody)
 617         }()
 618
 619         s.testServer.Handler.ServeHTTP(resp, req)
 620         if resp.Code != http.StatusSeeOther {
 621                 return resp
 622         }
 623         c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
 624         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 625
 626         u, _ = u.Parse(resp.Header().Get("Location"))
 627         req = &http.Request{
 628                 Method:     "GET",
 629                 Host:       u.Host,
 630                 URL:        u,
 631                 RequestURI: u.RequestURI(),
 632                 Header:     http.Header{},
 633         }
 634         for _, c := range cookies {
 635                 req.AddCookie(c)
 636         }
 637
 638         resp = httptest.NewRecorder()
 639         s.testServer.Handler.ServeHTTP(resp, req)
 640         c.Check(resp.Header().Get("Location"), check.Equals, "")
 641         return resp
 642 }
 643
 644 func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) {
 645         s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 646         s.testDirectoryListing(c)
 647 }
 648
 649 func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) {
 650         s.testServer.Config.cluster.Users.AnonymousUserToken = ""
 651         s.testDirectoryListing(c)
 652 }
 653
 654 func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
 655         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 656         authHeader := http.Header{
 657                 "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
 658         }
 659         for _, trial := range []struct {
 660                 uri      string
 661                 header   http.Header
 662                 expect   []string
 663                 redirect string
 664                 cutDirs  int
 665         }{
 666                 {
 667                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
 668                         header:  authHeader,
 669                         expect:  []string{"dir1/foo", "dir1/bar"},
 670                         cutDirs: 0,
 671                 },
 672                 {
 673                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
 674                         header:  authHeader,
 675                         expect:  []string{"foo", "bar"},
 676                         cutDirs: 1,
 677                 },
 678                 {
 679                         // URLs of this form ignore authHeader, and
 680                         // FooAndBarFilesInDirUUID isn't public, so
 681                         // this returns 404.
 682                         uri:    "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
 683                         header: authHeader,
 684                         expect: nil,
 685                 },
 686                 {
 687                         uri:     "download.example.com/users/active/foo_file_in_dir/",
 688                         header:  authHeader,
 689                         expect:  []string{"dir1/"},
 690                         cutDirs: 3,
 691                 },
 692                 {
 693                         uri:     "download.example.com/users/active/foo_file_in_dir/dir1/",
 694                         header:  authHeader,
 695                         expect:  []string{"bar"},
 696                         cutDirs: 4,
 697                 },
 698                 {
 699                         uri:     "download.example.com/",
 700                         header:  authHeader,
 701                         expect:  []string{"users/"},
 702                         cutDirs: 0,
 703                 },
 704                 {
 705                         uri:      "download.example.com/users",
 706                         header:   authHeader,
 707                         redirect: "/users/",
 708                         expect:   []string{"active/"},
 709                         cutDirs:  1,
 710                 },
 711                 {
 712                         uri:     "download.example.com/users/",
 713                         header:  authHeader,
 714                         expect:  []string{"active/"},
 715                         cutDirs: 1,
 716                 },
 717                 {
 718                         uri:      "download.example.com/users/active",
 719                         header:   authHeader,
 720                         redirect: "/users/active/",
 721                         expect:   []string{"foo_file_in_dir/"},
 722                         cutDirs:  2,
 723                 },
 724                 {
 725                         uri:     "download.example.com/users/active/",
 726                         header:  authHeader,
 727                         expect:  []string{"foo_file_in_dir/"},
 728                         cutDirs: 2,
 729                 },
 730                 {
 731                         uri:     "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
 732                         header:  nil,
 733                         expect:  []string{"dir1/foo", "dir1/bar"},
 734                         cutDirs: 4,
 735                 },
 736                 {
 737                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
 738                         header:  nil,
 739                         expect:  []string{"dir1/foo", "dir1/bar"},
 740                         cutDirs: 2,
 741                 },
 742                 {
 743                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken,
 744                         header:  nil,
 745                         expect:  []string{"dir1/foo", "dir1/bar"},
 746                         cutDirs: 2,
 747                 },
 748                 {
 749                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID,
 750                         header:  authHeader,
 751                         expect:  []string{"dir1/foo", "dir1/bar"},
 752                         cutDirs: 1,
 753                 },
 754                 {
 755                         uri:      "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
 756                         header:   authHeader,
 757                         redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
 758                         expect:   []string{"foo", "bar"},
 759                         cutDirs:  2,
 760                 },
 761                 {
 762                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
 763                         header:  authHeader,
 764                         expect:  []string{"foo", "bar"},
 765                         cutDirs: 3,
 766                 },
 767                 {
 768                         uri:      arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
 769                         header:   authHeader,
 770                         redirect: "/dir1/",
 771                         expect:   []string{"foo", "bar"},
 772                         cutDirs:  1,
 773                 },
 774                 {
 775                         uri:    "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
 776                         header: authHeader,
 777                         expect: nil,
 778                 },
 779                 {
 780                         uri:     "download.example.com/c=" + arvadostest.WazVersion1Collection,
 781                         header:  authHeader,
 782                         expect:  []string{"waz"},
 783                         cutDirs: 1,
 784                 },
 785                 {
 786                         uri:     "download.example.com/by_id/" + arvadostest.WazVersion1Collection,
 787                         header:  authHeader,
 788                         expect:  []string{"waz"},
 789                         cutDirs: 2,
 790                 },
 791         } {
 792                 comment := check.Commentf("HTML: %q => %q", trial.uri, trial.expect)
 793                 resp := httptest.NewRecorder()
 794                 u := mustParseURL("//" + trial.uri)
 795                 req := &http.Request{
 796                         Method:     "GET",
 797                         Host:       u.Host,
 798                         URL:        u,
 799                         RequestURI: u.RequestURI(),
 800                         Header:     copyHeader(trial.header),
 801                 }
 802                 s.testServer.Handler.ServeHTTP(resp, req)
 803                 var cookies []*http.Cookie
 804                 for resp.Code == http.StatusSeeOther {
 805                         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 806                         req = &http.Request{
 807                                 Method:     "GET",
 808                                 Host:       u.Host,
 809                                 URL:        u,
 810                                 RequestURI: u.RequestURI(),
 811                                 Header:     copyHeader(trial.header),
 812                         }
 813                         cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
 814                         for _, c := range cookies {
 815                                 req.AddCookie(c)
 816                         }
 817                         resp = httptest.NewRecorder()
 818                         s.testServer.Handler.ServeHTTP(resp, req)
 819                 }
 820                 if trial.redirect != "" {
 821                         c.Check(req.URL.Path, check.Equals, trial.redirect, comment)
 822                 }
 823                 if trial.expect == nil {
 824                         c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
 825                 } else {
 826                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
 827                         for _, e := range trial.expect {
 828                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`, comment)
 829                         }
 830                         c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`, comment)
 831                 }
 832
 833                 comment = check.Commentf("WebDAV: %q => %q", trial.uri, trial.expect)
 834                 req = &http.Request{
 835                         Method:     "OPTIONS",
 836                         Host:       u.Host,
 837                         URL:        u,
 838                         RequestURI: u.RequestURI(),
 839                         Header:     copyHeader(trial.header),
 840                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
 841                 }
 842                 resp = httptest.NewRecorder()
 843                 s.testServer.Handler.ServeHTTP(resp, req)
 844                 if trial.expect == nil {
 845                         c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
 846                 } else {
 847                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
 848                 }
 849
 850                 req = &http.Request{
 851                         Method:     "PROPFIND",
 852                         Host:       u.Host,
 853                         URL:        u,
 854                         RequestURI: u.RequestURI(),
 855                         Header:     copyHeader(trial.header),
 856                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
 857                 }
 858                 resp = httptest.NewRecorder()
 859                 s.testServer.Handler.ServeHTTP(resp, req)
 860                 if trial.expect == nil {
 861                         c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
 862                 } else {
 863                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment)
 864                         for _, e := range trial.expect {
 865                                 if strings.HasSuffix(e, "/") {
 866                                         e = filepath.Join(u.Path, e) + "/"
 867                                 } else {
 868                                         e = filepath.Join(u.Path, e)
 869                                 }
 870                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+e+`</D:href>.*`, comment)
 871                         }
 872                 }
 873         }
 874 }
 875
 876 func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
 877         arv := arvados.NewClientFromEnv()
 878         var newCollection arvados.Collection
 879         err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 880                 "collection": map[string]string{
 881                         "owner_uuid":    arvadostest.ActiveUserUUID,
 882                         "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
 883                         "name":          "keep-web test collection",
 884                 },
 885                 "ensure_unique_name": true,
 886         })
 887         c.Assert(err, check.IsNil)
 888         defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
 889
 890         var updated arvados.Collection
 891         for _, fnm := range []string{"foo.txt", "bar.txt"} {
 892                 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
 893                 u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
 894                 req := &http.Request{
 895                         Method:     "DELETE",
 896                         Host:       u.Host,
 897                         URL:        u,
 898                         RequestURI: u.RequestURI(),
 899                         Header: http.Header{
 900                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
 901                         },
 902                 }
 903                 resp := httptest.NewRecorder()
 904                 s.testServer.Handler.ServeHTTP(resp, req)
 905                 c.Check(resp.Code, check.Equals, http.StatusNoContent)
 906
 907                 updated = arvados.Collection{}
 908                 err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
 909                 c.Check(err, check.IsNil)
 910                 c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`)
 911                 c.Logf("updated manifest_text %q", updated.ManifestText)
 912         }
 913         c.Check(updated.ManifestText, check.Equals, "")
 914 }
 915
 916 func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) {
 917         s.testServer.Config.cluster.ManagementToken = arvadostest.ManagementToken
 918         authHeader := http.Header{
 919                 "Authorization": {"Bearer " + arvadostest.ManagementToken},
 920         }
 921
 922         resp := httptest.NewRecorder()
 923         u := mustParseURL("http://download.example.com/_health/ping")
 924         req := &http.Request{
 925                 Method:     "GET",
 926                 Host:       u.Host,
 927                 URL:        u,
 928                 RequestURI: u.RequestURI(),
 929                 Header:     authHeader,
 930         }
 931         s.testServer.Handler.ServeHTTP(resp, req)
 932
 933         c.Check(resp.Code, check.Equals, http.StatusOK)
 934         c.Check(resp.Body.String(), check.Matches, `{"health":"OK"}\n`)
 935 }
 936
 937 func (s *IntegrationSuite) TestFileContentType(c *check.C) {
 938         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 939
 940         client := s.testServer.Config.Client
 941         client.AuthToken = arvadostest.ActiveToken
 942         arv, err := arvadosclient.New(&client)
 943         c.Assert(err, check.Equals, nil)
 944         kc, err := keepclient.MakeKeepClient(arv)
 945         c.Assert(err, check.Equals, nil)
 946
 947         fs, err := (&arvados.Collection{}).FileSystem(&client, kc)
 948         c.Assert(err, check.IsNil)
 949
 950         trials := []struct {
 951                 filename    string
 952                 content     string
 953                 contentType string
 954         }{
 955                 {"picture.txt", "BMX bikes are small this year\n", "text/plain; charset=utf-8"},
 956                 {"picture.bmp", "BMX bikes are small this year\n", "image/x-ms-bmp"},
 957                 {"picture.jpg", "BMX bikes are small this year\n", "image/jpeg"},
 958                 {"picture1", "BMX bikes are small this year\n", "image/bmp"},            // content sniff; "BM" is the magic signature for .bmp
 959                 {"picture2", "Cars are small this year\n", "text/plain; charset=utf-8"}, // content sniff
 960         }
 961         for _, trial := range trials {
 962                 f, err := fs.OpenFile(trial.filename, os.O_CREATE|os.O_WRONLY, 0777)
 963                 c.Assert(err, check.IsNil)
 964                 _, err = f.Write([]byte(trial.content))
 965                 c.Assert(err, check.IsNil)
 966                 c.Assert(f.Close(), check.IsNil)
 967         }
 968         mtxt, err := fs.MarshalManifest(".")
 969         c.Assert(err, check.IsNil)
 970         var coll arvados.Collection
 971         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 972                 "collection": map[string]string{
 973                         "manifest_text": mtxt,
 974                 },
 975         })
 976         c.Assert(err, check.IsNil)
 977
 978         for _, trial := range trials {
 979                 u, _ := url.Parse("http://download.example.com/by_id/" + coll.UUID + "/" + trial.filename)
 980                 req := &http.Request{
 981                         Method:     "GET",
 982                         Host:       u.Host,
 983                         URL:        u,
 984                         RequestURI: u.RequestURI(),
 985                         Header: http.Header{
 986                                 "Authorization": {"Bearer " + client.AuthToken},
 987                         },
 988                 }
 989                 resp := httptest.NewRecorder()
 990                 s.testServer.Handler.ServeHTTP(resp, req)
 991                 c.Check(resp.Code, check.Equals, http.StatusOK)
 992                 c.Check(resp.Header().Get("Content-Type"), check.Equals, trial.contentType)
 993                 c.Check(resp.Body.String(), check.Equals, trial.content)
 994         }
 995 }
 996
 997 func copyHeader(h http.Header) http.Header {
 998         hc := http.Header{}
 999         for k, v := range h {
1000                 hc[k] = append([]string(nil), v...)
1001         }
1002         return hc
1003 }