services/keep-web/handler_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "fmt"
  11         "html"
  12         "io/ioutil"
  13         "net/http"
  14         "net/http/httptest"
  15         "net/url"
  16         "os"
  17         "path/filepath"
  18         "regexp"
  19         "strings"
  20         "time"
  21
  22         "git.arvados.org/arvados.git/lib/config"
  23         "git.arvados.org/arvados.git/sdk/go/arvados"
  24         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  25         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  26         "git.arvados.org/arvados.git/sdk/go/auth"
  27         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  28         "git.arvados.org/arvados.git/sdk/go/keepclient"
  29         "github.com/sirupsen/logrus"
  30         check "gopkg.in/check.v1"
  31 )
  32
  33 var _ = check.Suite(&UnitSuite{})
  34
  35 type UnitSuite struct {
  36         Config *arvados.Config
  37 }
  38
  39 func (s *UnitSuite) SetUpTest(c *check.C) {
  40         ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), ctxlog.TestLogger(c))
  41         ldr.Path = "-"
  42         cfg, err := ldr.Load()
  43         c.Assert(err, check.IsNil)
  44         s.Config = cfg
  45 }
  46
  47 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
  48         h := handler{Config: newConfig(s.Config)}
  49         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
  50         req := &http.Request{
  51                 Method:     "OPTIONS",
  52                 Host:       u.Host,
  53                 URL:        u,
  54                 RequestURI: u.RequestURI(),
  55                 Header: http.Header{
  56                         "Origin":                        {"https://workbench.example"},
  57                         "Access-Control-Request-Method": {"POST"},
  58                 },
  59         }
  60
  61         // Check preflight for an allowed request
  62         resp := httptest.NewRecorder()
  63         h.ServeHTTP(resp, req)
  64         c.Check(resp.Code, check.Equals, http.StatusOK)
  65         c.Check(resp.Body.String(), check.Equals, "")
  66         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
  67         c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
  68         c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout")
  69
  70         // Check preflight for a disallowed request
  71         resp = httptest.NewRecorder()
  72         req.Header.Set("Access-Control-Request-Method", "MAKE-COFFEE")
  73         h.ServeHTTP(resp, req)
  74         c.Check(resp.Body.String(), check.Equals, "")
  75         c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
  76 }
  77
  78 func (s *UnitSuite) TestEmptyResponse(c *check.C) {
  79         for _, trial := range []struct {
  80                 dataExists    bool
  81                 sendIMSHeader bool
  82                 expectStatus  int
  83                 logRegexp     string
  84         }{
  85                 // If we return no content due to a Keep read error,
  86                 // we should emit a log message.
  87                 {false, false, http.StatusOK, `(?ms).*only wrote 0 bytes.*`},
  88
  89                 // If we return no content because the client sent an
  90                 // If-Modified-Since header, our response should be
  91                 // 304, and we should not emit a log message.
  92                 {true, true, http.StatusNotModified, ``},
  93         } {
  94                 c.Logf("trial: %+v", trial)
  95                 arvadostest.StartKeep(2, true)
  96                 if trial.dataExists {
  97                         arv, err := arvadosclient.MakeArvadosClient()
  98                         c.Assert(err, check.IsNil)
  99                         arv.ApiToken = arvadostest.ActiveToken
 100                         kc, err := keepclient.MakeKeepClient(arv)
 101                         c.Assert(err, check.IsNil)
 102                         _, _, err = kc.PutB([]byte("foo"))
 103                         c.Assert(err, check.IsNil)
 104                 }
 105
 106                 h := handler{Config: newConfig(s.Config)}
 107                 u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
 108                 req := &http.Request{
 109                         Method:     "GET",
 110                         Host:       u.Host,
 111                         URL:        u,
 112                         RequestURI: u.RequestURI(),
 113                         Header: http.Header{
 114                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
 115                         },
 116                 }
 117                 if trial.sendIMSHeader {
 118                         req.Header.Set("If-Modified-Since", strings.Replace(time.Now().UTC().Format(time.RFC1123), "UTC", "GMT", -1))
 119                 }
 120
 121                 var logbuf bytes.Buffer
 122                 logger := logrus.New()
 123                 logger.Out = &logbuf
 124                 req = req.WithContext(ctxlog.Context(context.Background(), logger))
 125
 126                 resp := httptest.NewRecorder()
 127                 h.ServeHTTP(resp, req)
 128                 c.Check(resp.Code, check.Equals, trial.expectStatus)
 129                 c.Check(resp.Body.String(), check.Equals, "")
 130
 131                 c.Log(logbuf.String())
 132                 c.Check(logbuf.String(), check.Matches, trial.logRegexp)
 133         }
 134 }
 135
 136 func (s *UnitSuite) TestInvalidUUID(c *check.C) {
 137         bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-"
 138         token := arvadostest.ActiveToken
 139         for _, trial := range []string{
 140                 "http://keep-web/c=" + bogusID + "/foo",
 141                 "http://keep-web/c=" + bogusID + "/t=" + token + "/foo",
 142                 "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo",
 143                 "http://keep-web/collections/" + bogusID + "/foo",
 144                 "http://" + bogusID + ".keep-web/" + bogusID + "/foo",
 145                 "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
 146         } {
 147                 c.Log(trial)
 148                 u := mustParseURL(trial)
 149                 req := &http.Request{
 150                         Method:     "GET",
 151                         Host:       u.Host,
 152                         URL:        u,
 153                         RequestURI: u.RequestURI(),
 154                 }
 155                 resp := httptest.NewRecorder()
 156                 cfg := newConfig(s.Config)
 157                 cfg.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 158                 h := handler{Config: cfg}
 159                 h.ServeHTTP(resp, req)
 160                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 161         }
 162 }
 163
 164 func mustParseURL(s string) *url.URL {
 165         r, err := url.Parse(s)
 166         if err != nil {
 167                 panic("parse URL: " + s)
 168         }
 169         return r
 170 }
 171
 172 func (s *IntegrationSuite) TestVhost404(c *check.C) {
 173         for _, testURL := range []string{
 174                 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
 175                 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
 176         } {
 177                 resp := httptest.NewRecorder()
 178                 u := mustParseURL(testURL)
 179                 req := &http.Request{
 180                         Method:     "GET",
 181                         URL:        u,
 182                         RequestURI: u.RequestURI(),
 183                 }
 184                 s.testServer.Handler.ServeHTTP(resp, req)
 185                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 186                 c.Check(resp.Body.String(), check.Equals, notFoundMessage+"\n")
 187         }
 188 }
 189
 190 // An authorizer modifies an HTTP request to make use of the given
 191 // token -- by adding it to a header, cookie, query param, or whatever
 192 // -- and returns the HTTP status code we should expect from keep-web if
 193 // the token is invalid.
 194 type authorizer func(*http.Request, string) int
 195
 196 func (s *IntegrationSuite) TestVhostViaAuthzHeader(c *check.C) {
 197         s.doVhostRequests(c, authzViaAuthzHeader)
 198 }
 199 func authzViaAuthzHeader(r *http.Request, tok string) int {
 200         r.Header.Add("Authorization", "OAuth2 "+tok)
 201         return http.StatusUnauthorized
 202 }
 203
 204 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
 205         s.doVhostRequests(c, authzViaCookieValue)
 206 }
 207 func authzViaCookieValue(r *http.Request, tok string) int {
 208         r.AddCookie(&http.Cookie{
 209                 Name:  "arvados_api_token",
 210                 Value: auth.EncodeTokenCookie([]byte(tok)),
 211         })
 212         return http.StatusUnauthorized
 213 }
 214
 215 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
 216         s.doVhostRequests(c, authzViaPath)
 217 }
 218 func authzViaPath(r *http.Request, tok string) int {
 219         r.URL.Path = "/t=" + tok + r.URL.Path
 220         return http.StatusNotFound
 221 }
 222
 223 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
 224         s.doVhostRequests(c, authzViaQueryString)
 225 }
 226 func authzViaQueryString(r *http.Request, tok string) int {
 227         r.URL.RawQuery = "api_token=" + tok
 228         return http.StatusUnauthorized
 229 }
 230
 231 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
 232         s.doVhostRequests(c, authzViaPOST)
 233 }
 234 func authzViaPOST(r *http.Request, tok string) int {
 235         r.Method = "POST"
 236         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 237         r.Body = ioutil.NopCloser(strings.NewReader(
 238                 url.Values{"api_token": {tok}}.Encode()))
 239         return http.StatusUnauthorized
 240 }
 241
 242 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
 243         s.doVhostRequests(c, authzViaPOST)
 244 }
 245 func authzViaXHRPOST(r *http.Request, tok string) int {
 246         r.Method = "POST"
 247         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 248         r.Header.Add("Origin", "https://origin.example")
 249         r.Body = ioutil.NopCloser(strings.NewReader(
 250                 url.Values{
 251                         "api_token":   {tok},
 252                         "disposition": {"attachment"},
 253                 }.Encode()))
 254         return http.StatusUnauthorized
 255 }
 256
 257 // Try some combinations of {url, token} using the given authorization
 258 // mechanism, and verify the result is correct.
 259 func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
 260         for _, hostPath := range []string{
 261                 arvadostest.FooCollection + ".example.com/foo",
 262                 arvadostest.FooCollection + "--collections.example.com/foo",
 263                 arvadostest.FooCollection + "--collections.example.com/_/foo",
 264                 arvadostest.FooCollectionPDH + ".example.com/foo",
 265                 strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + "--collections.example.com/foo",
 266                 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
 267         } {
 268                 c.Log("doRequests: ", hostPath)
 269                 s.doVhostRequestsWithHostPath(c, authz, hostPath)
 270         }
 271 }
 272
 273 func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
 274         for _, tok := range []string{
 275                 arvadostest.ActiveToken,
 276                 arvadostest.ActiveToken[:15],
 277                 arvadostest.SpectatorToken,
 278                 "bogus",
 279                 "",
 280         } {
 281                 u := mustParseURL("http://" + hostPath)
 282                 req := &http.Request{
 283                         Method:     "GET",
 284                         Host:       u.Host,
 285                         URL:        u,
 286                         RequestURI: u.RequestURI(),
 287                         Header:     http.Header{},
 288                 }
 289                 failCode := authz(req, tok)
 290                 req, resp := s.doReq(req)
 291                 code, body := resp.Code, resp.Body.String()
 292
 293                 // If the initial request had a (non-empty) token
 294                 // showing in the query string, we should have been
 295                 // redirected in order to hide it in a cookie.
 296                 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
 297
 298                 if tok == arvadostest.ActiveToken {
 299                         c.Check(code, check.Equals, http.StatusOK)
 300                         c.Check(body, check.Equals, "foo")
 301                 } else {
 302                         c.Check(code >= 400, check.Equals, true)
 303                         c.Check(code < 500, check.Equals, true)
 304                         if tok == arvadostest.SpectatorToken {
 305                                 // Valid token never offers to retry
 306                                 // with different credentials.
 307                                 c.Check(code, check.Equals, http.StatusNotFound)
 308                         } else {
 309                                 // Invalid token can ask to retry
 310                                 // depending on the authz method.
 311                                 c.Check(code, check.Equals, failCode)
 312                         }
 313                         if code == 404 {
 314                                 c.Check(body, check.Equals, notFoundMessage+"\n")
 315                         } else {
 316                                 c.Check(body, check.Equals, unauthorizedMessage+"\n")
 317                         }
 318                 }
 319         }
 320 }
 321
 322 func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
 323         resp := httptest.NewRecorder()
 324         s.testServer.Handler.ServeHTTP(resp, req)
 325         if resp.Code != http.StatusSeeOther {
 326                 return req, resp
 327         }
 328         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 329         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 330         req = &http.Request{
 331                 Method:     "GET",
 332                 Host:       u.Host,
 333                 URL:        u,
 334                 RequestURI: u.RequestURI(),
 335                 Header:     http.Header{},
 336         }
 337         for _, c := range cookies {
 338                 req.AddCookie(c)
 339         }
 340         return s.doReq(req)
 341 }
 342
 343 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
 344         s.testVhostRedirectTokenToCookie(c, "GET",
 345                 arvadostest.FooCollection+".example.com/foo",
 346                 "?api_token="+arvadostest.ActiveToken,
 347                 "",
 348                 "",
 349                 http.StatusOK,
 350                 "foo",
 351         )
 352 }
 353
 354 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
 355         s.testVhostRedirectTokenToCookie(c, "GET",
 356                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
 357                 "",
 358                 "",
 359                 "",
 360                 http.StatusOK,
 361                 "foo",
 362         )
 363 }
 364
 365 // Bad token in URL is 404 Not Found because it doesn't make sense to
 366 // retry the same URL with different authorization.
 367 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
 368         s.testVhostRedirectTokenToCookie(c, "GET",
 369                 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
 370                 "",
 371                 "",
 372                 "",
 373                 http.StatusNotFound,
 374                 notFoundMessage+"\n",
 375         )
 376 }
 377
 378 // Bad token in a cookie (even if it got there via our own
 379 // query-string-to-cookie redirect) is, in principle, retryable at the
 380 // same URL so it's 401 Unauthorized.
 381 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
 382         s.testVhostRedirectTokenToCookie(c, "GET",
 383                 arvadostest.FooCollection+".example.com/foo",
 384                 "?api_token=thisisabogustoken",
 385                 "",
 386                 "",
 387                 http.StatusUnauthorized,
 388                 unauthorizedMessage+"\n",
 389         )
 390 }
 391
 392 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
 393         s.testVhostRedirectTokenToCookie(c, "GET",
 394                 "example.com/c="+arvadostest.FooCollection+"/foo",
 395                 "?api_token="+arvadostest.ActiveToken,
 396                 "",
 397                 "",
 398                 http.StatusBadRequest,
 399                 "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n",
 400         )
 401 }
 402
 403 // If client requests an attachment by putting ?disposition=attachment
 404 // in the query string, and gets redirected, the redirect target
 405 // should respond with an attachment.
 406 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
 407         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 408                 arvadostest.FooCollection+".example.com/foo",
 409                 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
 410                 "",
 411                 "",
 412                 http.StatusOK,
 413                 "foo",
 414         )
 415         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 416 }
 417
 418 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
 419         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 420         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 421                 "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
 422                 "?api_token="+arvadostest.ActiveToken,
 423                 "",
 424                 "",
 425                 http.StatusOK,
 426                 "foo",
 427         )
 428         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 429 }
 430
 431 func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
 432         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 433         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 434                 "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
 435                 "?api_token="+arvadostest.ActiveToken,
 436                 "",
 437                 "",
 438                 http.StatusOK,
 439                 "waz",
 440         )
 441         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 442         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 443                 "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz",
 444                 "?api_token="+arvadostest.ActiveToken,
 445                 "",
 446                 "",
 447                 http.StatusOK,
 448                 "waz",
 449         )
 450         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 451 }
 452
 453 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
 454         s.testServer.Config.cluster.Collections.TrustAllContent = true
 455         s.testVhostRedirectTokenToCookie(c, "GET",
 456                 "example.com/c="+arvadostest.FooCollection+"/foo",
 457                 "?api_token="+arvadostest.ActiveToken,
 458                 "",
 459                 "",
 460                 http.StatusOK,
 461                 "foo",
 462         )
 463 }
 464
 465 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
 466         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
 467
 468         s.testVhostRedirectTokenToCookie(c, "GET",
 469                 "example.com/c="+arvadostest.FooCollection+"/foo",
 470                 "?api_token="+arvadostest.ActiveToken,
 471                 "",
 472                 "",
 473                 http.StatusBadRequest,
 474                 "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n",
 475         )
 476
 477         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 478                 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
 479                 "?api_token="+arvadostest.ActiveToken,
 480                 "",
 481                 "",
 482                 http.StatusOK,
 483                 "foo",
 484         )
 485         c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
 486 }
 487
 488 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
 489         s.testVhostRedirectTokenToCookie(c, "POST",
 490                 arvadostest.FooCollection+".example.com/foo",
 491                 "",
 492                 "application/x-www-form-urlencoded",
 493                 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
 494                 http.StatusOK,
 495                 "foo",
 496         )
 497 }
 498
 499 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
 500         s.testVhostRedirectTokenToCookie(c, "POST",
 501                 arvadostest.FooCollection+".example.com/foo",
 502                 "",
 503                 "application/x-www-form-urlencoded",
 504                 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
 505                 http.StatusNotFound,
 506                 notFoundMessage+"\n",
 507         )
 508 }
 509
 510 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
 511         s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 512         s.testVhostRedirectTokenToCookie(c, "GET",
 513                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 514                 "",
 515                 "",
 516                 "",
 517                 http.StatusOK,
 518                 "Hello world\n",
 519         )
 520 }
 521
 522 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
 523         s.testServer.Config.cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
 524         s.testVhostRedirectTokenToCookie(c, "GET",
 525                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 526                 "",
 527                 "",
 528                 "",
 529                 http.StatusNotFound,
 530                 notFoundMessage+"\n",
 531         )
 532 }
 533
 534 func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
 535         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 536
 537         client := s.testServer.Config.Client
 538         client.AuthToken = arvadostest.ActiveToken
 539         fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
 540         c.Assert(err, check.IsNil)
 541         f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777)
 542         c.Assert(err, check.IsNil)
 543         f.Close()
 544         mtxt, err := fs.MarshalManifest(".")
 545         c.Assert(err, check.IsNil)
 546         var coll arvados.Collection
 547         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 548                 "collection": map[string]string{
 549                         "manifest_text": mtxt,
 550                 },
 551         })
 552         c.Assert(err, check.IsNil)
 553
 554         u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
 555         req := &http.Request{
 556                 Method:     "GET",
 557                 Host:       u.Host,
 558                 URL:        u,
 559                 RequestURI: u.RequestURI(),
 560                 Header: http.Header{
 561                         "Authorization": {"Bearer " + client.AuthToken},
 562                 },
 563         }
 564         resp := httptest.NewRecorder()
 565         s.testServer.Handler.ServeHTTP(resp, req)
 566         c.Check(resp.Code, check.Equals, http.StatusOK)
 567         c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\&#34;odd&#39; path chars.*`)
 568 }
 569
 570 func (s *IntegrationSuite) TestForwardSlashSubstitution(c *check.C) {
 571         arv := arvados.NewClientFromEnv()
 572         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 573         s.testServer.Config.cluster.Collections.ForwardSlashNameSubstitution = "{SOLIDUS}"
 574         name := "foo/bar/baz"
 575         nameShown := strings.Replace(name, "/", "{SOLIDUS}", -1)
 576         nameShownEscaped := strings.Replace(name, "/", "%7bSOLIDUS%7d", -1)
 577
 578         client := s.testServer.Config.Client
 579         client.AuthToken = arvadostest.ActiveToken
 580         fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
 581         c.Assert(err, check.IsNil)
 582         f, err := fs.OpenFile("filename", os.O_CREATE, 0777)
 583         c.Assert(err, check.IsNil)
 584         f.Close()
 585         mtxt, err := fs.MarshalManifest(".")
 586         c.Assert(err, check.IsNil)
 587         var coll arvados.Collection
 588         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 589                 "collection": map[string]string{
 590                         "manifest_text": mtxt,
 591                         "name":          name,
 592                         "owner_uuid":    arvadostest.AProjectUUID,
 593                 },
 594         })
 595         c.Assert(err, check.IsNil)
 596         defer arv.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
 597
 598         base := "http://download.example.com/by_id/" + coll.OwnerUUID + "/"
 599         for tryURL, expectRegexp := range map[string]string{
 600                 base:                          `(?ms).*href="./` + nameShownEscaped + `/"\S+` + nameShown + `.*`,
 601                 base + nameShownEscaped + "/": `(?ms).*href="./filename"\S+filename.*`,
 602         } {
 603                 u, _ := url.Parse(tryURL)
 604                 req := &http.Request{
 605                         Method:     "GET",
 606                         Host:       u.Host,
 607                         URL:        u,
 608                         RequestURI: u.RequestURI(),
 609                         Header: http.Header{
 610                                 "Authorization": {"Bearer " + client.AuthToken},
 611                         },
 612                 }
 613                 resp := httptest.NewRecorder()
 614                 s.testServer.Handler.ServeHTTP(resp, req)
 615                 c.Check(resp.Code, check.Equals, http.StatusOK)
 616                 c.Check(resp.Body.String(), check.Matches, expectRegexp)
 617         }
 618 }
 619
 620 // XHRs can't follow redirect-with-cookie so they rely on method=POST
 621 // and disposition=attachment (telling us it's acceptable to respond
 622 // with content instead of a redirect) and an Origin header that gets
 623 // added automatically by the browser (telling us it's desirable to do
 624 // so).
 625 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
 626         u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
 627         req := &http.Request{
 628                 Method:     "POST",
 629                 Host:       u.Host,
 630                 URL:        u,
 631                 RequestURI: u.RequestURI(),
 632                 Header: http.Header{
 633                         "Origin":       {"https://origin.example"},
 634                         "Content-Type": {"application/x-www-form-urlencoded"},
 635                 },
 636                 Body: ioutil.NopCloser(strings.NewReader(url.Values{
 637                         "api_token":   {arvadostest.ActiveToken},
 638                         "disposition": {"attachment"},
 639                 }.Encode())),
 640         }
 641         resp := httptest.NewRecorder()
 642         s.testServer.Handler.ServeHTTP(resp, req)
 643         c.Check(resp.Code, check.Equals, http.StatusOK)
 644         c.Check(resp.Body.String(), check.Equals, "foo")
 645         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
 646
 647         // GET + Origin header is representative of both AJAX GET
 648         // requests and inline images via <IMG crossorigin="anonymous"
 649         // src="...">.
 650         u.RawQuery = "api_token=" + url.QueryEscape(arvadostest.ActiveTokenV2)
 651         req = &http.Request{
 652                 Method:     "GET",
 653                 Host:       u.Host,
 654                 URL:        u,
 655                 RequestURI: u.RequestURI(),
 656                 Header: http.Header{
 657                         "Origin": {"https://origin.example"},
 658                 },
 659         }
 660         resp = httptest.NewRecorder()
 661         s.testServer.Handler.ServeHTTP(resp, req)
 662         c.Check(resp.Code, check.Equals, http.StatusOK)
 663         c.Check(resp.Body.String(), check.Equals, "foo")
 664         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
 665 }
 666
 667 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder {
 668         u, _ := url.Parse(`http://` + hostPath + queryString)
 669         req := &http.Request{
 670                 Method:     method,
 671                 Host:       u.Host,
 672                 URL:        u,
 673                 RequestURI: u.RequestURI(),
 674                 Header:     http.Header{"Content-Type": {contentType}},
 675                 Body:       ioutil.NopCloser(strings.NewReader(reqBody)),
 676         }
 677
 678         resp := httptest.NewRecorder()
 679         defer func() {
 680                 c.Check(resp.Code, check.Equals, expectStatus)
 681                 c.Check(resp.Body.String(), check.Equals, expectRespBody)
 682         }()
 683
 684         s.testServer.Handler.ServeHTTP(resp, req)
 685         if resp.Code != http.StatusSeeOther {
 686                 return resp
 687         }
 688         c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
 689         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 690
 691         u, _ = u.Parse(resp.Header().Get("Location"))
 692         req = &http.Request{
 693                 Method:     "GET",
 694                 Host:       u.Host,
 695                 URL:        u,
 696                 RequestURI: u.RequestURI(),
 697                 Header:     http.Header{},
 698         }
 699         for _, c := range cookies {
 700                 req.AddCookie(c)
 701         }
 702
 703         resp = httptest.NewRecorder()
 704         s.testServer.Handler.ServeHTTP(resp, req)
 705         c.Check(resp.Header().Get("Location"), check.Equals, "")
 706         return resp
 707 }
 708
 709 func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) {
 710         s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 711         s.testDirectoryListing(c)
 712 }
 713
 714 func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) {
 715         s.testServer.Config.cluster.Users.AnonymousUserToken = ""
 716         s.testDirectoryListing(c)
 717 }
 718
 719 func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
 720         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 721         authHeader := http.Header{
 722                 "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
 723         }
 724         for _, trial := range []struct {
 725                 uri      string
 726                 header   http.Header
 727                 expect   []string
 728                 redirect string
 729                 cutDirs  int
 730         }{
 731                 {
 732                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
 733                         header:  authHeader,
 734                         expect:  []string{"dir1/foo", "dir1/bar"},
 735                         cutDirs: 0,
 736                 },
 737                 {
 738                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
 739                         header:  authHeader,
 740                         expect:  []string{"foo", "bar"},
 741                         cutDirs: 1,
 742                 },
 743                 {
 744                         // URLs of this form ignore authHeader, and
 745                         // FooAndBarFilesInDirUUID isn't public, so
 746                         // this returns 404.
 747                         uri:    "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
 748                         header: authHeader,
 749                         expect: nil,
 750                 },
 751                 {
 752                         uri:     "download.example.com/users/active/foo_file_in_dir/",
 753                         header:  authHeader,
 754                         expect:  []string{"dir1/"},
 755                         cutDirs: 3,
 756                 },
 757                 {
 758                         uri:     "download.example.com/users/active/foo_file_in_dir/dir1/",
 759                         header:  authHeader,
 760                         expect:  []string{"bar"},
 761                         cutDirs: 4,
 762                 },
 763                 {
 764                         uri:     "download.example.com/",
 765                         header:  authHeader,
 766                         expect:  []string{"users/"},
 767                         cutDirs: 0,
 768                 },
 769                 {
 770                         uri:      "download.example.com/users",
 771                         header:   authHeader,
 772                         redirect: "/users/",
 773                         expect:   []string{"active/"},
 774                         cutDirs:  1,
 775                 },
 776                 {
 777                         uri:     "download.example.com/users/",
 778                         header:  authHeader,
 779                         expect:  []string{"active/"},
 780                         cutDirs: 1,
 781                 },
 782                 {
 783                         uri:      "download.example.com/users/active",
 784                         header:   authHeader,
 785                         redirect: "/users/active/",
 786                         expect:   []string{"foo_file_in_dir/"},
 787                         cutDirs:  2,
 788                 },
 789                 {
 790                         uri:     "download.example.com/users/active/",
 791                         header:  authHeader,
 792                         expect:  []string{"foo_file_in_dir/"},
 793                         cutDirs: 2,
 794                 },
 795                 {
 796                         uri:     "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
 797                         header:  nil,
 798                         expect:  []string{"dir1/foo", "dir1/bar"},
 799                         cutDirs: 4,
 800                 },
 801                 {
 802                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
 803                         header:  nil,
 804                         expect:  []string{"dir1/foo", "dir1/bar"},
 805                         cutDirs: 2,
 806                 },
 807                 {
 808                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken,
 809                         header:  nil,
 810                         expect:  []string{"dir1/foo", "dir1/bar"},
 811                         cutDirs: 2,
 812                 },
 813                 {
 814                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID,
 815                         header:  authHeader,
 816                         expect:  []string{"dir1/foo", "dir1/bar"},
 817                         cutDirs: 1,
 818                 },
 819                 {
 820                         uri:      "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
 821                         header:   authHeader,
 822                         redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
 823                         expect:   []string{"foo", "bar"},
 824                         cutDirs:  2,
 825                 },
 826                 {
 827                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
 828                         header:  authHeader,
 829                         expect:  []string{"foo", "bar"},
 830                         cutDirs: 3,
 831                 },
 832                 {
 833                         uri:      arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
 834                         header:   authHeader,
 835                         redirect: "/dir1/",
 836                         expect:   []string{"foo", "bar"},
 837                         cutDirs:  1,
 838                 },
 839                 {
 840                         uri:    "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
 841                         header: authHeader,
 842                         expect: nil,
 843                 },
 844                 {
 845                         uri:     "download.example.com/c=" + arvadostest.WazVersion1Collection,
 846                         header:  authHeader,
 847                         expect:  []string{"waz"},
 848                         cutDirs: 1,
 849                 },
 850                 {
 851                         uri:     "download.example.com/by_id/" + arvadostest.WazVersion1Collection,
 852                         header:  authHeader,
 853                         expect:  []string{"waz"},
 854                         cutDirs: 2,
 855                 },
 856         } {
 857                 comment := check.Commentf("HTML: %q => %q", trial.uri, trial.expect)
 858                 resp := httptest.NewRecorder()
 859                 u := mustParseURL("//" + trial.uri)
 860                 req := &http.Request{
 861                         Method:     "GET",
 862                         Host:       u.Host,
 863                         URL:        u,
 864                         RequestURI: u.RequestURI(),
 865                         Header:     copyHeader(trial.header),
 866                 }
 867                 s.testServer.Handler.ServeHTTP(resp, req)
 868                 var cookies []*http.Cookie
 869                 for resp.Code == http.StatusSeeOther {
 870                         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 871                         req = &http.Request{
 872                                 Method:     "GET",
 873                                 Host:       u.Host,
 874                                 URL:        u,
 875                                 RequestURI: u.RequestURI(),
 876                                 Header:     copyHeader(trial.header),
 877                         }
 878                         cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
 879                         for _, c := range cookies {
 880                                 req.AddCookie(c)
 881                         }
 882                         resp = httptest.NewRecorder()
 883                         s.testServer.Handler.ServeHTTP(resp, req)
 884                 }
 885                 if trial.redirect != "" {
 886                         c.Check(req.URL.Path, check.Equals, trial.redirect, comment)
 887                 }
 888                 if trial.expect == nil {
 889                         c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
 890                 } else {
 891                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
 892                         for _, e := range trial.expect {
 893                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`, comment)
 894                         }
 895                         c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`, comment)
 896                 }
 897
 898                 comment = check.Commentf("WebDAV: %q => %q", trial.uri, trial.expect)
 899                 req = &http.Request{
 900                         Method:     "OPTIONS",
 901                         Host:       u.Host,
 902                         URL:        u,
 903                         RequestURI: u.RequestURI(),
 904                         Header:     copyHeader(trial.header),
 905                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
 906                 }
 907                 resp = httptest.NewRecorder()
 908                 s.testServer.Handler.ServeHTTP(resp, req)
 909                 if trial.expect == nil {
 910                         c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
 911                 } else {
 912                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
 913                 }
 914
 915                 req = &http.Request{
 916                         Method:     "PROPFIND",
 917                         Host:       u.Host,
 918                         URL:        u,
 919                         RequestURI: u.RequestURI(),
 920                         Header:     copyHeader(trial.header),
 921                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
 922                 }
 923                 resp = httptest.NewRecorder()
 924                 s.testServer.Handler.ServeHTTP(resp, req)
 925                 if trial.expect == nil {
 926                         c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
 927                 } else {
 928                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment)
 929                         for _, e := range trial.expect {
 930                                 if strings.HasSuffix(e, "/") {
 931                                         e = filepath.Join(u.Path, e) + "/"
 932                                 } else {
 933                                         e = filepath.Join(u.Path, e)
 934                                 }
 935                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+e+`</D:href>.*`, comment)
 936                         }
 937                 }
 938         }
 939 }
 940
 941 func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
 942         arv := arvados.NewClientFromEnv()
 943         var newCollection arvados.Collection
 944         err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 945                 "collection": map[string]string{
 946                         "owner_uuid":    arvadostest.ActiveUserUUID,
 947                         "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
 948                         "name":          "keep-web test collection",
 949                 },
 950                 "ensure_unique_name": true,
 951         })
 952         c.Assert(err, check.IsNil)
 953         defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
 954
 955         var updated arvados.Collection
 956         for _, fnm := range []string{"foo.txt", "bar.txt"} {
 957                 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
 958                 u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
 959                 req := &http.Request{
 960                         Method:     "DELETE",
 961                         Host:       u.Host,
 962                         URL:        u,
 963                         RequestURI: u.RequestURI(),
 964                         Header: http.Header{
 965                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
 966                         },
 967                 }
 968                 resp := httptest.NewRecorder()
 969                 s.testServer.Handler.ServeHTTP(resp, req)
 970                 c.Check(resp.Code, check.Equals, http.StatusNoContent)
 971
 972                 updated = arvados.Collection{}
 973                 err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
 974                 c.Check(err, check.IsNil)
 975                 c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`)
 976                 c.Logf("updated manifest_text %q", updated.ManifestText)
 977         }
 978         c.Check(updated.ManifestText, check.Equals, "")
 979 }
 980
 981 func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) {
 982         s.testServer.Config.cluster.ManagementToken = arvadostest.ManagementToken
 983         authHeader := http.Header{
 984                 "Authorization": {"Bearer " + arvadostest.ManagementToken},
 985         }
 986
 987         resp := httptest.NewRecorder()
 988         u := mustParseURL("http://download.example.com/_health/ping")
 989         req := &http.Request{
 990                 Method:     "GET",
 991                 Host:       u.Host,
 992                 URL:        u,
 993                 RequestURI: u.RequestURI(),
 994                 Header:     authHeader,
 995         }
 996         s.testServer.Handler.ServeHTTP(resp, req)
 997
 998         c.Check(resp.Code, check.Equals, http.StatusOK)
 999         c.Check(resp.Body.String(), check.Matches, `{"health":"OK"}\n`)
1000 }
1001
1002 func (s *IntegrationSuite) TestFileContentType(c *check.C) {
1003         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1004
1005         client := s.testServer.Config.Client
1006         client.AuthToken = arvadostest.ActiveToken
1007         arv, err := arvadosclient.New(&client)
1008         c.Assert(err, check.Equals, nil)
1009         kc, err := keepclient.MakeKeepClient(arv)
1010         c.Assert(err, check.Equals, nil)
1011
1012         fs, err := (&arvados.Collection{}).FileSystem(&client, kc)
1013         c.Assert(err, check.IsNil)
1014
1015         trials := []struct {
1016                 filename    string
1017                 content     string
1018                 contentType string
1019         }{
1020                 {"picture.txt", "BMX bikes are small this year\n", "text/plain; charset=utf-8"},
1021                 {"picture.bmp", "BMX bikes are small this year\n", "image/x-ms-bmp"},
1022                 {"picture.jpg", "BMX bikes are small this year\n", "image/jpeg"},
1023                 {"picture1", "BMX bikes are small this year\n", "image/bmp"},            // content sniff; "BM" is the magic signature for .bmp
1024                 {"picture2", "Cars are small this year\n", "text/plain; charset=utf-8"}, // content sniff
1025         }
1026         for _, trial := range trials {
1027                 f, err := fs.OpenFile(trial.filename, os.O_CREATE|os.O_WRONLY, 0777)
1028                 c.Assert(err, check.IsNil)
1029                 _, err = f.Write([]byte(trial.content))
1030                 c.Assert(err, check.IsNil)
1031                 c.Assert(f.Close(), check.IsNil)
1032         }
1033         mtxt, err := fs.MarshalManifest(".")
1034         c.Assert(err, check.IsNil)
1035         var coll arvados.Collection
1036         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1037                 "collection": map[string]string{
1038                         "manifest_text": mtxt,
1039                 },
1040         })
1041         c.Assert(err, check.IsNil)
1042
1043         for _, trial := range trials {
1044                 u, _ := url.Parse("http://download.example.com/by_id/" + coll.UUID + "/" + trial.filename)
1045                 req := &http.Request{
1046                         Method:     "GET",
1047                         Host:       u.Host,
1048                         URL:        u,
1049                         RequestURI: u.RequestURI(),
1050                         Header: http.Header{
1051                                 "Authorization": {"Bearer " + client.AuthToken},
1052                         },
1053                 }
1054                 resp := httptest.NewRecorder()
1055                 s.testServer.Handler.ServeHTTP(resp, req)
1056                 c.Check(resp.Code, check.Equals, http.StatusOK)
1057                 c.Check(resp.Header().Get("Content-Type"), check.Equals, trial.contentType)
1058                 c.Check(resp.Body.String(), check.Equals, trial.content)
1059         }
1060 }
1061
1062 func (s *IntegrationSuite) TestKeepClientBlockCache(c *check.C) {
1063         s.testServer.Config.cluster.Collections.WebDAVCache.MaxBlockEntries = 42
1064         c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Not(check.Equals), 42)
1065         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/t=" + arvadostest.ActiveToken + "/foo")
1066         req := &http.Request{
1067                 Method:     "GET",
1068                 Host:       u.Host,
1069                 URL:        u,
1070                 RequestURI: u.RequestURI(),
1071         }
1072         resp := httptest.NewRecorder()
1073         s.testServer.Handler.ServeHTTP(resp, req)
1074         c.Check(resp.Code, check.Equals, http.StatusOK)
1075         c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Equals, 42)
1076 }
1077
1078 func copyHeader(h http.Header) http.Header {
1079         hc := http.Header{}
1080         for k, v := range h {
1081                 hc[k] = append([]string(nil), v...)
1082         }
1083         return hc
1084 }