services/keep-web/handler_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package keepweb
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "fmt"
  11         "html"
  12         "io"
  13         "io/ioutil"
  14         "net/http"
  15         "net/http/httptest"
  16         "net/url"
  17         "os"
  18         "path/filepath"
  19         "regexp"
  20         "strings"
  21         "time"
  22
  23         "git.arvados.org/arvados.git/lib/config"
  24         "git.arvados.org/arvados.git/sdk/go/arvados"
  25         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  26         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  27         "git.arvados.org/arvados.git/sdk/go/auth"
  28         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  29         "git.arvados.org/arvados.git/sdk/go/keepclient"
  30         "github.com/prometheus/client_golang/prometheus"
  31         "github.com/sirupsen/logrus"
  32         check "gopkg.in/check.v1"
  33 )
  34
  35 var _ = check.Suite(&UnitSuite{})
  36
  37 func init() {
  38         arvados.DebugLocksPanicMode = true
  39 }
  40
  41 type UnitSuite struct {
  42         cluster *arvados.Cluster
  43         handler *handler
  44 }
  45
  46 func (s *UnitSuite) SetUpTest(c *check.C) {
  47         logger := ctxlog.TestLogger(c)
  48         ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), logger)
  49         ldr.Path = "-"
  50         cfg, err := ldr.Load()
  51         c.Assert(err, check.IsNil)
  52         cc, err := cfg.GetCluster("")
  53         c.Assert(err, check.IsNil)
  54         s.cluster = cc
  55         s.handler = &handler{
  56                 Cluster: cc,
  57                 Cache: cache{
  58                         cluster:  cc,
  59                         logger:   logger,
  60                         registry: prometheus.NewRegistry(),
  61                 },
  62         }
  63 }
  64
  65 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
  66         h := s.handler
  67         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
  68         req := &http.Request{
  69                 Method:     "OPTIONS",
  70                 Host:       u.Host,
  71                 URL:        u,
  72                 RequestURI: u.RequestURI(),
  73                 Header: http.Header{
  74                         "Origin":                        {"https://workbench.example"},
  75                         "Access-Control-Request-Method": {"POST"},
  76                 },
  77         }
  78
  79         // Check preflight for an allowed request
  80         resp := httptest.NewRecorder()
  81         h.ServeHTTP(resp, req)
  82         c.Check(resp.Code, check.Equals, http.StatusOK)
  83         c.Check(resp.Body.String(), check.Equals, "")
  84         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
  85         c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
  86         c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout, Cache-Control")
  87
  88         // Check preflight for a disallowed request
  89         resp = httptest.NewRecorder()
  90         req.Header.Set("Access-Control-Request-Method", "MAKE-COFFEE")
  91         h.ServeHTTP(resp, req)
  92         c.Check(resp.Body.String(), check.Equals, "")
  93         c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
  94 }
  95
  96 func (s *UnitSuite) TestEmptyResponse(c *check.C) {
  97         for _, trial := range []struct {
  98                 dataExists    bool
  99                 sendIMSHeader bool
 100                 expectStatus  int
 101                 logRegexp     string
 102         }{
 103                 // If we return no content due to a Keep read error,
 104                 // we should emit a log message.
 105                 {false, false, http.StatusOK, `(?ms).*only wrote 0 bytes.*`},
 106
 107                 // If we return no content because the client sent an
 108                 // If-Modified-Since header, our response should be
 109                 // 304.  We still expect a "File download" log since it
 110                 // counts as a file access for auditing.
 111                 {true, true, http.StatusNotModified, `(?ms).*msg="File download".*`},
 112         } {
 113                 c.Logf("trial: %+v", trial)
 114                 arvadostest.StartKeep(2, true)
 115                 if trial.dataExists {
 116                         arv, err := arvadosclient.MakeArvadosClient()
 117                         c.Assert(err, check.IsNil)
 118                         arv.ApiToken = arvadostest.ActiveToken
 119                         kc, err := keepclient.MakeKeepClient(arv)
 120                         c.Assert(err, check.IsNil)
 121                         _, _, err = kc.PutB([]byte("foo"))
 122                         c.Assert(err, check.IsNil)
 123                 }
 124
 125                 u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
 126                 req := &http.Request{
 127                         Method:     "GET",
 128                         Host:       u.Host,
 129                         URL:        u,
 130                         RequestURI: u.RequestURI(),
 131                         Header: http.Header{
 132                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
 133                         },
 134                 }
 135                 if trial.sendIMSHeader {
 136                         req.Header.Set("If-Modified-Since", strings.Replace(time.Now().UTC().Format(time.RFC1123), "UTC", "GMT", -1))
 137                 }
 138
 139                 var logbuf bytes.Buffer
 140                 logger := logrus.New()
 141                 logger.Out = &logbuf
 142                 req = req.WithContext(ctxlog.Context(context.Background(), logger))
 143
 144                 resp := httptest.NewRecorder()
 145                 s.handler.ServeHTTP(resp, req)
 146                 c.Check(resp.Code, check.Equals, trial.expectStatus)
 147                 c.Check(resp.Body.String(), check.Equals, "")
 148
 149                 c.Log(logbuf.String())
 150                 c.Check(logbuf.String(), check.Matches, trial.logRegexp)
 151         }
 152 }
 153
 154 func (s *UnitSuite) TestInvalidUUID(c *check.C) {
 155         bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-"
 156         token := arvadostest.ActiveToken
 157         for _, trial := range []string{
 158                 "http://keep-web/c=" + bogusID + "/foo",
 159                 "http://keep-web/c=" + bogusID + "/t=" + token + "/foo",
 160                 "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo",
 161                 "http://keep-web/collections/" + bogusID + "/foo",
 162                 "http://" + bogusID + ".keep-web/" + bogusID + "/foo",
 163                 "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
 164         } {
 165                 c.Log(trial)
 166                 u := mustParseURL(trial)
 167                 req := &http.Request{
 168                         Method:     "GET",
 169                         Host:       u.Host,
 170                         URL:        u,
 171                         RequestURI: u.RequestURI(),
 172                 }
 173                 resp := httptest.NewRecorder()
 174                 s.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 175                 s.handler.ServeHTTP(resp, req)
 176                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 177         }
 178 }
 179
 180 func mustParseURL(s string) *url.URL {
 181         r, err := url.Parse(s)
 182         if err != nil {
 183                 panic("parse URL: " + s)
 184         }
 185         return r
 186 }
 187
 188 func (s *IntegrationSuite) TestVhost404(c *check.C) {
 189         for _, testURL := range []string{
 190                 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
 191                 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
 192         } {
 193                 resp := httptest.NewRecorder()
 194                 u := mustParseURL(testURL)
 195                 req := &http.Request{
 196                         Method:     "GET",
 197                         URL:        u,
 198                         RequestURI: u.RequestURI(),
 199                 }
 200                 s.handler.ServeHTTP(resp, req)
 201                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 202                 c.Check(resp.Body.String(), check.Equals, notFoundMessage+"\n")
 203         }
 204 }
 205
 206 // An authorizer modifies an HTTP request to make use of the given
 207 // token -- by adding it to a header, cookie, query param, or whatever
 208 // -- and returns the HTTP status code we should expect from keep-web if
 209 // the token is invalid.
 210 type authorizer func(*http.Request, string) int
 211
 212 func (s *IntegrationSuite) TestVhostViaAuthzHeaderOAuth2(c *check.C) {
 213         s.doVhostRequests(c, authzViaAuthzHeaderOAuth2)
 214 }
 215 func authzViaAuthzHeaderOAuth2(r *http.Request, tok string) int {
 216         r.Header.Add("Authorization", "Bearer "+tok)
 217         return http.StatusUnauthorized
 218 }
 219 func (s *IntegrationSuite) TestVhostViaAuthzHeaderBearer(c *check.C) {
 220         s.doVhostRequests(c, authzViaAuthzHeaderBearer)
 221 }
 222 func authzViaAuthzHeaderBearer(r *http.Request, tok string) int {
 223         r.Header.Add("Authorization", "Bearer "+tok)
 224         return http.StatusUnauthorized
 225 }
 226
 227 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
 228         s.doVhostRequests(c, authzViaCookieValue)
 229 }
 230 func authzViaCookieValue(r *http.Request, tok string) int {
 231         r.AddCookie(&http.Cookie{
 232                 Name:  "arvados_api_token",
 233                 Value: auth.EncodeTokenCookie([]byte(tok)),
 234         })
 235         return http.StatusUnauthorized
 236 }
 237
 238 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
 239         s.doVhostRequests(c, authzViaPath)
 240 }
 241 func authzViaPath(r *http.Request, tok string) int {
 242         r.URL.Path = "/t=" + tok + r.URL.Path
 243         return http.StatusNotFound
 244 }
 245
 246 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
 247         s.doVhostRequests(c, authzViaQueryString)
 248 }
 249 func authzViaQueryString(r *http.Request, tok string) int {
 250         r.URL.RawQuery = "api_token=" + tok
 251         return http.StatusUnauthorized
 252 }
 253
 254 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
 255         s.doVhostRequests(c, authzViaPOST)
 256 }
 257 func authzViaPOST(r *http.Request, tok string) int {
 258         r.Method = "POST"
 259         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 260         r.Body = ioutil.NopCloser(strings.NewReader(
 261                 url.Values{"api_token": {tok}}.Encode()))
 262         return http.StatusUnauthorized
 263 }
 264
 265 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
 266         s.doVhostRequests(c, authzViaPOST)
 267 }
 268 func authzViaXHRPOST(r *http.Request, tok string) int {
 269         r.Method = "POST"
 270         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 271         r.Header.Add("Origin", "https://origin.example")
 272         r.Body = ioutil.NopCloser(strings.NewReader(
 273                 url.Values{
 274                         "api_token":   {tok},
 275                         "disposition": {"attachment"},
 276                 }.Encode()))
 277         return http.StatusUnauthorized
 278 }
 279
 280 // Try some combinations of {url, token} using the given authorization
 281 // mechanism, and verify the result is correct.
 282 func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
 283         for _, hostPath := range []string{
 284                 arvadostest.FooCollection + ".example.com/foo",
 285                 arvadostest.FooCollection + "--collections.example.com/foo",
 286                 arvadostest.FooCollection + "--collections.example.com/_/foo",
 287                 arvadostest.FooCollectionPDH + ".example.com/foo",
 288                 strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + "--collections.example.com/foo",
 289                 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
 290         } {
 291                 c.Log("doRequests: ", hostPath)
 292                 s.doVhostRequestsWithHostPath(c, authz, hostPath)
 293         }
 294 }
 295
 296 func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
 297         for _, tok := range []string{
 298                 arvadostest.ActiveToken,
 299                 arvadostest.ActiveToken[:15],
 300                 arvadostest.SpectatorToken,
 301                 "bogus",
 302                 "",
 303         } {
 304                 u := mustParseURL("http://" + hostPath)
 305                 req := &http.Request{
 306                         Method:     "GET",
 307                         Host:       u.Host,
 308                         URL:        u,
 309                         RequestURI: u.RequestURI(),
 310                         Header:     http.Header{},
 311                 }
 312                 failCode := authz(req, tok)
 313                 req, resp := s.doReq(req)
 314                 code, body := resp.Code, resp.Body.String()
 315
 316                 // If the initial request had a (non-empty) token
 317                 // showing in the query string, we should have been
 318                 // redirected in order to hide it in a cookie.
 319                 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
 320
 321                 if tok == arvadostest.ActiveToken {
 322                         c.Check(code, check.Equals, http.StatusOK)
 323                         c.Check(body, check.Equals, "foo")
 324                 } else {
 325                         c.Check(code >= 400, check.Equals, true)
 326                         c.Check(code < 500, check.Equals, true)
 327                         if tok == arvadostest.SpectatorToken {
 328                                 // Valid token never offers to retry
 329                                 // with different credentials.
 330                                 c.Check(code, check.Equals, http.StatusNotFound)
 331                         } else {
 332                                 // Invalid token can ask to retry
 333                                 // depending on the authz method.
 334                                 c.Check(code, check.Equals, failCode)
 335                         }
 336                         if code == 404 {
 337                                 c.Check(body, check.Equals, notFoundMessage+"\n")
 338                         } else {
 339                                 c.Check(body, check.Equals, unauthorizedMessage+"\n")
 340                         }
 341                 }
 342         }
 343 }
 344
 345 func (s *IntegrationSuite) TestVhostPortMatch(c *check.C) {
 346         for _, host := range []string{"download.example.com", "DOWNLOAD.EXAMPLE.COM"} {
 347                 for _, port := range []string{"80", "443", "8000"} {
 348                         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = fmt.Sprintf("download.example.com:%v", port)
 349                         u := mustParseURL(fmt.Sprintf("http://%v/by_id/%v/foo", host, arvadostest.FooCollection))
 350                         req := &http.Request{
 351                                 Method:     "GET",
 352                                 Host:       u.Host,
 353                                 URL:        u,
 354                                 RequestURI: u.RequestURI(),
 355                                 Header:     http.Header{"Authorization": []string{"Bearer " + arvadostest.ActiveToken}},
 356                         }
 357                         req, resp := s.doReq(req)
 358                         code, _ := resp.Code, resp.Body.String()
 359
 360                         if port == "8000" {
 361                                 c.Check(code, check.Equals, 401)
 362                         } else {
 363                                 c.Check(code, check.Equals, 200)
 364                         }
 365                 }
 366         }
 367 }
 368
 369 func (s *IntegrationSuite) do(method string, urlstring string, token string, hdr http.Header) (*http.Request, *httptest.ResponseRecorder) {
 370         u := mustParseURL(urlstring)
 371         if hdr == nil && token != "" {
 372                 hdr = http.Header{"Authorization": {"Bearer " + token}}
 373         } else if hdr == nil {
 374                 hdr = http.Header{}
 375         } else if token != "" {
 376                 panic("must not pass both token and hdr")
 377         }
 378         return s.doReq(&http.Request{
 379                 Method:     method,
 380                 Host:       u.Host,
 381                 URL:        u,
 382                 RequestURI: u.RequestURI(),
 383                 Header:     hdr,
 384         })
 385 }
 386
 387 func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
 388         resp := httptest.NewRecorder()
 389         s.handler.ServeHTTP(resp, req)
 390         if resp.Code != http.StatusSeeOther {
 391                 return req, resp
 392         }
 393         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 394         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 395         req = &http.Request{
 396                 Method:     "GET",
 397                 Host:       u.Host,
 398                 URL:        u,
 399                 RequestURI: u.RequestURI(),
 400                 Header:     http.Header{},
 401         }
 402         for _, c := range cookies {
 403                 req.AddCookie(c)
 404         }
 405         return s.doReq(req)
 406 }
 407
 408 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
 409         s.testVhostRedirectTokenToCookie(c, "GET",
 410                 arvadostest.FooCollection+".example.com/foo",
 411                 "?api_token="+arvadostest.ActiveToken,
 412                 nil,
 413                 "",
 414                 http.StatusOK,
 415                 "foo",
 416         )
 417 }
 418
 419 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
 420         s.testVhostRedirectTokenToCookie(c, "GET",
 421                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
 422                 "",
 423                 nil,
 424                 "",
 425                 http.StatusOK,
 426                 "foo",
 427         )
 428 }
 429
 430 func (s *IntegrationSuite) TestCollectionSharingToken(c *check.C) {
 431         s.testVhostRedirectTokenToCookie(c, "GET",
 432                 "example.com/c="+arvadostest.FooFileCollectionUUID+"/t="+arvadostest.FooFileCollectionSharingToken+"/foo",
 433                 "",
 434                 nil,
 435                 "",
 436                 http.StatusOK,
 437                 "foo",
 438         )
 439         // Same valid sharing token, but requesting a different collection
 440         s.testVhostRedirectTokenToCookie(c, "GET",
 441                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.FooFileCollectionSharingToken+"/foo",
 442                 "",
 443                 nil,
 444                 "",
 445                 http.StatusNotFound,
 446                 regexp.QuoteMeta(notFoundMessage+"\n"),
 447         )
 448 }
 449
 450 // Bad token in URL is 404 Not Found because it doesn't make sense to
 451 // retry the same URL with different authorization.
 452 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
 453         s.testVhostRedirectTokenToCookie(c, "GET",
 454                 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
 455                 "",
 456                 nil,
 457                 "",
 458                 http.StatusNotFound,
 459                 regexp.QuoteMeta(notFoundMessage+"\n"),
 460         )
 461 }
 462
 463 // Bad token in a cookie (even if it got there via our own
 464 // query-string-to-cookie redirect) is, in principle, retryable via
 465 // wb2-login-and-redirect flow.
 466 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
 467         // Inline
 468         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 469                 arvadostest.FooCollection+".example.com/foo",
 470                 "?api_token=thisisabogustoken",
 471                 http.Header{"Sec-Fetch-Mode": {"navigate"}},
 472                 "",
 473                 http.StatusSeeOther,
 474                 "",
 475         )
 476         u, err := url.Parse(resp.Header().Get("Location"))
 477         c.Assert(err, check.IsNil)
 478         c.Logf("redirected to %s", u)
 479         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 480         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 481         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
 482
 483         // Download/attachment indicated by ?disposition=attachment
 484         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 485                 arvadostest.FooCollection+".example.com/foo",
 486                 "?api_token=thisisabogustoken&disposition=attachment",
 487                 http.Header{"Sec-Fetch-Mode": {"navigate"}},
 488                 "",
 489                 http.StatusSeeOther,
 490                 "",
 491         )
 492         u, err = url.Parse(resp.Header().Get("Location"))
 493         c.Assert(err, check.IsNil)
 494         c.Logf("redirected to %s", u)
 495         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 496         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "")
 497         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 498
 499         // Download/attachment indicated by vhost
 500         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 501                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
 502                 "?api_token=thisisabogustoken",
 503                 http.Header{"Sec-Fetch-Mode": {"navigate"}},
 504                 "",
 505                 http.StatusSeeOther,
 506                 "",
 507         )
 508         u, err = url.Parse(resp.Header().Get("Location"))
 509         c.Assert(err, check.IsNil)
 510         c.Logf("redirected to %s", u)
 511         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 512         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "")
 513         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 514
 515         // Without "Sec-Fetch-Mode: navigate" header, just 401.
 516         s.testVhostRedirectTokenToCookie(c, "GET",
 517                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
 518                 "?api_token=thisisabogustoken",
 519                 http.Header{"Sec-Fetch-Mode": {"cors"}},
 520                 "",
 521                 http.StatusUnauthorized,
 522                 regexp.QuoteMeta(unauthorizedMessage+"\n"),
 523         )
 524         s.testVhostRedirectTokenToCookie(c, "GET",
 525                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
 526                 "?api_token=thisisabogustoken",
 527                 nil,
 528                 "",
 529                 http.StatusUnauthorized,
 530                 regexp.QuoteMeta(unauthorizedMessage+"\n"),
 531         )
 532 }
 533
 534 func (s *IntegrationSuite) TestVhostRedirectWithNoCache(c *check.C) {
 535         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 536                 arvadostest.FooCollection+".example.com/foo",
 537                 "?api_token=thisisabogustoken",
 538                 http.Header{
 539                         "Sec-Fetch-Mode": {"navigate"},
 540                         "Cache-Control":  {"no-cache"},
 541                 },
 542                 "",
 543                 http.StatusSeeOther,
 544                 "",
 545         )
 546         u, err := url.Parse(resp.Header().Get("Location"))
 547         c.Assert(err, check.IsNil)
 548         c.Logf("redirected to %s", u)
 549         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 550         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 551         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
 552 }
 553
 554 func (s *IntegrationSuite) TestNoTokenWorkbench2LoginFlow(c *check.C) {
 555         for _, trial := range []struct {
 556                 anonToken    bool
 557                 cacheControl string
 558         }{
 559                 {},
 560                 {cacheControl: "no-cache"},
 561                 {anonToken: true},
 562                 {anonToken: true, cacheControl: "no-cache"},
 563         } {
 564                 c.Logf("trial: %+v", trial)
 565
 566                 if trial.anonToken {
 567                         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 568                 } else {
 569                         s.handler.Cluster.Users.AnonymousUserToken = ""
 570                 }
 571                 req, err := http.NewRequest("GET", "http://"+arvadostest.FooCollection+".example.com/foo", nil)
 572                 c.Assert(err, check.IsNil)
 573                 req.Header.Set("Sec-Fetch-Mode", "navigate")
 574                 if trial.cacheControl != "" {
 575                         req.Header.Set("Cache-Control", trial.cacheControl)
 576                 }
 577                 resp := httptest.NewRecorder()
 578                 s.handler.ServeHTTP(resp, req)
 579                 c.Check(resp.Code, check.Equals, http.StatusSeeOther)
 580                 u, err := url.Parse(resp.Header().Get("Location"))
 581                 c.Assert(err, check.IsNil)
 582                 c.Logf("redirected to %q", u)
 583                 c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 584                 c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 585                 c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
 586         }
 587 }
 588
 589 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
 590         s.testVhostRedirectTokenToCookie(c, "GET",
 591                 "example.com/c="+arvadostest.FooCollection+"/foo",
 592                 "?api_token="+arvadostest.ActiveToken,
 593                 nil,
 594                 "",
 595                 http.StatusBadRequest,
 596                 regexp.QuoteMeta("cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n"),
 597         )
 598 }
 599
 600 // If client requests an attachment by putting ?disposition=attachment
 601 // in the query string, and gets redirected, the redirect target
 602 // should respond with an attachment.
 603 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
 604         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 605                 arvadostest.FooCollection+".example.com/foo",
 606                 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
 607                 nil,
 608                 "",
 609                 http.StatusOK,
 610                 "foo",
 611         )
 612         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 613 }
 614
 615 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
 616         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 617         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 618                 "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
 619                 "?api_token="+arvadostest.ActiveToken,
 620                 nil,
 621                 "",
 622                 http.StatusOK,
 623                 "foo",
 624         )
 625         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 626 }
 627
 628 func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
 629         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 630         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 631                 "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
 632                 "?api_token="+arvadostest.ActiveToken,
 633                 nil,
 634                 "",
 635                 http.StatusOK,
 636                 "waz",
 637         )
 638         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 639         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 640                 "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz",
 641                 "?api_token="+arvadostest.ActiveToken,
 642                 nil,
 643                 "",
 644                 http.StatusOK,
 645                 "waz",
 646         )
 647         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 648 }
 649
 650 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
 651         s.handler.Cluster.Collections.TrustAllContent = true
 652         s.testVhostRedirectTokenToCookie(c, "GET",
 653                 "example.com/c="+arvadostest.FooCollection+"/foo",
 654                 "?api_token="+arvadostest.ActiveToken,
 655                 nil,
 656                 "",
 657                 http.StatusOK,
 658                 "foo",
 659         )
 660 }
 661
 662 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
 663         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
 664
 665         s.testVhostRedirectTokenToCookie(c, "GET",
 666                 "example.com/c="+arvadostest.FooCollection+"/foo",
 667                 "?api_token="+arvadostest.ActiveToken,
 668                 nil,
 669                 "",
 670                 http.StatusBadRequest,
 671                 regexp.QuoteMeta("cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n"),
 672         )
 673
 674         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 675                 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
 676                 "?api_token="+arvadostest.ActiveToken,
 677                 nil,
 678                 "",
 679                 http.StatusOK,
 680                 "foo",
 681         )
 682         c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
 683 }
 684
 685 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
 686         s.testVhostRedirectTokenToCookie(c, "POST",
 687                 arvadostest.FooCollection+".example.com/foo",
 688                 "",
 689                 http.Header{"Content-Type": {"application/x-www-form-urlencoded"}},
 690                 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
 691                 http.StatusOK,
 692                 "foo",
 693         )
 694 }
 695
 696 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
 697         s.testVhostRedirectTokenToCookie(c, "POST",
 698                 arvadostest.FooCollection+".example.com/foo",
 699                 "",
 700                 http.Header{"Content-Type": {"application/x-www-form-urlencoded"}},
 701                 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
 702                 http.StatusNotFound,
 703                 regexp.QuoteMeta(notFoundMessage+"\n"),
 704         )
 705 }
 706
 707 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
 708         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 709         s.testVhostRedirectTokenToCookie(c, "GET",
 710                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 711                 "",
 712                 nil,
 713                 "",
 714                 http.StatusOK,
 715                 "Hello world\n",
 716         )
 717 }
 718
 719 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
 720         s.handler.Cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
 721         s.testVhostRedirectTokenToCookie(c, "GET",
 722                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 723                 "",
 724                 nil,
 725                 "",
 726                 http.StatusUnauthorized,
 727                 "Authorization tokens are not accepted here: .*\n",
 728         )
 729 }
 730
 731 func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
 732         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 733
 734         client := arvados.NewClientFromEnv()
 735         client.AuthToken = arvadostest.ActiveToken
 736         fs, err := (&arvados.Collection{}).FileSystem(client, nil)
 737         c.Assert(err, check.IsNil)
 738         f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777)
 739         c.Assert(err, check.IsNil)
 740         f.Close()
 741         mtxt, err := fs.MarshalManifest(".")
 742         c.Assert(err, check.IsNil)
 743         var coll arvados.Collection
 744         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 745                 "collection": map[string]string{
 746                         "manifest_text": mtxt,
 747                 },
 748         })
 749         c.Assert(err, check.IsNil)
 750
 751         u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
 752         req := &http.Request{
 753                 Method:     "GET",
 754                 Host:       u.Host,
 755                 URL:        u,
 756                 RequestURI: u.RequestURI(),
 757                 Header: http.Header{
 758                         "Authorization": {"Bearer " + client.AuthToken},
 759                 },
 760         }
 761         resp := httptest.NewRecorder()
 762         s.handler.ServeHTTP(resp, req)
 763         c.Check(resp.Code, check.Equals, http.StatusOK)
 764         c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\&#34;odd&#39; path chars.*`)
 765 }
 766
 767 func (s *IntegrationSuite) TestForwardSlashSubstitution(c *check.C) {
 768         arv := arvados.NewClientFromEnv()
 769         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 770         s.handler.Cluster.Collections.ForwardSlashNameSubstitution = "{SOLIDUS}"
 771         name := "foo/bar/baz"
 772         nameShown := strings.Replace(name, "/", "{SOLIDUS}", -1)
 773         nameShownEscaped := strings.Replace(name, "/", "%7bSOLIDUS%7d", -1)
 774
 775         client := arvados.NewClientFromEnv()
 776         client.AuthToken = arvadostest.ActiveToken
 777         fs, err := (&arvados.Collection{}).FileSystem(client, nil)
 778         c.Assert(err, check.IsNil)
 779         f, err := fs.OpenFile("filename", os.O_CREATE, 0777)
 780         c.Assert(err, check.IsNil)
 781         f.Close()
 782         mtxt, err := fs.MarshalManifest(".")
 783         c.Assert(err, check.IsNil)
 784         var coll arvados.Collection
 785         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 786                 "collection": map[string]string{
 787                         "manifest_text": mtxt,
 788                         "name":          name,
 789                         "owner_uuid":    arvadostest.AProjectUUID,
 790                 },
 791         })
 792         c.Assert(err, check.IsNil)
 793         defer arv.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
 794
 795         base := "http://download.example.com/by_id/" + coll.OwnerUUID + "/"
 796         for tryURL, expectRegexp := range map[string]string{
 797                 base:                          `(?ms).*href="./` + nameShownEscaped + `/"\S+` + nameShown + `.*`,
 798                 base + nameShownEscaped + "/": `(?ms).*href="./filename"\S+filename.*`,
 799         } {
 800                 u, _ := url.Parse(tryURL)
 801                 req := &http.Request{
 802                         Method:     "GET",
 803                         Host:       u.Host,
 804                         URL:        u,
 805                         RequestURI: u.RequestURI(),
 806                         Header: http.Header{
 807                                 "Authorization": {"Bearer " + client.AuthToken},
 808                         },
 809                 }
 810                 resp := httptest.NewRecorder()
 811                 s.handler.ServeHTTP(resp, req)
 812                 c.Check(resp.Code, check.Equals, http.StatusOK)
 813                 c.Check(resp.Body.String(), check.Matches, expectRegexp)
 814         }
 815 }
 816
 817 // XHRs can't follow redirect-with-cookie so they rely on method=POST
 818 // and disposition=attachment (telling us it's acceptable to respond
 819 // with content instead of a redirect) and an Origin header that gets
 820 // added automatically by the browser (telling us it's desirable to do
 821 // so).
 822 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
 823         u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
 824         req := &http.Request{
 825                 Method:     "POST",
 826                 Host:       u.Host,
 827                 URL:        u,
 828                 RequestURI: u.RequestURI(),
 829                 Header: http.Header{
 830                         "Origin":       {"https://origin.example"},
 831                         "Content-Type": {"application/x-www-form-urlencoded"},
 832                 },
 833                 Body: ioutil.NopCloser(strings.NewReader(url.Values{
 834                         "api_token":   {arvadostest.ActiveToken},
 835                         "disposition": {"attachment"},
 836                 }.Encode())),
 837         }
 838         resp := httptest.NewRecorder()
 839         s.handler.ServeHTTP(resp, req)
 840         c.Check(resp.Code, check.Equals, http.StatusOK)
 841         c.Check(resp.Body.String(), check.Equals, "foo")
 842         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
 843
 844         // GET + Origin header is representative of both AJAX GET
 845         // requests and inline images via <IMG crossorigin="anonymous"
 846         // src="...">.
 847         u.RawQuery = "api_token=" + url.QueryEscape(arvadostest.ActiveTokenV2)
 848         req = &http.Request{
 849                 Method:     "GET",
 850                 Host:       u.Host,
 851                 URL:        u,
 852                 RequestURI: u.RequestURI(),
 853                 Header: http.Header{
 854                         "Origin": {"https://origin.example"},
 855                 },
 856         }
 857         resp = httptest.NewRecorder()
 858         s.handler.ServeHTTP(resp, req)
 859         c.Check(resp.Code, check.Equals, http.StatusOK)
 860         c.Check(resp.Body.String(), check.Equals, "foo")
 861         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
 862 }
 863
 864 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString string, reqHeader http.Header, reqBody string, expectStatus int, matchRespBody string) *httptest.ResponseRecorder {
 865         if reqHeader == nil {
 866                 reqHeader = http.Header{}
 867         }
 868         u, _ := url.Parse(`http://` + hostPath + queryString)
 869         c.Logf("requesting %s", u)
 870         req := &http.Request{
 871                 Method:     method,
 872                 Host:       u.Host,
 873                 URL:        u,
 874                 RequestURI: u.RequestURI(),
 875                 Header:     reqHeader,
 876                 Body:       ioutil.NopCloser(strings.NewReader(reqBody)),
 877         }
 878
 879         resp := httptest.NewRecorder()
 880         defer func() {
 881                 c.Check(resp.Code, check.Equals, expectStatus)
 882                 c.Check(resp.Body.String(), check.Matches, matchRespBody)
 883         }()
 884
 885         s.handler.ServeHTTP(resp, req)
 886         if resp.Code != http.StatusSeeOther {
 887                 return resp
 888         }
 889         c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
 890         c.Check(strings.Split(resp.Header().Get("Location"), "?")[0], check.Equals, "http://"+hostPath)
 891         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 892
 893         u, err := u.Parse(resp.Header().Get("Location"))
 894         c.Assert(err, check.IsNil)
 895         c.Logf("following redirect to %s", u)
 896         req = &http.Request{
 897                 Method:     "GET",
 898                 Host:       u.Host,
 899                 URL:        u,
 900                 RequestURI: u.RequestURI(),
 901                 Header:     reqHeader,
 902         }
 903         for _, c := range cookies {
 904                 req.AddCookie(c)
 905         }
 906
 907         resp = httptest.NewRecorder()
 908         s.handler.ServeHTTP(resp, req)
 909
 910         if resp.Code != http.StatusSeeOther {
 911                 c.Check(resp.Header().Get("Location"), check.Equals, "")
 912         }
 913         return resp
 914 }
 915
 916 func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) {
 917         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 918         s.testDirectoryListing(c)
 919 }
 920
 921 func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) {
 922         s.handler.Cluster.Users.AnonymousUserToken = ""
 923         s.testDirectoryListing(c)
 924 }
 925
 926 func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
 927         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 928         authHeader := http.Header{
 929                 "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
 930         }
 931         for _, trial := range []struct {
 932                 uri      string
 933                 header   http.Header
 934                 expect   []string
 935                 redirect string
 936                 cutDirs  int
 937         }{
 938                 {
 939                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
 940                         header:  authHeader,
 941                         expect:  []string{"dir1/foo", "dir1/bar"},
 942                         cutDirs: 0,
 943                 },
 944                 {
 945                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
 946                         header:  authHeader,
 947                         expect:  []string{"foo", "bar"},
 948                         cutDirs: 1,
 949                 },
 950                 {
 951                         // URLs of this form ignore authHeader, and
 952                         // FooAndBarFilesInDirUUID isn't public, so
 953                         // this returns 401.
 954                         uri:    "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
 955                         header: authHeader,
 956                         expect: nil,
 957                 },
 958                 {
 959                         uri:     "download.example.com/users/active/foo_file_in_dir/",
 960                         header:  authHeader,
 961                         expect:  []string{"dir1/"},
 962                         cutDirs: 3,
 963                 },
 964                 {
 965                         uri:     "download.example.com/users/active/foo_file_in_dir/dir1/",
 966                         header:  authHeader,
 967                         expect:  []string{"bar"},
 968                         cutDirs: 4,
 969                 },
 970                 {
 971                         uri:     "download.example.com/",
 972                         header:  authHeader,
 973                         expect:  []string{"users/"},
 974                         cutDirs: 0,
 975                 },
 976                 {
 977                         uri:      "download.example.com/users",
 978                         header:   authHeader,
 979                         redirect: "/users/",
 980                         expect:   []string{"active/"},
 981                         cutDirs:  1,
 982                 },
 983                 {
 984                         uri:     "download.example.com/users/",
 985                         header:  authHeader,
 986                         expect:  []string{"active/"},
 987                         cutDirs: 1,
 988                 },
 989                 {
 990                         uri:      "download.example.com/users/active",
 991                         header:   authHeader,
 992                         redirect: "/users/active/",
 993                         expect:   []string{"foo_file_in_dir/"},
 994                         cutDirs:  2,
 995                 },
 996                 {
 997                         uri:     "download.example.com/users/active/",
 998                         header:  authHeader,
 999                         expect:  []string{"foo_file_in_dir/"},
1000                         cutDirs: 2,
1001                 },
1002                 {
1003                         uri:     "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
1004                         header:  nil,
1005                         expect:  []string{"dir1/foo", "dir1/bar"},
1006                         cutDirs: 4,
1007                 },
1008                 {
1009                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
1010                         header:  nil,
1011                         expect:  []string{"dir1/foo", "dir1/bar"},
1012                         cutDirs: 2,
1013                 },
1014                 {
1015                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken,
1016                         header:  nil,
1017                         expect:  []string{"dir1/foo", "dir1/bar"},
1018                         cutDirs: 2,
1019                 },
1020                 {
1021                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID,
1022                         header:  authHeader,
1023                         expect:  []string{"dir1/foo", "dir1/bar"},
1024                         cutDirs: 1,
1025                 },
1026                 {
1027                         uri:      "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
1028                         header:   authHeader,
1029                         redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
1030                         expect:   []string{"foo", "bar"},
1031                         cutDirs:  2,
1032                 },
1033                 {
1034                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
1035                         header:  authHeader,
1036                         expect:  []string{"foo", "bar"},
1037                         cutDirs: 3,
1038                 },
1039                 {
1040                         uri:      arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
1041                         header:   authHeader,
1042                         redirect: "/dir1/",
1043                         expect:   []string{"foo", "bar"},
1044                         cutDirs:  1,
1045                 },
1046                 {
1047                         uri:    "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
1048                         header: authHeader,
1049                         expect: nil,
1050                 },
1051                 {
1052                         uri:     "download.example.com/c=" + arvadostest.WazVersion1Collection,
1053                         header:  authHeader,
1054                         expect:  []string{"waz"},
1055                         cutDirs: 1,
1056                 },
1057                 {
1058                         uri:     "download.example.com/by_id/" + arvadostest.WazVersion1Collection,
1059                         header:  authHeader,
1060                         expect:  []string{"waz"},
1061                         cutDirs: 2,
1062                 },
1063         } {
1064                 comment := check.Commentf("HTML: %q => %q", trial.uri, trial.expect)
1065                 resp := httptest.NewRecorder()
1066                 u := mustParseURL("//" + trial.uri)
1067                 req := &http.Request{
1068                         Method:     "GET",
1069                         Host:       u.Host,
1070                         URL:        u,
1071                         RequestURI: u.RequestURI(),
1072                         Header:     copyHeader(trial.header),
1073                 }
1074                 s.handler.ServeHTTP(resp, req)
1075                 var cookies []*http.Cookie
1076                 for resp.Code == http.StatusSeeOther {
1077                         u, _ := req.URL.Parse(resp.Header().Get("Location"))
1078                         req = &http.Request{
1079                                 Method:     "GET",
1080                                 Host:       u.Host,
1081                                 URL:        u,
1082                                 RequestURI: u.RequestURI(),
1083                                 Header:     copyHeader(trial.header),
1084                         }
1085                         cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
1086                         for _, c := range cookies {
1087                                 req.AddCookie(c)
1088                         }
1089                         resp = httptest.NewRecorder()
1090                         s.handler.ServeHTTP(resp, req)
1091                 }
1092                 if trial.redirect != "" {
1093                         c.Check(req.URL.Path, check.Equals, trial.redirect, comment)
1094                 }
1095                 if trial.expect == nil {
1096                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1097                 } else {
1098                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
1099                         for _, e := range trial.expect {
1100                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`, comment)
1101                         }
1102                         c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`, comment)
1103                 }
1104
1105                 comment = check.Commentf("WebDAV: %q => %q", trial.uri, trial.expect)
1106                 req = &http.Request{
1107                         Method:     "OPTIONS",
1108                         Host:       u.Host,
1109                         URL:        u,
1110                         RequestURI: u.RequestURI(),
1111                         Header:     copyHeader(trial.header),
1112                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
1113                 }
1114                 resp = httptest.NewRecorder()
1115                 s.handler.ServeHTTP(resp, req)
1116                 if trial.expect == nil {
1117                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1118                 } else {
1119                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
1120                 }
1121
1122                 req = &http.Request{
1123                         Method:     "PROPFIND",
1124                         Host:       u.Host,
1125                         URL:        u,
1126                         RequestURI: u.RequestURI(),
1127                         Header:     copyHeader(trial.header),
1128                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
1129                 }
1130                 resp = httptest.NewRecorder()
1131                 s.handler.ServeHTTP(resp, req)
1132                 if trial.expect == nil {
1133                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1134                 } else {
1135                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment)
1136                         for _, e := range trial.expect {
1137                                 if strings.HasSuffix(e, "/") {
1138                                         e = filepath.Join(u.Path, e) + "/"
1139                                 } else {
1140                                         e = filepath.Join(u.Path, e)
1141                                 }
1142                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+e+`</D:href>.*`, comment)
1143                         }
1144                 }
1145         }
1146 }
1147
1148 func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
1149         arv := arvados.NewClientFromEnv()
1150         var newCollection arvados.Collection
1151         err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1152                 "collection": map[string]string{
1153                         "owner_uuid":    arvadostest.ActiveUserUUID,
1154                         "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
1155                         "name":          "keep-web test collection",
1156                 },
1157                 "ensure_unique_name": true,
1158         })
1159         c.Assert(err, check.IsNil)
1160         defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
1161
1162         var updated arvados.Collection
1163         for _, fnm := range []string{"foo.txt", "bar.txt"} {
1164                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
1165                 u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
1166                 req := &http.Request{
1167                         Method:     "DELETE",
1168                         Host:       u.Host,
1169                         URL:        u,
1170                         RequestURI: u.RequestURI(),
1171                         Header: http.Header{
1172                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
1173                         },
1174                 }
1175                 resp := httptest.NewRecorder()
1176                 s.handler.ServeHTTP(resp, req)
1177                 c.Check(resp.Code, check.Equals, http.StatusNoContent)
1178
1179                 updated = arvados.Collection{}
1180                 err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
1181                 c.Check(err, check.IsNil)
1182                 c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`)
1183                 c.Logf("updated manifest_text %q", updated.ManifestText)
1184         }
1185         c.Check(updated.ManifestText, check.Equals, "")
1186 }
1187
1188 func (s *IntegrationSuite) TestFileContentType(c *check.C) {
1189         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1190
1191         client := arvados.NewClientFromEnv()
1192         client.AuthToken = arvadostest.ActiveToken
1193         arv, err := arvadosclient.New(client)
1194         c.Assert(err, check.Equals, nil)
1195         kc, err := keepclient.MakeKeepClient(arv)
1196         c.Assert(err, check.Equals, nil)
1197
1198         fs, err := (&arvados.Collection{}).FileSystem(client, kc)
1199         c.Assert(err, check.IsNil)
1200
1201         trials := []struct {
1202                 filename    string
1203                 content     string
1204                 contentType string
1205         }{
1206                 {"picture.txt", "BMX bikes are small this year\n", "text/plain; charset=utf-8"},
1207                 {"picture.bmp", "BMX bikes are small this year\n", "image/(x-ms-)?bmp"},
1208                 {"picture.jpg", "BMX bikes are small this year\n", "image/jpeg"},
1209                 {"picture1", "BMX bikes are small this year\n", "image/bmp"},            // content sniff; "BM" is the magic signature for .bmp
1210                 {"picture2", "Cars are small this year\n", "text/plain; charset=utf-8"}, // content sniff
1211         }
1212         for _, trial := range trials {
1213                 f, err := fs.OpenFile(trial.filename, os.O_CREATE|os.O_WRONLY, 0777)
1214                 c.Assert(err, check.IsNil)
1215                 _, err = f.Write([]byte(trial.content))
1216                 c.Assert(err, check.IsNil)
1217                 c.Assert(f.Close(), check.IsNil)
1218         }
1219         mtxt, err := fs.MarshalManifest(".")
1220         c.Assert(err, check.IsNil)
1221         var coll arvados.Collection
1222         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1223                 "collection": map[string]string{
1224                         "manifest_text": mtxt,
1225                 },
1226         })
1227         c.Assert(err, check.IsNil)
1228
1229         for _, trial := range trials {
1230                 u, _ := url.Parse("http://download.example.com/by_id/" + coll.UUID + "/" + trial.filename)
1231                 req := &http.Request{
1232                         Method:     "GET",
1233                         Host:       u.Host,
1234                         URL:        u,
1235                         RequestURI: u.RequestURI(),
1236                         Header: http.Header{
1237                                 "Authorization": {"Bearer " + client.AuthToken},
1238                         },
1239                 }
1240                 resp := httptest.NewRecorder()
1241                 s.handler.ServeHTTP(resp, req)
1242                 c.Check(resp.Code, check.Equals, http.StatusOK)
1243                 c.Check(resp.Header().Get("Content-Type"), check.Matches, trial.contentType)
1244                 c.Check(resp.Body.String(), check.Equals, trial.content)
1245         }
1246 }
1247
1248 func (s *IntegrationSuite) TestKeepClientBlockCache(c *check.C) {
1249         s.handler.Cluster.Collections.WebDAVCache.MaxBlockEntries = 42
1250         c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Not(check.Equals), 42)
1251         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/t=" + arvadostest.ActiveToken + "/foo")
1252         req := &http.Request{
1253                 Method:     "GET",
1254                 Host:       u.Host,
1255                 URL:        u,
1256                 RequestURI: u.RequestURI(),
1257         }
1258         resp := httptest.NewRecorder()
1259         s.handler.ServeHTTP(resp, req)
1260         c.Check(resp.Code, check.Equals, http.StatusOK)
1261         c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Equals, 42)
1262 }
1263
1264 // Writing to a collection shouldn't affect its entry in the
1265 // PDH-to-manifest cache.
1266 func (s *IntegrationSuite) TestCacheWriteCollectionSamePDH(c *check.C) {
1267         arv, err := arvadosclient.MakeArvadosClient()
1268         c.Assert(err, check.Equals, nil)
1269         arv.ApiToken = arvadostest.ActiveToken
1270
1271         u := mustParseURL("http://x.example/testfile")
1272         req := &http.Request{
1273                 Method:     "GET",
1274                 Host:       u.Host,
1275                 URL:        u,
1276                 RequestURI: u.RequestURI(),
1277                 Header:     http.Header{"Authorization": {"Bearer " + arv.ApiToken}},
1278         }
1279
1280         checkWithID := func(id string, status int) {
1281                 req.URL.Host = strings.Replace(id, "+", "-", -1) + ".example"
1282                 req.Host = req.URL.Host
1283                 resp := httptest.NewRecorder()
1284                 s.handler.ServeHTTP(resp, req)
1285                 c.Check(resp.Code, check.Equals, status)
1286         }
1287
1288         var colls [2]arvados.Collection
1289         for i := range colls {
1290                 err := arv.Create("collections",
1291                         map[string]interface{}{
1292                                 "ensure_unique_name": true,
1293                                 "collection": map[string]interface{}{
1294                                         "name": "test collection",
1295                                 },
1296                         }, &colls[i])
1297                 c.Assert(err, check.Equals, nil)
1298         }
1299
1300         // Populate cache with empty collection
1301         checkWithID(colls[0].PortableDataHash, http.StatusNotFound)
1302
1303         // write a file to colls[0]
1304         reqPut := *req
1305         reqPut.Method = "PUT"
1306         reqPut.URL.Host = colls[0].UUID + ".example"
1307         reqPut.Host = req.URL.Host
1308         reqPut.Body = ioutil.NopCloser(bytes.NewBufferString("testdata"))
1309         resp := httptest.NewRecorder()
1310         s.handler.ServeHTTP(resp, &reqPut)
1311         c.Check(resp.Code, check.Equals, http.StatusCreated)
1312
1313         // new file should not appear in colls[1]
1314         checkWithID(colls[1].PortableDataHash, http.StatusNotFound)
1315         checkWithID(colls[1].UUID, http.StatusNotFound)
1316
1317         checkWithID(colls[0].UUID, http.StatusOK)
1318 }
1319
1320 func copyHeader(h http.Header) http.Header {
1321         hc := http.Header{}
1322         for k, v := range h {
1323                 hc[k] = append([]string(nil), v...)
1324         }
1325         return hc
1326 }
1327
1328 func (s *IntegrationSuite) checkUploadDownloadRequest(c *check.C, req *http.Request,
1329         successCode int, direction string, perm bool, userUuid, collectionUuid, collectionPDH, filepath string) {
1330
1331         client := arvados.NewClientFromEnv()
1332         client.AuthToken = arvadostest.AdminToken
1333         var logentries arvados.LogList
1334         limit1 := 1
1335         err := client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil,
1336                 arvados.ResourceListParams{
1337                         Limit: &limit1,
1338                         Order: "created_at desc"})
1339         c.Check(err, check.IsNil)
1340         c.Check(logentries.Items, check.HasLen, 1)
1341         lastLogId := logentries.Items[0].ID
1342         c.Logf("lastLogId: %d", lastLogId)
1343
1344         var logbuf bytes.Buffer
1345         logger := logrus.New()
1346         logger.Out = &logbuf
1347         resp := httptest.NewRecorder()
1348         req = req.WithContext(ctxlog.Context(context.Background(), logger))
1349         s.handler.ServeHTTP(resp, req)
1350
1351         if perm {
1352                 c.Check(resp.Result().StatusCode, check.Equals, successCode)
1353                 c.Check(logbuf.String(), check.Matches, `(?ms).*msg="File `+direction+`".*`)
1354                 c.Check(logbuf.String(), check.Not(check.Matches), `(?ms).*level=error.*`)
1355
1356                 deadline := time.Now().Add(time.Second)
1357                 for {
1358                         c.Assert(time.Now().After(deadline), check.Equals, false, check.Commentf("timed out waiting for log entry"))
1359                         logentries = arvados.LogList{}
1360                         err = client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil,
1361                                 arvados.ResourceListParams{
1362                                         Filters: []arvados.Filter{
1363                                                 {Attr: "event_type", Operator: "=", Operand: "file_" + direction},
1364                                                 {Attr: "object_uuid", Operator: "=", Operand: userUuid},
1365                                         },
1366                                         Limit: &limit1,
1367                                         Order: "created_at desc",
1368                                 })
1369                         c.Assert(err, check.IsNil)
1370                         if len(logentries.Items) > 0 &&
1371                                 logentries.Items[0].ID > lastLogId &&
1372                                 logentries.Items[0].ObjectUUID == userUuid &&
1373                                 logentries.Items[0].Properties["collection_uuid"] == collectionUuid &&
1374                                 (collectionPDH == "" || logentries.Items[0].Properties["portable_data_hash"] == collectionPDH) &&
1375                                 logentries.Items[0].Properties["collection_file_path"] == filepath {
1376                                 break
1377                         }
1378                         c.Logf("logentries.Items: %+v", logentries.Items)
1379                         time.Sleep(50 * time.Millisecond)
1380                 }
1381         } else {
1382                 c.Check(resp.Result().StatusCode, check.Equals, http.StatusForbidden)
1383                 c.Check(logbuf.String(), check.Equals, "")
1384         }
1385 }
1386
1387 func (s *IntegrationSuite) TestDownloadLoggingPermission(c *check.C) {
1388         u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
1389
1390         s.handler.Cluster.Collections.TrustAllContent = true
1391
1392         for _, adminperm := range []bool{true, false} {
1393                 for _, userperm := range []bool{true, false} {
1394                         s.handler.Cluster.Collections.WebDAVPermission.Admin.Download = adminperm
1395                         s.handler.Cluster.Collections.WebDAVPermission.User.Download = userperm
1396
1397                         // Test admin permission
1398                         req := &http.Request{
1399                                 Method:     "GET",
1400                                 Host:       u.Host,
1401                                 URL:        u,
1402                                 RequestURI: u.RequestURI(),
1403                                 Header: http.Header{
1404                                         "Authorization": {"Bearer " + arvadostest.AdminToken},
1405                                 },
1406                         }
1407                         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", adminperm,
1408                                 arvadostest.AdminUserUUID, arvadostest.FooCollection, arvadostest.FooCollectionPDH, "foo")
1409
1410                         // Test user permission
1411                         req = &http.Request{
1412                                 Method:     "GET",
1413                                 Host:       u.Host,
1414                                 URL:        u,
1415                                 RequestURI: u.RequestURI(),
1416                                 Header: http.Header{
1417                                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
1418                                 },
1419                         }
1420                         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", userperm,
1421                                 arvadostest.ActiveUserUUID, arvadostest.FooCollection, arvadostest.FooCollectionPDH, "foo")
1422                 }
1423         }
1424
1425         s.handler.Cluster.Collections.WebDAVPermission.User.Download = true
1426
1427         for _, tryurl := range []string{"http://" + arvadostest.MultilevelCollection1 + ".keep-web.example/dir1/subdir/file1",
1428                 "http://keep-web/users/active/multilevel_collection_1/dir1/subdir/file1"} {
1429
1430                 u = mustParseURL(tryurl)
1431                 req := &http.Request{
1432                         Method:     "GET",
1433                         Host:       u.Host,
1434                         URL:        u,
1435                         RequestURI: u.RequestURI(),
1436                         Header: http.Header{
1437                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
1438                         },
1439                 }
1440                 s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true,
1441                         arvadostest.ActiveUserUUID, arvadostest.MultilevelCollection1, arvadostest.MultilevelCollection1PDH, "dir1/subdir/file1")
1442         }
1443
1444         u = mustParseURL("http://" + strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + ".keep-web.example/foo")
1445         req := &http.Request{
1446                 Method:     "GET",
1447                 Host:       u.Host,
1448                 URL:        u,
1449                 RequestURI: u.RequestURI(),
1450                 Header: http.Header{
1451                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
1452                 },
1453         }
1454         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true,
1455                 arvadostest.ActiveUserUUID, "", arvadostest.FooCollectionPDH, "foo")
1456 }
1457
1458 func (s *IntegrationSuite) TestUploadLoggingPermission(c *check.C) {
1459         for _, adminperm := range []bool{true, false} {
1460                 for _, userperm := range []bool{true, false} {
1461
1462                         arv := arvados.NewClientFromEnv()
1463                         arv.AuthToken = arvadostest.ActiveToken
1464
1465                         var coll arvados.Collection
1466                         err := arv.RequestAndDecode(&coll,
1467                                 "POST",
1468                                 "/arvados/v1/collections",
1469                                 nil,
1470                                 map[string]interface{}{
1471                                         "ensure_unique_name": true,
1472                                         "collection": map[string]interface{}{
1473                                                 "name": "test collection",
1474                                         },
1475                                 })
1476                         c.Assert(err, check.Equals, nil)
1477
1478                         u := mustParseURL("http://" + coll.UUID + ".keep-web.example/bar")
1479
1480                         s.handler.Cluster.Collections.WebDAVPermission.Admin.Upload = adminperm
1481                         s.handler.Cluster.Collections.WebDAVPermission.User.Upload = userperm
1482
1483                         // Test admin permission
1484                         req := &http.Request{
1485                                 Method:     "PUT",
1486                                 Host:       u.Host,
1487                                 URL:        u,
1488                                 RequestURI: u.RequestURI(),
1489                                 Header: http.Header{
1490                                         "Authorization": {"Bearer " + arvadostest.AdminToken},
1491                                 },
1492                                 Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
1493                         }
1494                         s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", adminperm,
1495                                 arvadostest.AdminUserUUID, coll.UUID, "", "bar")
1496
1497                         // Test user permission
1498                         req = &http.Request{
1499                                 Method:     "PUT",
1500                                 Host:       u.Host,
1501                                 URL:        u,
1502                                 RequestURI: u.RequestURI(),
1503                                 Header: http.Header{
1504                                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
1505                                 },
1506                                 Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
1507                         }
1508                         s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", userperm,
1509                                 arvadostest.ActiveUserUUID, coll.UUID, "", "bar")
1510                 }
1511         }
1512 }