services/keep-web/handler_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package keepweb
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "fmt"
  11         "io"
  12         "io/ioutil"
  13         "net/http"
  14         "net/http/httptest"
  15         "net/url"
  16         "os"
  17         "path/filepath"
  18         "regexp"
  19         "strings"
  20         "sync"
  21         "time"
  22
  23         "git.arvados.org/arvados.git/lib/config"
  24         "git.arvados.org/arvados.git/sdk/go/arvados"
  25         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  26         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  27         "git.arvados.org/arvados.git/sdk/go/auth"
  28         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  29         "git.arvados.org/arvados.git/sdk/go/keepclient"
  30         "github.com/prometheus/client_golang/prometheus"
  31         "github.com/sirupsen/logrus"
  32         "golang.org/x/net/html"
  33         check "gopkg.in/check.v1"
  34 )
  35
  36 var _ = check.Suite(&UnitSuite{})
  37
  38 func init() {
  39         arvados.DebugLocksPanicMode = true
  40 }
  41
  42 type UnitSuite struct {
  43         cluster *arvados.Cluster
  44         handler *handler
  45 }
  46
  47 func (s *UnitSuite) SetUpTest(c *check.C) {
  48         logger := ctxlog.TestLogger(c)
  49         ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), logger)
  50         ldr.Path = "-"
  51         cfg, err := ldr.Load()
  52         c.Assert(err, check.IsNil)
  53         cc, err := cfg.GetCluster("")
  54         c.Assert(err, check.IsNil)
  55         s.cluster = cc
  56         s.handler = &handler{
  57                 Cluster: cc,
  58                 Cache: cache{
  59                         cluster:  cc,
  60                         logger:   logger,
  61                         registry: prometheus.NewRegistry(),
  62                 },
  63                 metrics: newMetrics(prometheus.NewRegistry()),
  64         }
  65 }
  66
  67 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
  68         h := s.handler
  69         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
  70         req := &http.Request{
  71                 Method:     "OPTIONS",
  72                 Host:       u.Host,
  73                 URL:        u,
  74                 RequestURI: u.RequestURI(),
  75                 Header: http.Header{
  76                         "Origin":                        {"https://workbench.example"},
  77                         "Access-Control-Request-Method": {"POST"},
  78                 },
  79         }
  80
  81         // Check preflight for an allowed request
  82         resp := httptest.NewRecorder()
  83         h.ServeHTTP(resp, req)
  84         c.Check(resp.Code, check.Equals, http.StatusOK)
  85         c.Check(resp.Body.String(), check.Equals, "")
  86         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
  87         c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
  88         c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout, Cache-Control")
  89
  90         // Check preflight for a disallowed request
  91         resp = httptest.NewRecorder()
  92         req.Header.Set("Access-Control-Request-Method", "MAKE-COFFEE")
  93         h.ServeHTTP(resp, req)
  94         c.Check(resp.Body.String(), check.Equals, "")
  95         c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
  96 }
  97
  98 func (s *UnitSuite) TestWebdavPrefixAndSource(c *check.C) {
  99         for _, trial := range []struct {
 100                 method   string
 101                 path     string
 102                 prefix   string
 103                 source   string
 104                 notFound bool
 105                 seeOther bool
 106         }{
 107                 {
 108                         method: "PROPFIND",
 109                         path:   "/",
 110                 },
 111                 {
 112                         method: "PROPFIND",
 113                         path:   "/dir1",
 114                 },
 115                 {
 116                         method: "PROPFIND",
 117                         path:   "/dir1/",
 118                 },
 119                 {
 120                         method: "PROPFIND",
 121                         path:   "/dir1/foo",
 122                         prefix: "/dir1",
 123                         source: "/dir1",
 124                 },
 125                 {
 126                         method: "PROPFIND",
 127                         path:   "/prefix/dir1/foo",
 128                         prefix: "/prefix/",
 129                         source: "",
 130                 },
 131                 {
 132                         method: "PROPFIND",
 133                         path:   "/prefix/dir1/foo",
 134                         prefix: "/prefix",
 135                         source: "",
 136                 },
 137                 {
 138                         method: "PROPFIND",
 139                         path:   "/prefix/dir1/foo",
 140                         prefix: "/prefix/",
 141                         source: "/",
 142                 },
 143                 {
 144                         method: "PROPFIND",
 145                         path:   "/prefix/foo",
 146                         prefix: "/prefix/",
 147                         source: "/dir1/",
 148                 },
 149                 {
 150                         method: "GET",
 151                         path:   "/prefix/foo",
 152                         prefix: "/prefix/",
 153                         source: "/dir1/",
 154                 },
 155                 {
 156                         method: "PROPFIND",
 157                         path:   "/prefix/",
 158                         prefix: "/prefix",
 159                         source: "/dir1",
 160                 },
 161                 {
 162                         method: "PROPFIND",
 163                         path:   "/prefix",
 164                         prefix: "/prefix",
 165                         source: "/dir1/",
 166                 },
 167                 {
 168                         method:   "GET",
 169                         path:     "/prefix",
 170                         prefix:   "/prefix",
 171                         source:   "/dir1",
 172                         seeOther: true,
 173                 },
 174                 {
 175                         method:   "PROPFIND",
 176                         path:     "/dir1/foo",
 177                         prefix:   "",
 178                         source:   "/dir1",
 179                         notFound: true,
 180                 },
 181         } {
 182                 c.Logf("trial %+v", trial)
 183                 u := mustParseURL("http://" + arvadostest.FooBarDirCollection + ".keep-web.example" + trial.path)
 184                 req := &http.Request{
 185                         Method:     trial.method,
 186                         Host:       u.Host,
 187                         URL:        u,
 188                         RequestURI: u.RequestURI(),
 189                         Header: http.Header{
 190                                 "Authorization":   {"Bearer " + arvadostest.ActiveTokenV2},
 191                                 "X-Webdav-Prefix": {trial.prefix},
 192                                 "X-Webdav-Source": {trial.source},
 193                         },
 194                         Body: ioutil.NopCloser(bytes.NewReader(nil)),
 195                 }
 196
 197                 resp := httptest.NewRecorder()
 198                 s.handler.ServeHTTP(resp, req)
 199                 if trial.notFound {
 200                         c.Check(resp.Code, check.Equals, http.StatusNotFound)
 201                 } else if trial.method == "PROPFIND" {
 202                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus)
 203                         c.Check(resp.Body.String(), check.Matches, `(?ms).*>\n?$`)
 204                 } else if trial.seeOther {
 205                         c.Check(resp.Code, check.Equals, http.StatusSeeOther)
 206                 } else {
 207                         c.Check(resp.Code, check.Equals, http.StatusOK)
 208                 }
 209         }
 210 }
 211
 212 func (s *UnitSuite) TestEmptyResponse(c *check.C) {
 213         // Ensure we start with an empty cache
 214         defer os.Setenv("HOME", os.Getenv("HOME"))
 215         os.Setenv("HOME", c.MkDir())
 216
 217         for _, trial := range []struct {
 218                 dataExists    bool
 219                 sendIMSHeader bool
 220                 expectStatus  int
 221                 logRegexp     string
 222         }{
 223                 // If we return no content due to a Keep read error,
 224                 // we should emit a log message.
 225                 {false, false, http.StatusOK, `(?ms).*only wrote 0 bytes.*`},
 226
 227                 // If we return no content because the client sent an
 228                 // If-Modified-Since header, our response should be
 229                 // 304.  We still expect a "File download" log since it
 230                 // counts as a file access for auditing.
 231                 {true, true, http.StatusNotModified, `(?ms).*msg="File download".*`},
 232         } {
 233                 c.Logf("trial: %+v", trial)
 234                 arvadostest.StartKeep(2, true)
 235                 if trial.dataExists {
 236                         arv, err := arvadosclient.MakeArvadosClient()
 237                         c.Assert(err, check.IsNil)
 238                         arv.ApiToken = arvadostest.ActiveToken
 239                         kc, err := keepclient.MakeKeepClient(arv)
 240                         c.Assert(err, check.IsNil)
 241                         _, _, err = kc.PutB([]byte("foo"))
 242                         c.Assert(err, check.IsNil)
 243                 }
 244
 245                 u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
 246                 req := &http.Request{
 247                         Method:     "GET",
 248                         Host:       u.Host,
 249                         URL:        u,
 250                         RequestURI: u.RequestURI(),
 251                         Header: http.Header{
 252                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
 253                         },
 254                 }
 255                 if trial.sendIMSHeader {
 256                         req.Header.Set("If-Modified-Since", strings.Replace(time.Now().UTC().Format(time.RFC1123), "UTC", "GMT", -1))
 257                 }
 258
 259                 var logbuf bytes.Buffer
 260                 logger := logrus.New()
 261                 logger.Out = &logbuf
 262                 req = req.WithContext(ctxlog.Context(context.Background(), logger))
 263
 264                 resp := httptest.NewRecorder()
 265                 s.handler.ServeHTTP(resp, req)
 266                 c.Check(resp.Code, check.Equals, trial.expectStatus)
 267                 c.Check(resp.Body.String(), check.Equals, "")
 268
 269                 c.Log(logbuf.String())
 270                 c.Check(logbuf.String(), check.Matches, trial.logRegexp)
 271         }
 272 }
 273
 274 func (s *UnitSuite) TestInvalidUUID(c *check.C) {
 275         bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-"
 276         token := arvadostest.ActiveToken
 277         for _, trial := range []string{
 278                 "http://keep-web/c=" + bogusID + "/foo",
 279                 "http://keep-web/c=" + bogusID + "/t=" + token + "/foo",
 280                 "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo",
 281                 "http://keep-web/collections/" + bogusID + "/foo",
 282                 "http://" + bogusID + ".keep-web/" + bogusID + "/foo",
 283                 "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
 284         } {
 285                 c.Log(trial)
 286                 u := mustParseURL(trial)
 287                 req := &http.Request{
 288                         Method:     "GET",
 289                         Host:       u.Host,
 290                         URL:        u,
 291                         RequestURI: u.RequestURI(),
 292                 }
 293                 resp := httptest.NewRecorder()
 294                 s.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 295                 s.handler.ServeHTTP(resp, req)
 296                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 297         }
 298 }
 299
 300 func mustParseURL(s string) *url.URL {
 301         r, err := url.Parse(s)
 302         if err != nil {
 303                 panic("parse URL: " + s)
 304         }
 305         return r
 306 }
 307
 308 func (s *IntegrationSuite) TestVhost404(c *check.C) {
 309         for _, testURL := range []string{
 310                 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
 311                 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
 312         } {
 313                 resp := httptest.NewRecorder()
 314                 u := mustParseURL(testURL)
 315                 req := &http.Request{
 316                         Method:     "GET",
 317                         URL:        u,
 318                         RequestURI: u.RequestURI(),
 319                 }
 320                 s.handler.ServeHTTP(resp, req)
 321                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 322                 c.Check(resp.Body.String(), check.Equals, notFoundMessage+"\n")
 323         }
 324 }
 325
 326 // An authorizer modifies an HTTP request to make use of the given
 327 // token -- by adding it to a header, cookie, query param, or whatever
 328 // -- and returns the HTTP status code we should expect from keep-web if
 329 // the token is invalid.
 330 type authorizer func(*http.Request, string) int
 331
 332 func (s *IntegrationSuite) TestVhostViaAuthzHeaderOAuth2(c *check.C) {
 333         s.doVhostRequests(c, authzViaAuthzHeaderOAuth2)
 334 }
 335 func authzViaAuthzHeaderOAuth2(r *http.Request, tok string) int {
 336         r.Header.Add("Authorization", "OAuth2 "+tok)
 337         return http.StatusUnauthorized
 338 }
 339
 340 func (s *IntegrationSuite) TestVhostViaAuthzHeaderBearer(c *check.C) {
 341         s.doVhostRequests(c, authzViaAuthzHeaderBearer)
 342 }
 343 func authzViaAuthzHeaderBearer(r *http.Request, tok string) int {
 344         r.Header.Add("Authorization", "Bearer "+tok)
 345         return http.StatusUnauthorized
 346 }
 347
 348 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
 349         s.doVhostRequests(c, authzViaCookieValue)
 350 }
 351 func authzViaCookieValue(r *http.Request, tok string) int {
 352         r.AddCookie(&http.Cookie{
 353                 Name:  "arvados_api_token",
 354                 Value: auth.EncodeTokenCookie([]byte(tok)),
 355         })
 356         return http.StatusUnauthorized
 357 }
 358
 359 func (s *IntegrationSuite) TestVhostViaHTTPBasicAuth(c *check.C) {
 360         s.doVhostRequests(c, authzViaHTTPBasicAuth)
 361 }
 362 func authzViaHTTPBasicAuth(r *http.Request, tok string) int {
 363         r.AddCookie(&http.Cookie{
 364                 Name:  "arvados_api_token",
 365                 Value: auth.EncodeTokenCookie([]byte(tok)),
 366         })
 367         return http.StatusUnauthorized
 368 }
 369
 370 func (s *IntegrationSuite) TestVhostViaHTTPBasicAuthWithExtraSpaceChars(c *check.C) {
 371         s.doVhostRequests(c, func(r *http.Request, tok string) int {
 372                 r.AddCookie(&http.Cookie{
 373                         Name:  "arvados_api_token",
 374                         Value: auth.EncodeTokenCookie([]byte(" " + tok + "\n")),
 375                 })
 376                 return http.StatusUnauthorized
 377         })
 378 }
 379
 380 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
 381         s.doVhostRequests(c, authzViaPath)
 382 }
 383 func authzViaPath(r *http.Request, tok string) int {
 384         r.URL.Path = "/t=" + tok + r.URL.Path
 385         return http.StatusNotFound
 386 }
 387
 388 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
 389         s.doVhostRequests(c, authzViaQueryString)
 390 }
 391 func authzViaQueryString(r *http.Request, tok string) int {
 392         r.URL.RawQuery = "api_token=" + tok
 393         return http.StatusUnauthorized
 394 }
 395
 396 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
 397         s.doVhostRequests(c, authzViaPOST)
 398 }
 399 func authzViaPOST(r *http.Request, tok string) int {
 400         r.Method = "POST"
 401         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 402         r.Body = ioutil.NopCloser(strings.NewReader(
 403                 url.Values{"api_token": {tok}}.Encode()))
 404         return http.StatusUnauthorized
 405 }
 406
 407 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
 408         s.doVhostRequests(c, authzViaPOST)
 409 }
 410 func authzViaXHRPOST(r *http.Request, tok string) int {
 411         r.Method = "POST"
 412         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 413         r.Header.Add("Origin", "https://origin.example")
 414         r.Body = ioutil.NopCloser(strings.NewReader(
 415                 url.Values{
 416                         "api_token":   {tok},
 417                         "disposition": {"attachment"},
 418                 }.Encode()))
 419         return http.StatusUnauthorized
 420 }
 421
 422 // Try some combinations of {url, token} using the given authorization
 423 // mechanism, and verify the result is correct.
 424 func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
 425         for _, hostPath := range []string{
 426                 arvadostest.FooCollection + ".example.com/foo",
 427                 arvadostest.FooCollection + "--collections.example.com/foo",
 428                 arvadostest.FooCollection + "--collections.example.com/_/foo",
 429                 arvadostest.FooCollectionPDH + ".example.com/foo",
 430                 strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + "--collections.example.com/foo",
 431                 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
 432         } {
 433                 c.Log("doRequests: ", hostPath)
 434                 s.doVhostRequestsWithHostPath(c, authz, hostPath)
 435         }
 436 }
 437
 438 func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
 439         for _, tok := range []string{
 440                 arvadostest.ActiveToken,
 441                 arvadostest.ActiveToken[:15],
 442                 arvadostest.SpectatorToken,
 443                 "bogus",
 444                 "",
 445         } {
 446                 u := mustParseURL("http://" + hostPath)
 447                 req := &http.Request{
 448                         Method:     "GET",
 449                         Host:       u.Host,
 450                         URL:        u,
 451                         RequestURI: u.RequestURI(),
 452                         Header:     http.Header{},
 453                 }
 454                 failCode := authz(req, tok)
 455                 req, resp := s.doReq(req)
 456                 code, body := resp.Code, resp.Body.String()
 457
 458                 // If the initial request had a (non-empty) token
 459                 // showing in the query string, we should have been
 460                 // redirected in order to hide it in a cookie.
 461                 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
 462
 463                 if tok == arvadostest.ActiveToken {
 464                         c.Check(code, check.Equals, http.StatusOK)
 465                         c.Check(body, check.Equals, "foo")
 466                 } else {
 467                         c.Check(code >= 400, check.Equals, true)
 468                         c.Check(code < 500, check.Equals, true)
 469                         if tok == arvadostest.SpectatorToken {
 470                                 // Valid token never offers to retry
 471                                 // with different credentials.
 472                                 c.Check(code, check.Equals, http.StatusNotFound)
 473                         } else {
 474                                 // Invalid token can ask to retry
 475                                 // depending on the authz method.
 476                                 c.Check(code, check.Equals, failCode)
 477                         }
 478                         if code == 404 {
 479                                 c.Check(body, check.Equals, notFoundMessage+"\n")
 480                         } else {
 481                                 c.Check(body, check.Equals, unauthorizedMessage+"\n")
 482                         }
 483                 }
 484         }
 485 }
 486
 487 func (s *IntegrationSuite) TestVhostPortMatch(c *check.C) {
 488         for _, host := range []string{"download.example.com", "DOWNLOAD.EXAMPLE.COM"} {
 489                 for _, port := range []string{"80", "443", "8000"} {
 490                         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = fmt.Sprintf("download.example.com:%v", port)
 491                         u := mustParseURL(fmt.Sprintf("http://%v/by_id/%v/foo", host, arvadostest.FooCollection))
 492                         req := &http.Request{
 493                                 Method:     "GET",
 494                                 Host:       u.Host,
 495                                 URL:        u,
 496                                 RequestURI: u.RequestURI(),
 497                                 Header:     http.Header{"Authorization": []string{"Bearer " + arvadostest.ActiveToken}},
 498                         }
 499                         req, resp := s.doReq(req)
 500                         code, _ := resp.Code, resp.Body.String()
 501
 502                         if port == "8000" {
 503                                 c.Check(code, check.Equals, 401)
 504                         } else {
 505                                 c.Check(code, check.Equals, 200)
 506                         }
 507                 }
 508         }
 509 }
 510
 511 func (s *IntegrationSuite) do(method string, urlstring string, token string, hdr http.Header) (*http.Request, *httptest.ResponseRecorder) {
 512         u := mustParseURL(urlstring)
 513         if hdr == nil && token != "" {
 514                 hdr = http.Header{"Authorization": {"Bearer " + token}}
 515         } else if hdr == nil {
 516                 hdr = http.Header{}
 517         } else if token != "" {
 518                 panic("must not pass both token and hdr")
 519         }
 520         return s.doReq(&http.Request{
 521                 Method:     method,
 522                 Host:       u.Host,
 523                 URL:        u,
 524                 RequestURI: u.RequestURI(),
 525                 Header:     hdr,
 526         })
 527 }
 528
 529 func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
 530         resp := httptest.NewRecorder()
 531         s.handler.ServeHTTP(resp, req)
 532         if resp.Code != http.StatusSeeOther {
 533                 return req, resp
 534         }
 535         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 536         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 537         req = &http.Request{
 538                 Method:     "GET",
 539                 Host:       u.Host,
 540                 URL:        u,
 541                 RequestURI: u.RequestURI(),
 542                 Header:     http.Header{},
 543         }
 544         for _, c := range cookies {
 545                 req.AddCookie(c)
 546         }
 547         return s.doReq(req)
 548 }
 549
 550 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
 551         s.testVhostRedirectTokenToCookie(c, "GET",
 552                 arvadostest.FooCollection+".example.com/foo",
 553                 "?api_token="+arvadostest.ActiveToken,
 554                 nil,
 555                 "",
 556                 http.StatusOK,
 557                 "foo",
 558         )
 559 }
 560
 561 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
 562         s.testVhostRedirectTokenToCookie(c, "GET",
 563                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
 564                 "",
 565                 nil,
 566                 "",
 567                 http.StatusOK,
 568                 "foo",
 569         )
 570 }
 571
 572 func (s *IntegrationSuite) TestCollectionSharingToken(c *check.C) {
 573         s.testVhostRedirectTokenToCookie(c, "GET",
 574                 "example.com/c="+arvadostest.FooFileCollectionUUID+"/t="+arvadostest.FooFileCollectionSharingToken+"/foo",
 575                 "",
 576                 nil,
 577                 "",
 578                 http.StatusOK,
 579                 "foo",
 580         )
 581         // Same valid sharing token, but requesting a different collection
 582         s.testVhostRedirectTokenToCookie(c, "GET",
 583                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.FooFileCollectionSharingToken+"/foo",
 584                 "",
 585                 nil,
 586                 "",
 587                 http.StatusNotFound,
 588                 regexp.QuoteMeta(notFoundMessage+"\n"),
 589         )
 590 }
 591
 592 // Bad token in URL is 404 Not Found because it doesn't make sense to
 593 // retry the same URL with different authorization.
 594 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
 595         s.testVhostRedirectTokenToCookie(c, "GET",
 596                 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
 597                 "",
 598                 nil,
 599                 "",
 600                 http.StatusNotFound,
 601                 regexp.QuoteMeta(notFoundMessage+"\n"),
 602         )
 603 }
 604
 605 // Bad token in a cookie (even if it got there via our own
 606 // query-string-to-cookie redirect) is, in principle, retryable via
 607 // wb2-login-and-redirect flow.
 608 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
 609         // Inline
 610         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 611                 arvadostest.FooCollection+".example.com/foo",
 612                 "?api_token=thisisabogustoken",
 613                 http.Header{"Sec-Fetch-Mode": {"navigate"}},
 614                 "",
 615                 http.StatusSeeOther,
 616                 "",
 617         )
 618         u, err := url.Parse(resp.Header().Get("Location"))
 619         c.Assert(err, check.IsNil)
 620         c.Logf("redirected to %s", u)
 621         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 622         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 623         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
 624
 625         // Download/attachment indicated by ?disposition=attachment
 626         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 627                 arvadostest.FooCollection+".example.com/foo",
 628                 "?api_token=thisisabogustoken&disposition=attachment",
 629                 http.Header{"Sec-Fetch-Mode": {"navigate"}},
 630                 "",
 631                 http.StatusSeeOther,
 632                 "",
 633         )
 634         u, err = url.Parse(resp.Header().Get("Location"))
 635         c.Assert(err, check.IsNil)
 636         c.Logf("redirected to %s", u)
 637         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 638         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "")
 639         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 640
 641         // Download/attachment indicated by vhost
 642         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 643                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
 644                 "?api_token=thisisabogustoken",
 645                 http.Header{"Sec-Fetch-Mode": {"navigate"}},
 646                 "",
 647                 http.StatusSeeOther,
 648                 "",
 649         )
 650         u, err = url.Parse(resp.Header().Get("Location"))
 651         c.Assert(err, check.IsNil)
 652         c.Logf("redirected to %s", u)
 653         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 654         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "")
 655         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 656
 657         // Without "Sec-Fetch-Mode: navigate" header, just 401.
 658         s.testVhostRedirectTokenToCookie(c, "GET",
 659                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
 660                 "?api_token=thisisabogustoken",
 661                 http.Header{"Sec-Fetch-Mode": {"cors"}},
 662                 "",
 663                 http.StatusUnauthorized,
 664                 regexp.QuoteMeta(unauthorizedMessage+"\n"),
 665         )
 666         s.testVhostRedirectTokenToCookie(c, "GET",
 667                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
 668                 "?api_token=thisisabogustoken",
 669                 nil,
 670                 "",
 671                 http.StatusUnauthorized,
 672                 regexp.QuoteMeta(unauthorizedMessage+"\n"),
 673         )
 674 }
 675
 676 func (s *IntegrationSuite) TestVhostRedirectWithNoCache(c *check.C) {
 677         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 678                 arvadostest.FooCollection+".example.com/foo",
 679                 "?api_token=thisisabogustoken",
 680                 http.Header{
 681                         "Sec-Fetch-Mode": {"navigate"},
 682                         "Cache-Control":  {"no-cache"},
 683                 },
 684                 "",
 685                 http.StatusSeeOther,
 686                 "",
 687         )
 688         u, err := url.Parse(resp.Header().Get("Location"))
 689         c.Assert(err, check.IsNil)
 690         c.Logf("redirected to %s", u)
 691         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 692         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 693         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
 694 }
 695
 696 func (s *IntegrationSuite) TestNoTokenWorkbench2LoginFlow(c *check.C) {
 697         for _, trial := range []struct {
 698                 anonToken    bool
 699                 cacheControl string
 700         }{
 701                 {},
 702                 {cacheControl: "no-cache"},
 703                 {anonToken: true},
 704                 {anonToken: true, cacheControl: "no-cache"},
 705         } {
 706                 c.Logf("trial: %+v", trial)
 707
 708                 if trial.anonToken {
 709                         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 710                 } else {
 711                         s.handler.Cluster.Users.AnonymousUserToken = ""
 712                 }
 713                 req, err := http.NewRequest("GET", "http://"+arvadostest.FooCollection+".example.com/foo", nil)
 714                 c.Assert(err, check.IsNil)
 715                 req.Header.Set("Sec-Fetch-Mode", "navigate")
 716                 if trial.cacheControl != "" {
 717                         req.Header.Set("Cache-Control", trial.cacheControl)
 718                 }
 719                 resp := httptest.NewRecorder()
 720                 s.handler.ServeHTTP(resp, req)
 721                 c.Check(resp.Code, check.Equals, http.StatusSeeOther)
 722                 u, err := url.Parse(resp.Header().Get("Location"))
 723                 c.Assert(err, check.IsNil)
 724                 c.Logf("redirected to %q", u)
 725                 c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 726                 c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 727                 c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
 728         }
 729 }
 730
 731 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
 732         s.testVhostRedirectTokenToCookie(c, "GET",
 733                 "example.com/c="+arvadostest.FooCollection+"/foo",
 734                 "?api_token="+arvadostest.ActiveToken,
 735                 nil,
 736                 "",
 737                 http.StatusBadRequest,
 738                 regexp.QuoteMeta("cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n"),
 739         )
 740 }
 741
 742 // If client requests an attachment by putting ?disposition=attachment
 743 // in the query string, and gets redirected, the redirect target
 744 // should respond with an attachment.
 745 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
 746         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 747                 arvadostest.FooCollection+".example.com/foo",
 748                 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
 749                 nil,
 750                 "",
 751                 http.StatusOK,
 752                 "foo",
 753         )
 754         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 755 }
 756
 757 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
 758         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 759         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 760                 "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
 761                 "?api_token="+arvadostest.ActiveToken,
 762                 nil,
 763                 "",
 764                 http.StatusOK,
 765                 "foo",
 766         )
 767         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 768 }
 769
 770 func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
 771         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 772         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 773                 "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
 774                 "?api_token="+arvadostest.ActiveToken,
 775                 nil,
 776                 "",
 777                 http.StatusOK,
 778                 "waz",
 779         )
 780         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 781         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 782                 "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz",
 783                 "?api_token="+arvadostest.ActiveToken,
 784                 nil,
 785                 "",
 786                 http.StatusOK,
 787                 "waz",
 788         )
 789         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 790 }
 791
 792 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
 793         s.handler.Cluster.Collections.TrustAllContent = true
 794         s.testVhostRedirectTokenToCookie(c, "GET",
 795                 "example.com/c="+arvadostest.FooCollection+"/foo",
 796                 "?api_token="+arvadostest.ActiveToken,
 797                 nil,
 798                 "",
 799                 http.StatusOK,
 800                 "foo",
 801         )
 802 }
 803
 804 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
 805         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
 806
 807         s.testVhostRedirectTokenToCookie(c, "GET",
 808                 "example.com/c="+arvadostest.FooCollection+"/foo",
 809                 "?api_token="+arvadostest.ActiveToken,
 810                 nil,
 811                 "",
 812                 http.StatusBadRequest,
 813                 regexp.QuoteMeta("cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n"),
 814         )
 815
 816         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 817                 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
 818                 "?api_token="+arvadostest.ActiveToken,
 819                 nil,
 820                 "",
 821                 http.StatusOK,
 822                 "foo",
 823         )
 824         c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
 825 }
 826
 827 func (s *IntegrationSuite) TestVhostRedirectMultipleTokens(c *check.C) {
 828         baseUrl := arvadostest.FooCollection + ".example.com/foo"
 829         query := url.Values{}
 830
 831         // The intent of these tests is to check that requests are redirected
 832         // correctly in the presence of multiple API tokens. The exact response
 833         // codes and content are not closely considered: they're just how
 834         // keep-web responded when we made the smallest possible fix. Changing
 835         // those responses may be okay, but you should still test all these
 836         // different cases and the associated redirect logic.
 837         query["api_token"] = []string{arvadostest.ActiveToken, arvadostest.AnonymousToken}
 838         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
 839         query["api_token"] = []string{arvadostest.ActiveToken, arvadostest.AnonymousToken, ""}
 840         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
 841         query["api_token"] = []string{arvadostest.ActiveToken, "", arvadostest.AnonymousToken}
 842         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
 843         query["api_token"] = []string{"", arvadostest.ActiveToken}
 844         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
 845
 846         expectContent := regexp.QuoteMeta(unauthorizedMessage + "\n")
 847         query["api_token"] = []string{arvadostest.AnonymousToken, "invalidtoo"}
 848         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusUnauthorized, expectContent)
 849         query["api_token"] = []string{arvadostest.AnonymousToken, ""}
 850         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusUnauthorized, expectContent)
 851         query["api_token"] = []string{"", arvadostest.AnonymousToken}
 852         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusUnauthorized, expectContent)
 853 }
 854
 855 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
 856         s.testVhostRedirectTokenToCookie(c, "POST",
 857                 arvadostest.FooCollection+".example.com/foo",
 858                 "",
 859                 http.Header{"Content-Type": {"application/x-www-form-urlencoded"}},
 860                 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
 861                 http.StatusOK,
 862                 "foo",
 863         )
 864 }
 865
 866 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
 867         s.testVhostRedirectTokenToCookie(c, "POST",
 868                 arvadostest.FooCollection+".example.com/foo",
 869                 "",
 870                 http.Header{"Content-Type": {"application/x-www-form-urlencoded"}},
 871                 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
 872                 http.StatusNotFound,
 873                 regexp.QuoteMeta(notFoundMessage+"\n"),
 874         )
 875 }
 876
 877 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
 878         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 879         s.testVhostRedirectTokenToCookie(c, "GET",
 880                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 881                 "",
 882                 nil,
 883                 "",
 884                 http.StatusOK,
 885                 "Hello world\n",
 886         )
 887 }
 888
 889 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
 890         s.handler.Cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
 891         s.testVhostRedirectTokenToCookie(c, "GET",
 892                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 893                 "",
 894                 nil,
 895                 "",
 896                 http.StatusUnauthorized,
 897                 "Authorization tokens are not accepted here: .*\n",
 898         )
 899 }
 900
 901 func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
 902         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 903
 904         client := arvados.NewClientFromEnv()
 905         client.AuthToken = arvadostest.ActiveToken
 906         fs, err := (&arvados.Collection{}).FileSystem(client, nil)
 907         c.Assert(err, check.IsNil)
 908         path := `https:\\"odd' path chars`
 909         f, err := fs.OpenFile(path, os.O_CREATE, 0777)
 910         c.Assert(err, check.IsNil)
 911         f.Close()
 912         mtxt, err := fs.MarshalManifest(".")
 913         c.Assert(err, check.IsNil)
 914         var coll arvados.Collection
 915         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 916                 "collection": map[string]string{
 917                         "manifest_text": mtxt,
 918                 },
 919         })
 920         c.Assert(err, check.IsNil)
 921
 922         u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
 923         req := &http.Request{
 924                 Method:     "GET",
 925                 Host:       u.Host,
 926                 URL:        u,
 927                 RequestURI: u.RequestURI(),
 928                 Header: http.Header{
 929                         "Authorization": {"Bearer " + client.AuthToken},
 930                 },
 931         }
 932         resp := httptest.NewRecorder()
 933         s.handler.ServeHTTP(resp, req)
 934         c.Check(resp.Code, check.Equals, http.StatusOK)
 935         doc, err := html.Parse(resp.Body)
 936         c.Assert(err, check.IsNil)
 937         pathHrefMap := getPathHrefMap(doc)
 938         c.Check(pathHrefMap, check.HasLen, 1) // the one leaf added to collection
 939         href, hasPath := pathHrefMap[path]
 940         c.Assert(hasPath, check.Equals, true) // the path is listed
 941         relUrl := mustParseURL(href)
 942         c.Check(relUrl.Path, check.Equals, "./"+path) // href can be decoded back to path
 943 }
 944
 945 func (s *IntegrationSuite) TestForwardSlashSubstitution(c *check.C) {
 946         arv := arvados.NewClientFromEnv()
 947         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 948         s.handler.Cluster.Collections.ForwardSlashNameSubstitution = "{SOLIDUS}"
 949         name := "foo/bar/baz"
 950         nameShown := strings.Replace(name, "/", "{SOLIDUS}", -1)
 951
 952         client := arvados.NewClientFromEnv()
 953         client.AuthToken = arvadostest.ActiveToken
 954         fs, err := (&arvados.Collection{}).FileSystem(client, nil)
 955         c.Assert(err, check.IsNil)
 956         f, err := fs.OpenFile("filename", os.O_CREATE, 0777)
 957         c.Assert(err, check.IsNil)
 958         f.Close()
 959         mtxt, err := fs.MarshalManifest(".")
 960         c.Assert(err, check.IsNil)
 961         var coll arvados.Collection
 962         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 963                 "collection": map[string]string{
 964                         "manifest_text": mtxt,
 965                         "name":          name,
 966                         "owner_uuid":    arvadostest.AProjectUUID,
 967                 },
 968         })
 969         c.Assert(err, check.IsNil)
 970         defer arv.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
 971
 972         base := "http://download.example.com/by_id/" + coll.OwnerUUID + "/"
 973         for tryURL, expectedAnchorText := range map[string]string{
 974                 base:                   nameShown + "/",
 975                 base + nameShown + "/": "filename",
 976         } {
 977                 u, _ := url.Parse(tryURL)
 978                 req := &http.Request{
 979                         Method:     "GET",
 980                         Host:       u.Host,
 981                         URL:        u,
 982                         RequestURI: u.RequestURI(),
 983                         Header: http.Header{
 984                                 "Authorization": {"Bearer " + client.AuthToken},
 985                         },
 986                 }
 987                 resp := httptest.NewRecorder()
 988                 s.handler.ServeHTTP(resp, req)
 989                 c.Check(resp.Code, check.Equals, http.StatusOK)
 990                 doc, err := html.Parse(resp.Body)
 991                 c.Assert(err, check.IsNil) // valid HTML
 992                 pathHrefMap := getPathHrefMap(doc)
 993                 href, hasExpected := pathHrefMap[expectedAnchorText]
 994                 c.Assert(hasExpected, check.Equals, true) // has expected anchor text
 995                 c.Assert(href, check.Not(check.Equals), "")
 996                 relUrl := mustParseURL(href)
 997                 c.Check(relUrl.Path, check.Equals, "./"+expectedAnchorText) // decoded href maps back to the anchor text
 998         }
 999 }
1000
1001 // XHRs can't follow redirect-with-cookie so they rely on method=POST
1002 // and disposition=attachment (telling us it's acceptable to respond
1003 // with content instead of a redirect) and an Origin header that gets
1004 // added automatically by the browser (telling us it's desirable to do
1005 // so).
1006 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
1007         u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
1008         req := &http.Request{
1009                 Method:     "POST",
1010                 Host:       u.Host,
1011                 URL:        u,
1012                 RequestURI: u.RequestURI(),
1013                 Header: http.Header{
1014                         "Origin":       {"https://origin.example"},
1015                         "Content-Type": {"application/x-www-form-urlencoded"},
1016                 },
1017                 Body: ioutil.NopCloser(strings.NewReader(url.Values{
1018                         "api_token":   {arvadostest.ActiveToken},
1019                         "disposition": {"attachment"},
1020                 }.Encode())),
1021         }
1022         resp := httptest.NewRecorder()
1023         s.handler.ServeHTTP(resp, req)
1024         c.Check(resp.Code, check.Equals, http.StatusOK)
1025         c.Check(resp.Body.String(), check.Equals, "foo")
1026         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
1027
1028         // GET + Origin header is representative of both AJAX GET
1029         // requests and inline images via <IMG crossorigin="anonymous"
1030         // src="...">.
1031         u.RawQuery = "api_token=" + url.QueryEscape(arvadostest.ActiveTokenV2)
1032         req = &http.Request{
1033                 Method:     "GET",
1034                 Host:       u.Host,
1035                 URL:        u,
1036                 RequestURI: u.RequestURI(),
1037                 Header: http.Header{
1038                         "Origin": {"https://origin.example"},
1039                 },
1040         }
1041         resp = httptest.NewRecorder()
1042         s.handler.ServeHTTP(resp, req)
1043         c.Check(resp.Code, check.Equals, http.StatusOK)
1044         c.Check(resp.Body.String(), check.Equals, "foo")
1045         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
1046 }
1047
1048 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString string, reqHeader http.Header, reqBody string, expectStatus int, matchRespBody string) *httptest.ResponseRecorder {
1049         if reqHeader == nil {
1050                 reqHeader = http.Header{}
1051         }
1052         u, _ := url.Parse(`http://` + hostPath + queryString)
1053         c.Logf("requesting %s", u)
1054         req := &http.Request{
1055                 Method:     method,
1056                 Host:       u.Host,
1057                 URL:        u,
1058                 RequestURI: u.RequestURI(),
1059                 Header:     reqHeader,
1060                 Body:       ioutil.NopCloser(strings.NewReader(reqBody)),
1061         }
1062
1063         resp := httptest.NewRecorder()
1064         defer func() {
1065                 c.Check(resp.Code, check.Equals, expectStatus)
1066                 c.Check(resp.Body.String(), check.Matches, matchRespBody)
1067         }()
1068
1069         s.handler.ServeHTTP(resp, req)
1070         if resp.Code != http.StatusSeeOther {
1071                 attachment, _ := regexp.MatchString(`^attachment(;|$)`, resp.Header().Get("Content-Disposition"))
1072                 // Since we're not redirecting, check that any api_token in the URL is
1073                 // handled safely.
1074                 // If there is no token in the URL, then we're good.
1075                 // Otherwise, if the response code is an error, the body is expected to
1076                 // be static content, and nothing that might maliciously introspect the
1077                 // URL. It's considered safe and allowed.
1078                 // Otherwise, if the response content has attachment disposition,
1079                 // that's considered safe for all the reasons explained in the
1080                 // safeAttachment comment in handler.go.
1081                 c.Check(!u.Query().Has("api_token") || resp.Code >= 400 || attachment, check.Equals, true)
1082                 return resp
1083         }
1084
1085         loc, err := url.Parse(resp.Header().Get("Location"))
1086         c.Assert(err, check.IsNil)
1087         c.Check(loc.Scheme, check.Equals, u.Scheme)
1088         c.Check(loc.Host, check.Equals, u.Host)
1089         c.Check(loc.RawPath, check.Equals, u.RawPath)
1090         // If the response was a redirect, it should never include an API token.
1091         c.Check(loc.Query().Has("api_token"), check.Equals, false)
1092         c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
1093         cookies := (&http.Response{Header: resp.Header()}).Cookies()
1094
1095         c.Logf("following redirect to %s", u)
1096         req = &http.Request{
1097                 Method:     "GET",
1098                 Host:       loc.Host,
1099                 URL:        loc,
1100                 RequestURI: loc.RequestURI(),
1101                 Header:     reqHeader,
1102         }
1103         for _, c := range cookies {
1104                 req.AddCookie(c)
1105         }
1106
1107         resp = httptest.NewRecorder()
1108         s.handler.ServeHTTP(resp, req)
1109
1110         if resp.Code != http.StatusSeeOther {
1111                 c.Check(resp.Header().Get("Location"), check.Equals, "")
1112         }
1113         return resp
1114 }
1115
1116 func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) {
1117         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
1118         s.testDirectoryListing(c)
1119 }
1120
1121 func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) {
1122         s.handler.Cluster.Users.AnonymousUserToken = ""
1123         s.testDirectoryListing(c)
1124 }
1125
1126 func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
1127         // The "ownership cycle" test fixtures are reachable from the
1128         // "filter group without filters" group, causing webdav's
1129         // walkfs to recurse indefinitely. Avoid that by deleting one
1130         // of the bogus fixtures.
1131         arv := arvados.NewClientFromEnv()
1132         err := arv.RequestAndDecode(nil, "DELETE", "arvados/v1/groups/zzzzz-j7d0g-cx2al9cqkmsf1hs", nil, nil)
1133         if err != nil {
1134                 c.Assert(err, check.FitsTypeOf, &arvados.TransactionError{})
1135                 c.Check(err.(*arvados.TransactionError).StatusCode, check.Equals, 404)
1136         }
1137
1138         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1139         authHeader := http.Header{
1140                 "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
1141         }
1142         for _, trial := range []struct {
1143                 uri      string
1144                 header   http.Header
1145                 expect   []string
1146                 redirect string
1147                 cutDirs  int
1148         }{
1149                 {
1150                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
1151                         header:  authHeader,
1152                         expect:  []string{"dir1/foo", "dir1/bar"},
1153                         cutDirs: 0,
1154                 },
1155                 {
1156                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
1157                         header:  authHeader,
1158                         expect:  []string{"foo", "bar"},
1159                         cutDirs: 1,
1160                 },
1161                 {
1162                         // URLs of this form ignore authHeader, and
1163                         // FooAndBarFilesInDirUUID isn't public, so
1164                         // this returns 401.
1165                         uri:    "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
1166                         header: authHeader,
1167                         expect: nil,
1168                 },
1169                 {
1170                         uri:     "download.example.com/users/active/foo_file_in_dir/",
1171                         header:  authHeader,
1172                         expect:  []string{"dir1/"},
1173                         cutDirs: 3,
1174                 },
1175                 {
1176                         uri:     "download.example.com/users/active/foo_file_in_dir/dir1/",
1177                         header:  authHeader,
1178                         expect:  []string{"bar"},
1179                         cutDirs: 4,
1180                 },
1181                 {
1182                         uri:     "download.example.com/",
1183                         header:  authHeader,
1184                         expect:  []string{"users/"},
1185                         cutDirs: 0,
1186                 },
1187                 {
1188                         uri:      "download.example.com/users",
1189                         header:   authHeader,
1190                         redirect: "/users/",
1191                         expect:   []string{"active/"},
1192                         cutDirs:  1,
1193                 },
1194                 {
1195                         uri:     "download.example.com/users/",
1196                         header:  authHeader,
1197                         expect:  []string{"active/"},
1198                         cutDirs: 1,
1199                 },
1200                 {
1201                         uri:      "download.example.com/users/active",
1202                         header:   authHeader,
1203                         redirect: "/users/active/",
1204                         expect:   []string{"foo_file_in_dir/"},
1205                         cutDirs:  2,
1206                 },
1207                 {
1208                         uri:     "download.example.com/users/active/",
1209                         header:  authHeader,
1210                         expect:  []string{"foo_file_in_dir/"},
1211                         cutDirs: 2,
1212                 },
1213                 {
1214                         uri:     "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
1215                         header:  nil,
1216                         expect:  []string{"dir1/foo", "dir1/bar"},
1217                         cutDirs: 4,
1218                 },
1219                 {
1220                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
1221                         header:  nil,
1222                         expect:  []string{"dir1/foo", "dir1/bar"},
1223                         cutDirs: 2,
1224                 },
1225                 {
1226                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken,
1227                         header:  nil,
1228                         expect:  []string{"dir1/foo", "dir1/bar"},
1229                         cutDirs: 2,
1230                 },
1231                 {
1232                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID,
1233                         header:  authHeader,
1234                         expect:  []string{"dir1/foo", "dir1/bar"},
1235                         cutDirs: 1,
1236                 },
1237                 {
1238                         uri:      "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
1239                         header:   authHeader,
1240                         redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
1241                         expect:   []string{"foo", "bar"},
1242                         cutDirs:  2,
1243                 },
1244                 {
1245                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
1246                         header:  authHeader,
1247                         expect:  []string{"foo", "bar"},
1248                         cutDirs: 3,
1249                 },
1250                 {
1251                         uri:      arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
1252                         header:   authHeader,
1253                         redirect: "/dir1/",
1254                         expect:   []string{"foo", "bar"},
1255                         cutDirs:  1,
1256                 },
1257                 {
1258                         uri:    "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
1259                         header: authHeader,
1260                         expect: nil,
1261                 },
1262                 {
1263                         uri:     "download.example.com/c=" + arvadostest.WazVersion1Collection,
1264                         header:  authHeader,
1265                         expect:  []string{"waz"},
1266                         cutDirs: 1,
1267                 },
1268                 {
1269                         uri:     "download.example.com/by_id/" + arvadostest.WazVersion1Collection,
1270                         header:  authHeader,
1271                         expect:  []string{"waz"},
1272                         cutDirs: 2,
1273                 },
1274                 {
1275                         uri:     "download.example.com/users/active/This filter group/",
1276                         header:  authHeader,
1277                         expect:  []string{"A Subproject/"},
1278                         cutDirs: 3,
1279                 },
1280                 {
1281                         uri:     "download.example.com/users/active/This filter group/A Subproject",
1282                         header:  authHeader,
1283                         expect:  []string{"baz_file/"},
1284                         cutDirs: 4,
1285                 },
1286                 {
1287                         uri:     "download.example.com/by_id/" + arvadostest.AFilterGroupUUID,
1288                         header:  authHeader,
1289                         expect:  []string{"A Subproject/"},
1290                         cutDirs: 2,
1291                 },
1292                 {
1293                         uri:     "download.example.com/by_id/" + arvadostest.AFilterGroupUUID + "/A Subproject",
1294                         header:  authHeader,
1295                         expect:  []string{"baz_file/"},
1296                         cutDirs: 3,
1297                 },
1298         } {
1299                 comment := check.Commentf("HTML: %q redir %q => %q", trial.uri, trial.redirect, trial.expect)
1300                 resp := httptest.NewRecorder()
1301                 u := mustParseURL("//" + trial.uri)
1302                 req := &http.Request{
1303                         Method:     "GET",
1304                         Host:       u.Host,
1305                         URL:        u,
1306                         RequestURI: u.RequestURI(),
1307                         Header:     copyHeader(trial.header),
1308                 }
1309                 s.handler.ServeHTTP(resp, req)
1310                 var cookies []*http.Cookie
1311                 for resp.Code == http.StatusSeeOther {
1312                         u, _ := req.URL.Parse(resp.Header().Get("Location"))
1313                         req = &http.Request{
1314                                 Method:     "GET",
1315                                 Host:       u.Host,
1316                                 URL:        u,
1317                                 RequestURI: u.RequestURI(),
1318                                 Header:     copyHeader(trial.header),
1319                         }
1320                         cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
1321                         for _, c := range cookies {
1322                                 req.AddCookie(c)
1323                         }
1324                         resp = httptest.NewRecorder()
1325                         s.handler.ServeHTTP(resp, req)
1326                 }
1327                 if trial.redirect != "" {
1328                         c.Check(req.URL.Path, check.Equals, trial.redirect, comment)
1329                 }
1330                 if trial.expect == nil {
1331                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1332                 } else {
1333                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
1334                         listingPageDoc, err := html.Parse(resp.Body)
1335                         c.Check(err, check.IsNil, comment) // valid HTML document
1336                         pathHrefMap := getPathHrefMap(listingPageDoc)
1337                         c.Assert(pathHrefMap, check.Not(check.HasLen), 0, comment)
1338                         for _, e := range trial.expect {
1339                                 href, hasE := pathHrefMap[e]
1340                                 c.Check(hasE, check.Equals, true, comment) // expected path is listed
1341                                 relUrl := mustParseURL(href)
1342                                 c.Check(relUrl.Path, check.Equals, "./"+e, comment) // href can be decoded back to path
1343                         }
1344                         wgetCommand := getWgetExamplePre(listingPageDoc)
1345                         wgetExpected := regexp.MustCompile(`^\$ wget .*--cut-dirs=(\d+) .*'(https?://[^']+)'$`)
1346                         wgetMatchGroups := wgetExpected.FindStringSubmatch(wgetCommand)
1347                         c.Assert(wgetMatchGroups, check.NotNil)                                     // wget command matches
1348                         c.Check(wgetMatchGroups[1], check.Equals, fmt.Sprintf("%d", trial.cutDirs)) // correct level of cut dirs in wget command
1349                         printedUrl := mustParseURL(wgetMatchGroups[2])
1350                         c.Check(printedUrl.Host, check.Equals, req.URL.Host)
1351                         c.Check(printedUrl.Path, check.Equals, req.URL.Path) // URL arg in wget command can be decoded to the right path
1352                 }
1353
1354                 comment = check.Commentf("WebDAV: %q => %q", trial.uri, trial.expect)
1355                 req = &http.Request{
1356                         Method:     "OPTIONS",
1357                         Host:       u.Host,
1358                         URL:        u,
1359                         RequestURI: u.RequestURI(),
1360                         Header:     copyHeader(trial.header),
1361                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
1362                 }
1363                 resp = httptest.NewRecorder()
1364                 s.handler.ServeHTTP(resp, req)
1365                 if trial.expect == nil {
1366                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1367                 } else {
1368                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
1369                 }
1370
1371                 req = &http.Request{
1372                         Method:     "PROPFIND",
1373                         Host:       u.Host,
1374                         URL:        u,
1375                         RequestURI: u.RequestURI(),
1376                         Header:     copyHeader(trial.header),
1377                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
1378                 }
1379                 resp = httptest.NewRecorder()
1380                 s.handler.ServeHTTP(resp, req)
1381                 // This check avoids logging a big XML document in the
1382                 // event webdav throws a 500 error after sending
1383                 // headers for a 207.
1384                 if !c.Check(strings.HasSuffix(resp.Body.String(), "Internal Server Error"), check.Equals, false) {
1385                         continue
1386                 }
1387                 if trial.expect == nil {
1388                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1389                 } else {
1390                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment)
1391                         for _, e := range trial.expect {
1392                                 if strings.HasSuffix(e, "/") {
1393                                         e = filepath.Join(u.Path, e) + "/"
1394                                 } else {
1395                                         e = filepath.Join(u.Path, e)
1396                                 }
1397                                 e = strings.Replace(e, " ", "%20", -1)
1398                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+e+`</D:href>.*`, comment)
1399                         }
1400                 }
1401         }
1402 }
1403
1404 // Shallow-traverse the HTML document, gathering the nodes satisfying the
1405 // predicate function in the output slice. If a node matches the predicate,
1406 // none of its children will be visited.
1407 func getNodes(document *html.Node, predicate func(*html.Node) bool) []*html.Node {
1408         var acc []*html.Node
1409         var traverse func(*html.Node, []*html.Node) []*html.Node
1410         traverse = func(root *html.Node, sofar []*html.Node) []*html.Node {
1411                 if root == nil {
1412                         return sofar
1413                 }
1414                 if predicate(root) {
1415                         return append(sofar, root)
1416                 }
1417                 for cur := root.FirstChild; cur != nil; cur = cur.NextSibling {
1418                         sofar = traverse(cur, sofar)
1419                 }
1420                 return sofar
1421         }
1422         return traverse(document, acc)
1423 }
1424
1425 // Returns true if a node has the attribute targetAttr with the given value
1426 func matchesAttributeValue(node *html.Node, targetAttr string, value string) bool {
1427         for _, attr := range node.Attr {
1428                 if attr.Key == targetAttr && attr.Val == value {
1429                         return true
1430                 }
1431         }
1432         return false
1433 }
1434
1435 // Concatenate the content of text-node children of node; only direct
1436 // children are visited, and any non-text children are skipped.
1437 func getNodeText(node *html.Node) string {
1438         var recv strings.Builder
1439         for c := node.FirstChild; c != nil; c = c.NextSibling {
1440                 if c.Type == html.TextNode {
1441                         recv.WriteString(c.Data)
1442                 }
1443         }
1444         return recv.String()
1445 }
1446
1447 // Returns a map from the directory listing item string (a path) to the href
1448 // value of its <a> tag (an encoded relative URL)
1449 func getPathHrefMap(document *html.Node) map[string]string {
1450         isItemATag := func(node *html.Node) bool {
1451                 return node.Type == html.ElementNode && node.Data == "a" && matchesAttributeValue(node, "class", "item")
1452         }
1453         aTags := getNodes(document, isItemATag)
1454         output := make(map[string]string)
1455         for _, elem := range aTags {
1456                 textContent := getNodeText(elem)
1457                 for _, attr := range elem.Attr {
1458                         if attr.Key == "href" {
1459                                 output[textContent] = attr.Val
1460                                 break
1461                         }
1462                 }
1463         }
1464         return output
1465 }
1466
1467 func getWgetExamplePre(document *html.Node) string {
1468         isWgetPre := func(node *html.Node) bool {
1469                 return node.Type == html.ElementNode && matchesAttributeValue(node, "id", "wget-example")
1470         }
1471         elements := getNodes(document, isWgetPre)
1472         if len(elements) != 1 {
1473                 return ""
1474         }
1475         return getNodeText(elements[0])
1476 }
1477
1478 func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
1479         arv := arvados.NewClientFromEnv()
1480         var newCollection arvados.Collection
1481         err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1482                 "collection": map[string]string{
1483                         "owner_uuid":    arvadostest.ActiveUserUUID,
1484                         "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
1485                         "name":          "keep-web test collection",
1486                 },
1487                 "ensure_unique_name": true,
1488         })
1489         c.Assert(err, check.IsNil)
1490         defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
1491
1492         var updated arvados.Collection
1493         for _, fnm := range []string{"foo.txt", "bar.txt"} {
1494                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
1495                 u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
1496                 req := &http.Request{
1497                         Method:     "DELETE",
1498                         Host:       u.Host,
1499                         URL:        u,
1500                         RequestURI: u.RequestURI(),
1501                         Header: http.Header{
1502                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
1503                         },
1504                 }
1505                 resp := httptest.NewRecorder()
1506                 s.handler.ServeHTTP(resp, req)
1507                 c.Check(resp.Code, check.Equals, http.StatusNoContent)
1508
1509                 updated = arvados.Collection{}
1510                 err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
1511                 c.Check(err, check.IsNil)
1512                 c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`)
1513                 c.Logf("updated manifest_text %q", updated.ManifestText)
1514         }
1515         c.Check(updated.ManifestText, check.Equals, "")
1516 }
1517
1518 func (s *IntegrationSuite) TestFileContentType(c *check.C) {
1519         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1520
1521         client := arvados.NewClientFromEnv()
1522         client.AuthToken = arvadostest.ActiveToken
1523         arv, err := arvadosclient.New(client)
1524         c.Assert(err, check.Equals, nil)
1525         kc, err := keepclient.MakeKeepClient(arv)
1526         c.Assert(err, check.Equals, nil)
1527
1528         fs, err := (&arvados.Collection{}).FileSystem(client, kc)
1529         c.Assert(err, check.IsNil)
1530
1531         trials := []struct {
1532                 filename    string
1533                 content     string
1534                 contentType string
1535         }{
1536                 {"picture.txt", "BMX bikes are small this year\n", "text/plain; charset=utf-8"},
1537                 {"picture.bmp", "BMX bikes are small this year\n", "image/(x-ms-)?bmp"},
1538                 {"picture.jpg", "BMX bikes are small this year\n", "image/jpeg"},
1539                 {"picture1", "BMX bikes are small this year\n", "image/bmp"},            // content sniff; "BM" is the magic signature for .bmp
1540                 {"picture2", "Cars are small this year\n", "text/plain; charset=utf-8"}, // content sniff
1541         }
1542         for _, trial := range trials {
1543                 f, err := fs.OpenFile(trial.filename, os.O_CREATE|os.O_WRONLY, 0777)
1544                 c.Assert(err, check.IsNil)
1545                 _, err = f.Write([]byte(trial.content))
1546                 c.Assert(err, check.IsNil)
1547                 c.Assert(f.Close(), check.IsNil)
1548         }
1549         mtxt, err := fs.MarshalManifest(".")
1550         c.Assert(err, check.IsNil)
1551         var coll arvados.Collection
1552         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1553                 "collection": map[string]string{
1554                         "manifest_text": mtxt,
1555                 },
1556         })
1557         c.Assert(err, check.IsNil)
1558
1559         for _, trial := range trials {
1560                 u, _ := url.Parse("http://download.example.com/by_id/" + coll.UUID + "/" + trial.filename)
1561                 req := &http.Request{
1562                         Method:     "GET",
1563                         Host:       u.Host,
1564                         URL:        u,
1565                         RequestURI: u.RequestURI(),
1566                         Header: http.Header{
1567                                 "Authorization": {"Bearer " + client.AuthToken},
1568                         },
1569                 }
1570                 resp := httptest.NewRecorder()
1571                 s.handler.ServeHTTP(resp, req)
1572                 c.Check(resp.Code, check.Equals, http.StatusOK)
1573                 c.Check(resp.Header().Get("Content-Type"), check.Matches, trial.contentType)
1574                 c.Check(resp.Body.String(), check.Equals, trial.content)
1575         }
1576 }
1577
1578 func (s *IntegrationSuite) TestCacheSize(c *check.C) {
1579         req, err := http.NewRequest("GET", "http://"+arvadostest.FooCollection+".example.com/foo", nil)
1580         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
1581         c.Assert(err, check.IsNil)
1582         resp := httptest.NewRecorder()
1583         s.handler.ServeHTTP(resp, req)
1584         c.Assert(resp.Code, check.Equals, http.StatusOK)
1585         c.Check(s.handler.Cache.sessions[arvadostest.ActiveTokenV2].client.DiskCacheSize.Percent(), check.Equals, int64(10))
1586 }
1587
1588 // Writing to a collection shouldn't affect its entry in the
1589 // PDH-to-manifest cache.
1590 func (s *IntegrationSuite) TestCacheWriteCollectionSamePDH(c *check.C) {
1591         arv, err := arvadosclient.MakeArvadosClient()
1592         c.Assert(err, check.Equals, nil)
1593         arv.ApiToken = arvadostest.ActiveToken
1594
1595         u := mustParseURL("http://x.example/testfile")
1596         req := &http.Request{
1597                 Method:     "GET",
1598                 Host:       u.Host,
1599                 URL:        u,
1600                 RequestURI: u.RequestURI(),
1601                 Header:     http.Header{"Authorization": {"Bearer " + arv.ApiToken}},
1602         }
1603
1604         checkWithID := func(id string, status int) {
1605                 req.URL.Host = strings.Replace(id, "+", "-", -1) + ".example"
1606                 req.Host = req.URL.Host
1607                 resp := httptest.NewRecorder()
1608                 s.handler.ServeHTTP(resp, req)
1609                 c.Check(resp.Code, check.Equals, status)
1610         }
1611
1612         var colls [2]arvados.Collection
1613         for i := range colls {
1614                 err := arv.Create("collections",
1615                         map[string]interface{}{
1616                                 "ensure_unique_name": true,
1617                                 "collection": map[string]interface{}{
1618                                         "name": "test collection",
1619                                 },
1620                         }, &colls[i])
1621                 c.Assert(err, check.Equals, nil)
1622         }
1623
1624         // Populate cache with empty collection
1625         checkWithID(colls[0].PortableDataHash, http.StatusNotFound)
1626
1627         // write a file to colls[0]
1628         reqPut := *req
1629         reqPut.Method = "PUT"
1630         reqPut.URL.Host = colls[0].UUID + ".example"
1631         reqPut.Host = req.URL.Host
1632         reqPut.Body = ioutil.NopCloser(bytes.NewBufferString("testdata"))
1633         resp := httptest.NewRecorder()
1634         s.handler.ServeHTTP(resp, &reqPut)
1635         c.Check(resp.Code, check.Equals, http.StatusCreated)
1636
1637         // new file should not appear in colls[1]
1638         checkWithID(colls[1].PortableDataHash, http.StatusNotFound)
1639         checkWithID(colls[1].UUID, http.StatusNotFound)
1640
1641         checkWithID(colls[0].UUID, http.StatusOK)
1642 }
1643
1644 func copyHeader(h http.Header) http.Header {
1645         hc := http.Header{}
1646         for k, v := range h {
1647                 hc[k] = append([]string(nil), v...)
1648         }
1649         return hc
1650 }
1651
1652 func (s *IntegrationSuite) checkUploadDownloadRequest(c *check.C, req *http.Request,
1653         successCode int, direction string, perm bool, userUuid, collectionUuid, collectionPDH, filepath string) {
1654
1655         client := arvados.NewClientFromEnv()
1656         client.AuthToken = arvadostest.AdminToken
1657         var logentries arvados.LogList
1658         limit1 := 1
1659         err := client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil,
1660                 arvados.ResourceListParams{
1661                         Limit: &limit1,
1662                         Order: "created_at desc"})
1663         c.Check(err, check.IsNil)
1664         c.Check(logentries.Items, check.HasLen, 1)
1665         lastLogId := logentries.Items[0].ID
1666         c.Logf("lastLogId: %d", lastLogId)
1667
1668         var logbuf bytes.Buffer
1669         logger := logrus.New()
1670         logger.Out = &logbuf
1671         resp := httptest.NewRecorder()
1672         req = req.WithContext(ctxlog.Context(context.Background(), logger))
1673         s.handler.ServeHTTP(resp, req)
1674
1675         if perm {
1676                 c.Check(resp.Result().StatusCode, check.Equals, successCode)
1677                 c.Check(logbuf.String(), check.Matches, `(?ms).*msg="File `+direction+`".*`)
1678                 c.Check(logbuf.String(), check.Not(check.Matches), `(?ms).*level=error.*`)
1679
1680                 deadline := time.Now().Add(time.Second)
1681                 for {
1682                         c.Assert(time.Now().After(deadline), check.Equals, false, check.Commentf("timed out waiting for log entry"))
1683                         logentries = arvados.LogList{}
1684                         err = client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil,
1685                                 arvados.ResourceListParams{
1686                                         Filters: []arvados.Filter{
1687                                                 {Attr: "event_type", Operator: "=", Operand: "file_" + direction},
1688                                                 {Attr: "object_uuid", Operator: "=", Operand: userUuid},
1689                                         },
1690                                         Limit: &limit1,
1691                                         Order: "created_at desc",
1692                                 })
1693                         c.Assert(err, check.IsNil)
1694                         if len(logentries.Items) > 0 &&
1695                                 logentries.Items[0].ID > lastLogId &&
1696                                 logentries.Items[0].ObjectUUID == userUuid &&
1697                                 logentries.Items[0].Properties["collection_uuid"] == collectionUuid &&
1698                                 (collectionPDH == "" || logentries.Items[0].Properties["portable_data_hash"] == collectionPDH) &&
1699                                 logentries.Items[0].Properties["collection_file_path"] == filepath {
1700                                 break
1701                         }
1702                         c.Logf("logentries.Items: %+v", logentries.Items)
1703                         time.Sleep(50 * time.Millisecond)
1704                 }
1705         } else {
1706                 c.Check(resp.Result().StatusCode, check.Equals, http.StatusForbidden)
1707                 c.Check(logbuf.String(), check.Equals, "")
1708         }
1709 }
1710
1711 func (s *IntegrationSuite) TestDownloadLoggingPermission(c *check.C) {
1712         u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
1713
1714         s.handler.Cluster.Collections.TrustAllContent = true
1715
1716         for _, adminperm := range []bool{true, false} {
1717                 for _, userperm := range []bool{true, false} {
1718                         s.handler.Cluster.Collections.WebDAVPermission.Admin.Download = adminperm
1719                         s.handler.Cluster.Collections.WebDAVPermission.User.Download = userperm
1720
1721                         // Test admin permission
1722                         req := &http.Request{
1723                                 Method:     "GET",
1724                                 Host:       u.Host,
1725                                 URL:        u,
1726                                 RequestURI: u.RequestURI(),
1727                                 Header: http.Header{
1728                                         "Authorization": {"Bearer " + arvadostest.AdminToken},
1729                                 },
1730                         }
1731                         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", adminperm,
1732                                 arvadostest.AdminUserUUID, arvadostest.FooCollection, arvadostest.FooCollectionPDH, "foo")
1733
1734                         // Test user permission
1735                         req = &http.Request{
1736                                 Method:     "GET",
1737                                 Host:       u.Host,
1738                                 URL:        u,
1739                                 RequestURI: u.RequestURI(),
1740                                 Header: http.Header{
1741                                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
1742                                 },
1743                         }
1744                         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", userperm,
1745                                 arvadostest.ActiveUserUUID, arvadostest.FooCollection, arvadostest.FooCollectionPDH, "foo")
1746                 }
1747         }
1748
1749         s.handler.Cluster.Collections.WebDAVPermission.User.Download = true
1750
1751         for _, tryurl := range []string{"http://" + arvadostest.MultilevelCollection1 + ".keep-web.example/dir1/subdir/file1",
1752                 "http://keep-web/users/active/multilevel_collection_1/dir1/subdir/file1"} {
1753
1754                 u = mustParseURL(tryurl)
1755                 req := &http.Request{
1756                         Method:     "GET",
1757                         Host:       u.Host,
1758                         URL:        u,
1759                         RequestURI: u.RequestURI(),
1760                         Header: http.Header{
1761                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
1762                         },
1763                 }
1764                 s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true,
1765                         arvadostest.ActiveUserUUID, arvadostest.MultilevelCollection1, arvadostest.MultilevelCollection1PDH, "dir1/subdir/file1")
1766         }
1767
1768         u = mustParseURL("http://" + strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + ".keep-web.example/foo")
1769         req := &http.Request{
1770                 Method:     "GET",
1771                 Host:       u.Host,
1772                 URL:        u,
1773                 RequestURI: u.RequestURI(),
1774                 Header: http.Header{
1775                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
1776                 },
1777         }
1778         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true,
1779                 arvadostest.ActiveUserUUID, "", arvadostest.FooCollectionPDH, "foo")
1780 }
1781
1782 func (s *IntegrationSuite) TestUploadLoggingPermission(c *check.C) {
1783         for _, adminperm := range []bool{true, false} {
1784                 for _, userperm := range []bool{true, false} {
1785
1786                         arv := arvados.NewClientFromEnv()
1787                         arv.AuthToken = arvadostest.ActiveToken
1788
1789                         var coll arvados.Collection
1790                         err := arv.RequestAndDecode(&coll,
1791                                 "POST",
1792                                 "/arvados/v1/collections",
1793                                 nil,
1794                                 map[string]interface{}{
1795                                         "ensure_unique_name": true,
1796                                         "collection": map[string]interface{}{
1797                                                 "name": "test collection",
1798                                         },
1799                                 })
1800                         c.Assert(err, check.Equals, nil)
1801
1802                         u := mustParseURL("http://" + coll.UUID + ".keep-web.example/bar")
1803
1804                         s.handler.Cluster.Collections.WebDAVPermission.Admin.Upload = adminperm
1805                         s.handler.Cluster.Collections.WebDAVPermission.User.Upload = userperm
1806
1807                         // Test admin permission
1808                         req := &http.Request{
1809                                 Method:     "PUT",
1810                                 Host:       u.Host,
1811                                 URL:        u,
1812                                 RequestURI: u.RequestURI(),
1813                                 Header: http.Header{
1814                                         "Authorization": {"Bearer " + arvadostest.AdminToken},
1815                                 },
1816                                 Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
1817                         }
1818                         s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", adminperm,
1819                                 arvadostest.AdminUserUUID, coll.UUID, "", "bar")
1820
1821                         // Test user permission
1822                         req = &http.Request{
1823                                 Method:     "PUT",
1824                                 Host:       u.Host,
1825                                 URL:        u,
1826                                 RequestURI: u.RequestURI(),
1827                                 Header: http.Header{
1828                                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
1829                                 },
1830                                 Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
1831                         }
1832                         s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", userperm,
1833                                 arvadostest.ActiveUserUUID, coll.UUID, "", "bar")
1834                 }
1835         }
1836 }
1837
1838 func (s *IntegrationSuite) TestConcurrentWrites(c *check.C) {
1839         s.handler.Cluster.Collections.WebDAVCache.TTL = arvados.Duration(time.Second * 2)
1840         lockTidyInterval = time.Second
1841         client := arvados.NewClientFromEnv()
1842         client.AuthToken = arvadostest.ActiveTokenV2
1843         // Start small, and increase concurrency (2^2, 4^2, ...)
1844         // only until hitting failure. Avoids unnecessarily long
1845         // failure reports.
1846         for n := 2; n < 16 && !c.Failed(); n = n * 2 {
1847                 c.Logf("%s: n=%d", c.TestName(), n)
1848
1849                 var coll arvados.Collection
1850                 err := client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, nil)
1851                 c.Assert(err, check.IsNil)
1852                 defer client.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
1853
1854                 var wg sync.WaitGroup
1855                 for i := 0; i < n && !c.Failed(); i++ {
1856                         i := i
1857                         wg.Add(1)
1858                         go func() {
1859                                 defer wg.Done()
1860                                 u := mustParseURL(fmt.Sprintf("http://%s.collections.example.com/i=%d", coll.UUID, i))
1861                                 resp := httptest.NewRecorder()
1862                                 req, err := http.NewRequest("MKCOL", u.String(), nil)
1863                                 c.Assert(err, check.IsNil)
1864                                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
1865                                 s.handler.ServeHTTP(resp, req)
1866                                 c.Assert(resp.Code, check.Equals, http.StatusCreated)
1867                                 for j := 0; j < n && !c.Failed(); j++ {
1868                                         j := j
1869                                         wg.Add(1)
1870                                         go func() {
1871                                                 defer wg.Done()
1872                                                 content := fmt.Sprintf("i=%d/j=%d", i, j)
1873                                                 u := mustParseURL("http://" + coll.UUID + ".collections.example.com/" + content)
1874
1875                                                 resp := httptest.NewRecorder()
1876                                                 req, err := http.NewRequest("PUT", u.String(), strings.NewReader(content))
1877                                                 c.Assert(err, check.IsNil)
1878                                                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
1879                                                 s.handler.ServeHTTP(resp, req)
1880                                                 c.Check(resp.Code, check.Equals, http.StatusCreated)
1881
1882                                                 time.Sleep(time.Second)
1883                                                 resp = httptest.NewRecorder()
1884                                                 req, err = http.NewRequest("GET", u.String(), nil)
1885                                                 c.Assert(err, check.IsNil)
1886                                                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
1887                                                 s.handler.ServeHTTP(resp, req)
1888                                                 c.Check(resp.Code, check.Equals, http.StatusOK)
1889                                                 c.Check(resp.Body.String(), check.Equals, content)
1890                                         }()
1891                                 }
1892                         }()
1893                 }
1894                 wg.Wait()
1895                 for i := 0; i < n; i++ {
1896                         u := mustParseURL(fmt.Sprintf("http://%s.collections.example.com/i=%d", coll.UUID, i))
1897                         resp := httptest.NewRecorder()
1898                         req, err := http.NewRequest("PROPFIND", u.String(), &bytes.Buffer{})
1899                         c.Assert(err, check.IsNil)
1900                         req.Header.Set("Authorization", "Bearer "+client.AuthToken)
1901                         s.handler.ServeHTTP(resp, req)
1902                         c.Assert(resp.Code, check.Equals, http.StatusMultiStatus)
1903                 }
1904         }
1905 }