services/keep-web/handler_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package keepweb
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "fmt"
  11         "html"
  12         "io"
  13         "io/ioutil"
  14         "net/http"
  15         "net/http/httptest"
  16         "net/url"
  17         "os"
  18         "path/filepath"
  19         "regexp"
  20         "strings"
  21         "sync"
  22         "time"
  23
  24         "git.arvados.org/arvados.git/lib/config"
  25         "git.arvados.org/arvados.git/sdk/go/arvados"
  26         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  27         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  28         "git.arvados.org/arvados.git/sdk/go/auth"
  29         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  30         "git.arvados.org/arvados.git/sdk/go/keepclient"
  31         "github.com/prometheus/client_golang/prometheus"
  32         "github.com/sirupsen/logrus"
  33         check "gopkg.in/check.v1"
  34 )
  35
  36 var _ = check.Suite(&UnitSuite{})
  37
  38 func init() {
  39         arvados.DebugLocksPanicMode = true
  40 }
  41
  42 type UnitSuite struct {
  43         cluster *arvados.Cluster
  44         handler *handler
  45 }
  46
  47 func (s *UnitSuite) SetUpTest(c *check.C) {
  48         logger := ctxlog.TestLogger(c)
  49         ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), logger)
  50         ldr.Path = "-"
  51         cfg, err := ldr.Load()
  52         c.Assert(err, check.IsNil)
  53         cc, err := cfg.GetCluster("")
  54         c.Assert(err, check.IsNil)
  55         s.cluster = cc
  56         s.handler = &handler{
  57                 Cluster: cc,
  58                 Cache: cache{
  59                         cluster:  cc,
  60                         logger:   logger,
  61                         registry: prometheus.NewRegistry(),
  62                 },
  63         }
  64 }
  65
  66 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
  67         h := s.handler
  68         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
  69         req := &http.Request{
  70                 Method:     "OPTIONS",
  71                 Host:       u.Host,
  72                 URL:        u,
  73                 RequestURI: u.RequestURI(),
  74                 Header: http.Header{
  75                         "Origin":                        {"https://workbench.example"},
  76                         "Access-Control-Request-Method": {"POST"},
  77                 },
  78         }
  79
  80         // Check preflight for an allowed request
  81         resp := httptest.NewRecorder()
  82         h.ServeHTTP(resp, req)
  83         c.Check(resp.Code, check.Equals, http.StatusOK)
  84         c.Check(resp.Body.String(), check.Equals, "")
  85         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
  86         c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
  87         c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout, Cache-Control")
  88
  89         // Check preflight for a disallowed request
  90         resp = httptest.NewRecorder()
  91         req.Header.Set("Access-Control-Request-Method", "MAKE-COFFEE")
  92         h.ServeHTTP(resp, req)
  93         c.Check(resp.Body.String(), check.Equals, "")
  94         c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
  95 }
  96
  97 func (s *UnitSuite) TestWebdavPrefixAndSource(c *check.C) {
  98         for _, trial := range []struct {
  99                 method   string
 100                 path     string
 101                 prefix   string
 102                 source   string
 103                 notFound bool
 104                 seeOther bool
 105         }{
 106                 {
 107                         method: "PROPFIND",
 108                         path:   "/",
 109                 },
 110                 {
 111                         method: "PROPFIND",
 112                         path:   "/dir1",
 113                 },
 114                 {
 115                         method: "PROPFIND",
 116                         path:   "/dir1/",
 117                 },
 118                 {
 119                         method: "PROPFIND",
 120                         path:   "/dir1/foo",
 121                         prefix: "/dir1",
 122                         source: "/dir1",
 123                 },
 124                 {
 125                         method: "PROPFIND",
 126                         path:   "/prefix/dir1/foo",
 127                         prefix: "/prefix/",
 128                         source: "",
 129                 },
 130                 {
 131                         method: "PROPFIND",
 132                         path:   "/prefix/dir1/foo",
 133                         prefix: "/prefix",
 134                         source: "",
 135                 },
 136                 {
 137                         method: "PROPFIND",
 138                         path:   "/prefix/dir1/foo",
 139                         prefix: "/prefix/",
 140                         source: "/",
 141                 },
 142                 {
 143                         method: "PROPFIND",
 144                         path:   "/prefix/foo",
 145                         prefix: "/prefix/",
 146                         source: "/dir1/",
 147                 },
 148                 {
 149                         method: "GET",
 150                         path:   "/prefix/foo",
 151                         prefix: "/prefix/",
 152                         source: "/dir1/",
 153                 },
 154                 {
 155                         method: "PROPFIND",
 156                         path:   "/prefix/",
 157                         prefix: "/prefix",
 158                         source: "/dir1",
 159                 },
 160                 {
 161                         method: "PROPFIND",
 162                         path:   "/prefix",
 163                         prefix: "/prefix",
 164                         source: "/dir1/",
 165                 },
 166                 {
 167                         method:   "GET",
 168                         path:     "/prefix",
 169                         prefix:   "/prefix",
 170                         source:   "/dir1",
 171                         seeOther: true,
 172                 },
 173                 {
 174                         method:   "PROPFIND",
 175                         path:     "/dir1/foo",
 176                         prefix:   "",
 177                         source:   "/dir1",
 178                         notFound: true,
 179                 },
 180         } {
 181                 c.Logf("trial %+v", trial)
 182                 u := mustParseURL("http://" + arvadostest.FooBarDirCollection + ".keep-web.example" + trial.path)
 183                 req := &http.Request{
 184                         Method:     trial.method,
 185                         Host:       u.Host,
 186                         URL:        u,
 187                         RequestURI: u.RequestURI(),
 188                         Header: http.Header{
 189                                 "Authorization":   {"Bearer " + arvadostest.ActiveTokenV2},
 190                                 "X-Webdav-Prefix": {trial.prefix},
 191                                 "X-Webdav-Source": {trial.source},
 192                         },
 193                         Body: ioutil.NopCloser(bytes.NewReader(nil)),
 194                 }
 195
 196                 resp := httptest.NewRecorder()
 197                 s.handler.ServeHTTP(resp, req)
 198                 if trial.notFound {
 199                         c.Check(resp.Code, check.Equals, http.StatusNotFound)
 200                 } else if trial.method == "PROPFIND" {
 201                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus)
 202                         c.Check(resp.Body.String(), check.Matches, `(?ms).*>\n?$`)
 203                 } else if trial.seeOther {
 204                         c.Check(resp.Code, check.Equals, http.StatusSeeOther)
 205                 } else {
 206                         c.Check(resp.Code, check.Equals, http.StatusOK)
 207                 }
 208         }
 209 }
 210
 211 func (s *UnitSuite) TestEmptyResponse(c *check.C) {
 212         for _, trial := range []struct {
 213                 dataExists    bool
 214                 sendIMSHeader bool
 215                 expectStatus  int
 216                 logRegexp     string
 217         }{
 218                 // If we return no content due to a Keep read error,
 219                 // we should emit a log message.
 220                 {false, false, http.StatusOK, `(?ms).*only wrote 0 bytes.*`},
 221
 222                 // If we return no content because the client sent an
 223                 // If-Modified-Since header, our response should be
 224                 // 304.  We still expect a "File download" log since it
 225                 // counts as a file access for auditing.
 226                 {true, true, http.StatusNotModified, `(?ms).*msg="File download".*`},
 227         } {
 228                 c.Logf("trial: %+v", trial)
 229                 arvadostest.StartKeep(2, true)
 230                 if trial.dataExists {
 231                         arv, err := arvadosclient.MakeArvadosClient()
 232                         c.Assert(err, check.IsNil)
 233                         arv.ApiToken = arvadostest.ActiveToken
 234                         kc, err := keepclient.MakeKeepClient(arv)
 235                         c.Assert(err, check.IsNil)
 236                         _, _, err = kc.PutB([]byte("foo"))
 237                         c.Assert(err, check.IsNil)
 238                 }
 239
 240                 u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
 241                 req := &http.Request{
 242                         Method:     "GET",
 243                         Host:       u.Host,
 244                         URL:        u,
 245                         RequestURI: u.RequestURI(),
 246                         Header: http.Header{
 247                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
 248                         },
 249                 }
 250                 if trial.sendIMSHeader {
 251                         req.Header.Set("If-Modified-Since", strings.Replace(time.Now().UTC().Format(time.RFC1123), "UTC", "GMT", -1))
 252                 }
 253
 254                 var logbuf bytes.Buffer
 255                 logger := logrus.New()
 256                 logger.Out = &logbuf
 257                 req = req.WithContext(ctxlog.Context(context.Background(), logger))
 258
 259                 resp := httptest.NewRecorder()
 260                 s.handler.ServeHTTP(resp, req)
 261                 c.Check(resp.Code, check.Equals, trial.expectStatus)
 262                 c.Check(resp.Body.String(), check.Equals, "")
 263
 264                 c.Log(logbuf.String())
 265                 c.Check(logbuf.String(), check.Matches, trial.logRegexp)
 266         }
 267 }
 268
 269 func (s *UnitSuite) TestInvalidUUID(c *check.C) {
 270         bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-"
 271         token := arvadostest.ActiveToken
 272         for _, trial := range []string{
 273                 "http://keep-web/c=" + bogusID + "/foo",
 274                 "http://keep-web/c=" + bogusID + "/t=" + token + "/foo",
 275                 "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo",
 276                 "http://keep-web/collections/" + bogusID + "/foo",
 277                 "http://" + bogusID + ".keep-web/" + bogusID + "/foo",
 278                 "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
 279         } {
 280                 c.Log(trial)
 281                 u := mustParseURL(trial)
 282                 req := &http.Request{
 283                         Method:     "GET",
 284                         Host:       u.Host,
 285                         URL:        u,
 286                         RequestURI: u.RequestURI(),
 287                 }
 288                 resp := httptest.NewRecorder()
 289                 s.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 290                 s.handler.ServeHTTP(resp, req)
 291                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 292         }
 293 }
 294
 295 func mustParseURL(s string) *url.URL {
 296         r, err := url.Parse(s)
 297         if err != nil {
 298                 panic("parse URL: " + s)
 299         }
 300         return r
 301 }
 302
 303 func (s *IntegrationSuite) TestVhost404(c *check.C) {
 304         for _, testURL := range []string{
 305                 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
 306                 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
 307         } {
 308                 resp := httptest.NewRecorder()
 309                 u := mustParseURL(testURL)
 310                 req := &http.Request{
 311                         Method:     "GET",
 312                         URL:        u,
 313                         RequestURI: u.RequestURI(),
 314                 }
 315                 s.handler.ServeHTTP(resp, req)
 316                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 317                 c.Check(resp.Body.String(), check.Equals, notFoundMessage+"\n")
 318         }
 319 }
 320
 321 // An authorizer modifies an HTTP request to make use of the given
 322 // token -- by adding it to a header, cookie, query param, or whatever
 323 // -- and returns the HTTP status code we should expect from keep-web if
 324 // the token is invalid.
 325 type authorizer func(*http.Request, string) int
 326
 327 func (s *IntegrationSuite) TestVhostViaAuthzHeaderOAuth2(c *check.C) {
 328         s.doVhostRequests(c, authzViaAuthzHeaderOAuth2)
 329 }
 330 func authzViaAuthzHeaderOAuth2(r *http.Request, tok string) int {
 331         r.Header.Add("Authorization", "OAuth2 "+tok)
 332         return http.StatusUnauthorized
 333 }
 334
 335 func (s *IntegrationSuite) TestVhostViaAuthzHeaderBearer(c *check.C) {
 336         s.doVhostRequests(c, authzViaAuthzHeaderBearer)
 337 }
 338 func authzViaAuthzHeaderBearer(r *http.Request, tok string) int {
 339         r.Header.Add("Authorization", "Bearer "+tok)
 340         return http.StatusUnauthorized
 341 }
 342
 343 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
 344         s.doVhostRequests(c, authzViaCookieValue)
 345 }
 346 func authzViaCookieValue(r *http.Request, tok string) int {
 347         r.AddCookie(&http.Cookie{
 348                 Name:  "arvados_api_token",
 349                 Value: auth.EncodeTokenCookie([]byte(tok)),
 350         })
 351         return http.StatusUnauthorized
 352 }
 353
 354 func (s *IntegrationSuite) TestVhostViaHTTPBasicAuth(c *check.C) {
 355         s.doVhostRequests(c, authzViaHTTPBasicAuth)
 356 }
 357 func authzViaHTTPBasicAuth(r *http.Request, tok string) int {
 358         r.AddCookie(&http.Cookie{
 359                 Name:  "arvados_api_token",
 360                 Value: auth.EncodeTokenCookie([]byte(tok)),
 361         })
 362         return http.StatusUnauthorized
 363 }
 364
 365 func (s *IntegrationSuite) TestVhostViaHTTPBasicAuthWithExtraSpaceChars(c *check.C) {
 366         s.doVhostRequests(c, func(r *http.Request, tok string) int {
 367                 r.AddCookie(&http.Cookie{
 368                         Name:  "arvados_api_token",
 369                         Value: auth.EncodeTokenCookie([]byte(" " + tok + "\n")),
 370                 })
 371                 return http.StatusUnauthorized
 372         })
 373 }
 374
 375 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
 376         s.doVhostRequests(c, authzViaPath)
 377 }
 378 func authzViaPath(r *http.Request, tok string) int {
 379         r.URL.Path = "/t=" + tok + r.URL.Path
 380         return http.StatusNotFound
 381 }
 382
 383 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
 384         s.doVhostRequests(c, authzViaQueryString)
 385 }
 386 func authzViaQueryString(r *http.Request, tok string) int {
 387         r.URL.RawQuery = "api_token=" + tok
 388         return http.StatusUnauthorized
 389 }
 390
 391 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
 392         s.doVhostRequests(c, authzViaPOST)
 393 }
 394 func authzViaPOST(r *http.Request, tok string) int {
 395         r.Method = "POST"
 396         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 397         r.Body = ioutil.NopCloser(strings.NewReader(
 398                 url.Values{"api_token": {tok}}.Encode()))
 399         return http.StatusUnauthorized
 400 }
 401
 402 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
 403         s.doVhostRequests(c, authzViaPOST)
 404 }
 405 func authzViaXHRPOST(r *http.Request, tok string) int {
 406         r.Method = "POST"
 407         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 408         r.Header.Add("Origin", "https://origin.example")
 409         r.Body = ioutil.NopCloser(strings.NewReader(
 410                 url.Values{
 411                         "api_token":   {tok},
 412                         "disposition": {"attachment"},
 413                 }.Encode()))
 414         return http.StatusUnauthorized
 415 }
 416
 417 // Try some combinations of {url, token} using the given authorization
 418 // mechanism, and verify the result is correct.
 419 func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
 420         for _, hostPath := range []string{
 421                 arvadostest.FooCollection + ".example.com/foo",
 422                 arvadostest.FooCollection + "--collections.example.com/foo",
 423                 arvadostest.FooCollection + "--collections.example.com/_/foo",
 424                 arvadostest.FooCollectionPDH + ".example.com/foo",
 425                 strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + "--collections.example.com/foo",
 426                 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
 427         } {
 428                 c.Log("doRequests: ", hostPath)
 429                 s.doVhostRequestsWithHostPath(c, authz, hostPath)
 430         }
 431 }
 432
 433 func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
 434         for _, tok := range []string{
 435                 arvadostest.ActiveToken,
 436                 arvadostest.ActiveToken[:15],
 437                 arvadostest.SpectatorToken,
 438                 "bogus",
 439                 "",
 440         } {
 441                 u := mustParseURL("http://" + hostPath)
 442                 req := &http.Request{
 443                         Method:     "GET",
 444                         Host:       u.Host,
 445                         URL:        u,
 446                         RequestURI: u.RequestURI(),
 447                         Header:     http.Header{},
 448                 }
 449                 failCode := authz(req, tok)
 450                 req, resp := s.doReq(req)
 451                 code, body := resp.Code, resp.Body.String()
 452
 453                 // If the initial request had a (non-empty) token
 454                 // showing in the query string, we should have been
 455                 // redirected in order to hide it in a cookie.
 456                 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
 457
 458                 if tok == arvadostest.ActiveToken {
 459                         c.Check(code, check.Equals, http.StatusOK)
 460                         c.Check(body, check.Equals, "foo")
 461                 } else {
 462                         c.Check(code >= 400, check.Equals, true)
 463                         c.Check(code < 500, check.Equals, true)
 464                         if tok == arvadostest.SpectatorToken {
 465                                 // Valid token never offers to retry
 466                                 // with different credentials.
 467                                 c.Check(code, check.Equals, http.StatusNotFound)
 468                         } else {
 469                                 // Invalid token can ask to retry
 470                                 // depending on the authz method.
 471                                 c.Check(code, check.Equals, failCode)
 472                         }
 473                         if code == 404 {
 474                                 c.Check(body, check.Equals, notFoundMessage+"\n")
 475                         } else {
 476                                 c.Check(body, check.Equals, unauthorizedMessage+"\n")
 477                         }
 478                 }
 479         }
 480 }
 481
 482 func (s *IntegrationSuite) TestVhostPortMatch(c *check.C) {
 483         for _, host := range []string{"download.example.com", "DOWNLOAD.EXAMPLE.COM"} {
 484                 for _, port := range []string{"80", "443", "8000"} {
 485                         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = fmt.Sprintf("download.example.com:%v", port)
 486                         u := mustParseURL(fmt.Sprintf("http://%v/by_id/%v/foo", host, arvadostest.FooCollection))
 487                         req := &http.Request{
 488                                 Method:     "GET",
 489                                 Host:       u.Host,
 490                                 URL:        u,
 491                                 RequestURI: u.RequestURI(),
 492                                 Header:     http.Header{"Authorization": []string{"Bearer " + arvadostest.ActiveToken}},
 493                         }
 494                         req, resp := s.doReq(req)
 495                         code, _ := resp.Code, resp.Body.String()
 496
 497                         if port == "8000" {
 498                                 c.Check(code, check.Equals, 401)
 499                         } else {
 500                                 c.Check(code, check.Equals, 200)
 501                         }
 502                 }
 503         }
 504 }
 505
 506 func (s *IntegrationSuite) do(method string, urlstring string, token string, hdr http.Header) (*http.Request, *httptest.ResponseRecorder) {
 507         u := mustParseURL(urlstring)
 508         if hdr == nil && token != "" {
 509                 hdr = http.Header{"Authorization": {"Bearer " + token}}
 510         } else if hdr == nil {
 511                 hdr = http.Header{}
 512         } else if token != "" {
 513                 panic("must not pass both token and hdr")
 514         }
 515         return s.doReq(&http.Request{
 516                 Method:     method,
 517                 Host:       u.Host,
 518                 URL:        u,
 519                 RequestURI: u.RequestURI(),
 520                 Header:     hdr,
 521         })
 522 }
 523
 524 func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
 525         resp := httptest.NewRecorder()
 526         s.handler.ServeHTTP(resp, req)
 527         if resp.Code != http.StatusSeeOther {
 528                 return req, resp
 529         }
 530         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 531         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 532         req = &http.Request{
 533                 Method:     "GET",
 534                 Host:       u.Host,
 535                 URL:        u,
 536                 RequestURI: u.RequestURI(),
 537                 Header:     http.Header{},
 538         }
 539         for _, c := range cookies {
 540                 req.AddCookie(c)
 541         }
 542         return s.doReq(req)
 543 }
 544
 545 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
 546         s.testVhostRedirectTokenToCookie(c, "GET",
 547                 arvadostest.FooCollection+".example.com/foo",
 548                 "?api_token="+arvadostest.ActiveToken,
 549                 nil,
 550                 "",
 551                 http.StatusOK,
 552                 "foo",
 553         )
 554 }
 555
 556 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
 557         s.testVhostRedirectTokenToCookie(c, "GET",
 558                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
 559                 "",
 560                 nil,
 561                 "",
 562                 http.StatusOK,
 563                 "foo",
 564         )
 565 }
 566
 567 func (s *IntegrationSuite) TestCollectionSharingToken(c *check.C) {
 568         s.testVhostRedirectTokenToCookie(c, "GET",
 569                 "example.com/c="+arvadostest.FooFileCollectionUUID+"/t="+arvadostest.FooFileCollectionSharingToken+"/foo",
 570                 "",
 571                 nil,
 572                 "",
 573                 http.StatusOK,
 574                 "foo",
 575         )
 576         // Same valid sharing token, but requesting a different collection
 577         s.testVhostRedirectTokenToCookie(c, "GET",
 578                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.FooFileCollectionSharingToken+"/foo",
 579                 "",
 580                 nil,
 581                 "",
 582                 http.StatusNotFound,
 583                 regexp.QuoteMeta(notFoundMessage+"\n"),
 584         )
 585 }
 586
 587 // Bad token in URL is 404 Not Found because it doesn't make sense to
 588 // retry the same URL with different authorization.
 589 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
 590         s.testVhostRedirectTokenToCookie(c, "GET",
 591                 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
 592                 "",
 593                 nil,
 594                 "",
 595                 http.StatusNotFound,
 596                 regexp.QuoteMeta(notFoundMessage+"\n"),
 597         )
 598 }
 599
 600 // Bad token in a cookie (even if it got there via our own
 601 // query-string-to-cookie redirect) is, in principle, retryable via
 602 // wb2-login-and-redirect flow.
 603 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
 604         // Inline
 605         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 606                 arvadostest.FooCollection+".example.com/foo",
 607                 "?api_token=thisisabogustoken",
 608                 http.Header{"Sec-Fetch-Mode": {"navigate"}},
 609                 "",
 610                 http.StatusSeeOther,
 611                 "",
 612         )
 613         u, err := url.Parse(resp.Header().Get("Location"))
 614         c.Assert(err, check.IsNil)
 615         c.Logf("redirected to %s", u)
 616         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 617         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 618         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
 619
 620         // Download/attachment indicated by ?disposition=attachment
 621         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 622                 arvadostest.FooCollection+".example.com/foo",
 623                 "?api_token=thisisabogustoken&disposition=attachment",
 624                 http.Header{"Sec-Fetch-Mode": {"navigate"}},
 625                 "",
 626                 http.StatusSeeOther,
 627                 "",
 628         )
 629         u, err = url.Parse(resp.Header().Get("Location"))
 630         c.Assert(err, check.IsNil)
 631         c.Logf("redirected to %s", u)
 632         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 633         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "")
 634         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 635
 636         // Download/attachment indicated by vhost
 637         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 638                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
 639                 "?api_token=thisisabogustoken",
 640                 http.Header{"Sec-Fetch-Mode": {"navigate"}},
 641                 "",
 642                 http.StatusSeeOther,
 643                 "",
 644         )
 645         u, err = url.Parse(resp.Header().Get("Location"))
 646         c.Assert(err, check.IsNil)
 647         c.Logf("redirected to %s", u)
 648         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 649         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "")
 650         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 651
 652         // Without "Sec-Fetch-Mode: navigate" header, just 401.
 653         s.testVhostRedirectTokenToCookie(c, "GET",
 654                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
 655                 "?api_token=thisisabogustoken",
 656                 http.Header{"Sec-Fetch-Mode": {"cors"}},
 657                 "",
 658                 http.StatusUnauthorized,
 659                 regexp.QuoteMeta(unauthorizedMessage+"\n"),
 660         )
 661         s.testVhostRedirectTokenToCookie(c, "GET",
 662                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
 663                 "?api_token=thisisabogustoken",
 664                 nil,
 665                 "",
 666                 http.StatusUnauthorized,
 667                 regexp.QuoteMeta(unauthorizedMessage+"\n"),
 668         )
 669 }
 670
 671 func (s *IntegrationSuite) TestVhostRedirectWithNoCache(c *check.C) {
 672         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 673                 arvadostest.FooCollection+".example.com/foo",
 674                 "?api_token=thisisabogustoken",
 675                 http.Header{
 676                         "Sec-Fetch-Mode": {"navigate"},
 677                         "Cache-Control":  {"no-cache"},
 678                 },
 679                 "",
 680                 http.StatusSeeOther,
 681                 "",
 682         )
 683         u, err := url.Parse(resp.Header().Get("Location"))
 684         c.Assert(err, check.IsNil)
 685         c.Logf("redirected to %s", u)
 686         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 687         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 688         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
 689 }
 690
 691 func (s *IntegrationSuite) TestNoTokenWorkbench2LoginFlow(c *check.C) {
 692         for _, trial := range []struct {
 693                 anonToken    bool
 694                 cacheControl string
 695         }{
 696                 {},
 697                 {cacheControl: "no-cache"},
 698                 {anonToken: true},
 699                 {anonToken: true, cacheControl: "no-cache"},
 700         } {
 701                 c.Logf("trial: %+v", trial)
 702
 703                 if trial.anonToken {
 704                         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 705                 } else {
 706                         s.handler.Cluster.Users.AnonymousUserToken = ""
 707                 }
 708                 req, err := http.NewRequest("GET", "http://"+arvadostest.FooCollection+".example.com/foo", nil)
 709                 c.Assert(err, check.IsNil)
 710                 req.Header.Set("Sec-Fetch-Mode", "navigate")
 711                 if trial.cacheControl != "" {
 712                         req.Header.Set("Cache-Control", trial.cacheControl)
 713                 }
 714                 resp := httptest.NewRecorder()
 715                 s.handler.ServeHTTP(resp, req)
 716                 c.Check(resp.Code, check.Equals, http.StatusSeeOther)
 717                 u, err := url.Parse(resp.Header().Get("Location"))
 718                 c.Assert(err, check.IsNil)
 719                 c.Logf("redirected to %q", u)
 720                 c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 721                 c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 722                 c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
 723         }
 724 }
 725
 726 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
 727         s.testVhostRedirectTokenToCookie(c, "GET",
 728                 "example.com/c="+arvadostest.FooCollection+"/foo",
 729                 "?api_token="+arvadostest.ActiveToken,
 730                 nil,
 731                 "",
 732                 http.StatusBadRequest,
 733                 regexp.QuoteMeta("cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n"),
 734         )
 735 }
 736
 737 // If client requests an attachment by putting ?disposition=attachment
 738 // in the query string, and gets redirected, the redirect target
 739 // should respond with an attachment.
 740 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
 741         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 742                 arvadostest.FooCollection+".example.com/foo",
 743                 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
 744                 nil,
 745                 "",
 746                 http.StatusOK,
 747                 "foo",
 748         )
 749         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 750 }
 751
 752 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
 753         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 754         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 755                 "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
 756                 "?api_token="+arvadostest.ActiveToken,
 757                 nil,
 758                 "",
 759                 http.StatusOK,
 760                 "foo",
 761         )
 762         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 763 }
 764
 765 func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
 766         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 767         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 768                 "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
 769                 "?api_token="+arvadostest.ActiveToken,
 770                 nil,
 771                 "",
 772                 http.StatusOK,
 773                 "waz",
 774         )
 775         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 776         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 777                 "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz",
 778                 "?api_token="+arvadostest.ActiveToken,
 779                 nil,
 780                 "",
 781                 http.StatusOK,
 782                 "waz",
 783         )
 784         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 785 }
 786
 787 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
 788         s.handler.Cluster.Collections.TrustAllContent = true
 789         s.testVhostRedirectTokenToCookie(c, "GET",
 790                 "example.com/c="+arvadostest.FooCollection+"/foo",
 791                 "?api_token="+arvadostest.ActiveToken,
 792                 nil,
 793                 "",
 794                 http.StatusOK,
 795                 "foo",
 796         )
 797 }
 798
 799 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
 800         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
 801
 802         s.testVhostRedirectTokenToCookie(c, "GET",
 803                 "example.com/c="+arvadostest.FooCollection+"/foo",
 804                 "?api_token="+arvadostest.ActiveToken,
 805                 nil,
 806                 "",
 807                 http.StatusBadRequest,
 808                 regexp.QuoteMeta("cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n"),
 809         )
 810
 811         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 812                 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
 813                 "?api_token="+arvadostest.ActiveToken,
 814                 nil,
 815                 "",
 816                 http.StatusOK,
 817                 "foo",
 818         )
 819         c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
 820 }
 821
 822 func (s *IntegrationSuite) TestVhostRedirectMultipleTokens(c *check.C) {
 823         baseUrl := arvadostest.FooCollection + ".example.com/foo"
 824         query := url.Values{}
 825
 826         // The intent of these tests is to check that requests are redirected
 827         // correctly in the presence of multiple API tokens. The exact response
 828         // codes and content are not closely considered: they're just how
 829         // keep-web responded when we made the smallest possible fix. Changing
 830         // those responses may be okay, but you should still test all these
 831         // different cases and the associated redirect logic.
 832         query["api_token"] = []string{arvadostest.ActiveToken, arvadostest.AnonymousToken}
 833         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
 834         query["api_token"] = []string{arvadostest.ActiveToken, arvadostest.AnonymousToken, ""}
 835         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
 836         query["api_token"] = []string{arvadostest.ActiveToken, "", arvadostest.AnonymousToken}
 837         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
 838         query["api_token"] = []string{"", arvadostest.ActiveToken}
 839         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
 840
 841         expectContent := regexp.QuoteMeta(unauthorizedMessage + "\n")
 842         query["api_token"] = []string{arvadostest.AnonymousToken, "invalidtoo"}
 843         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusUnauthorized, expectContent)
 844         query["api_token"] = []string{arvadostest.AnonymousToken, ""}
 845         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusUnauthorized, expectContent)
 846         query["api_token"] = []string{"", arvadostest.AnonymousToken}
 847         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusUnauthorized, expectContent)
 848 }
 849
 850 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
 851         s.testVhostRedirectTokenToCookie(c, "POST",
 852                 arvadostest.FooCollection+".example.com/foo",
 853                 "",
 854                 http.Header{"Content-Type": {"application/x-www-form-urlencoded"}},
 855                 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
 856                 http.StatusOK,
 857                 "foo",
 858         )
 859 }
 860
 861 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
 862         s.testVhostRedirectTokenToCookie(c, "POST",
 863                 arvadostest.FooCollection+".example.com/foo",
 864                 "",
 865                 http.Header{"Content-Type": {"application/x-www-form-urlencoded"}},
 866                 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
 867                 http.StatusNotFound,
 868                 regexp.QuoteMeta(notFoundMessage+"\n"),
 869         )
 870 }
 871
 872 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
 873         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 874         s.testVhostRedirectTokenToCookie(c, "GET",
 875                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 876                 "",
 877                 nil,
 878                 "",
 879                 http.StatusOK,
 880                 "Hello world\n",
 881         )
 882 }
 883
 884 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
 885         s.handler.Cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
 886         s.testVhostRedirectTokenToCookie(c, "GET",
 887                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 888                 "",
 889                 nil,
 890                 "",
 891                 http.StatusUnauthorized,
 892                 "Authorization tokens are not accepted here: .*\n",
 893         )
 894 }
 895
 896 func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
 897         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 898
 899         client := arvados.NewClientFromEnv()
 900         client.AuthToken = arvadostest.ActiveToken
 901         fs, err := (&arvados.Collection{}).FileSystem(client, nil)
 902         c.Assert(err, check.IsNil)
 903         f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777)
 904         c.Assert(err, check.IsNil)
 905         f.Close()
 906         mtxt, err := fs.MarshalManifest(".")
 907         c.Assert(err, check.IsNil)
 908         var coll arvados.Collection
 909         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 910                 "collection": map[string]string{
 911                         "manifest_text": mtxt,
 912                 },
 913         })
 914         c.Assert(err, check.IsNil)
 915
 916         u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
 917         req := &http.Request{
 918                 Method:     "GET",
 919                 Host:       u.Host,
 920                 URL:        u,
 921                 RequestURI: u.RequestURI(),
 922                 Header: http.Header{
 923                         "Authorization": {"Bearer " + client.AuthToken},
 924                 },
 925         }
 926         resp := httptest.NewRecorder()
 927         s.handler.ServeHTTP(resp, req)
 928         c.Check(resp.Code, check.Equals, http.StatusOK)
 929         c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\&#34;odd&#39; path chars.*`)
 930 }
 931
 932 func (s *IntegrationSuite) TestForwardSlashSubstitution(c *check.C) {
 933         arv := arvados.NewClientFromEnv()
 934         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 935         s.handler.Cluster.Collections.ForwardSlashNameSubstitution = "{SOLIDUS}"
 936         name := "foo/bar/baz"
 937         nameShown := strings.Replace(name, "/", "{SOLIDUS}", -1)
 938         nameShownEscaped := strings.Replace(name, "/", "%7bSOLIDUS%7d", -1)
 939
 940         client := arvados.NewClientFromEnv()
 941         client.AuthToken = arvadostest.ActiveToken
 942         fs, err := (&arvados.Collection{}).FileSystem(client, nil)
 943         c.Assert(err, check.IsNil)
 944         f, err := fs.OpenFile("filename", os.O_CREATE, 0777)
 945         c.Assert(err, check.IsNil)
 946         f.Close()
 947         mtxt, err := fs.MarshalManifest(".")
 948         c.Assert(err, check.IsNil)
 949         var coll arvados.Collection
 950         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 951                 "collection": map[string]string{
 952                         "manifest_text": mtxt,
 953                         "name":          name,
 954                         "owner_uuid":    arvadostest.AProjectUUID,
 955                 },
 956         })
 957         c.Assert(err, check.IsNil)
 958         defer arv.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
 959
 960         base := "http://download.example.com/by_id/" + coll.OwnerUUID + "/"
 961         for tryURL, expectRegexp := range map[string]string{
 962                 base:                          `(?ms).*href="./` + nameShownEscaped + `/"\S+` + nameShown + `.*`,
 963                 base + nameShownEscaped + "/": `(?ms).*href="./filename"\S+filename.*`,
 964         } {
 965                 u, _ := url.Parse(tryURL)
 966                 req := &http.Request{
 967                         Method:     "GET",
 968                         Host:       u.Host,
 969                         URL:        u,
 970                         RequestURI: u.RequestURI(),
 971                         Header: http.Header{
 972                                 "Authorization": {"Bearer " + client.AuthToken},
 973                         },
 974                 }
 975                 resp := httptest.NewRecorder()
 976                 s.handler.ServeHTTP(resp, req)
 977                 c.Check(resp.Code, check.Equals, http.StatusOK)
 978                 c.Check(resp.Body.String(), check.Matches, expectRegexp)
 979         }
 980 }
 981
 982 // XHRs can't follow redirect-with-cookie so they rely on method=POST
 983 // and disposition=attachment (telling us it's acceptable to respond
 984 // with content instead of a redirect) and an Origin header that gets
 985 // added automatically by the browser (telling us it's desirable to do
 986 // so).
 987 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
 988         u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
 989         req := &http.Request{
 990                 Method:     "POST",
 991                 Host:       u.Host,
 992                 URL:        u,
 993                 RequestURI: u.RequestURI(),
 994                 Header: http.Header{
 995                         "Origin":       {"https://origin.example"},
 996                         "Content-Type": {"application/x-www-form-urlencoded"},
 997                 },
 998                 Body: ioutil.NopCloser(strings.NewReader(url.Values{
 999                         "api_token":   {arvadostest.ActiveToken},
1000                         "disposition": {"attachment"},
1001                 }.Encode())),
1002         }
1003         resp := httptest.NewRecorder()
1004         s.handler.ServeHTTP(resp, req)
1005         c.Check(resp.Code, check.Equals, http.StatusOK)
1006         c.Check(resp.Body.String(), check.Equals, "foo")
1007         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
1008
1009         // GET + Origin header is representative of both AJAX GET
1010         // requests and inline images via <IMG crossorigin="anonymous"
1011         // src="...">.
1012         u.RawQuery = "api_token=" + url.QueryEscape(arvadostest.ActiveTokenV2)
1013         req = &http.Request{
1014                 Method:     "GET",
1015                 Host:       u.Host,
1016                 URL:        u,
1017                 RequestURI: u.RequestURI(),
1018                 Header: http.Header{
1019                         "Origin": {"https://origin.example"},
1020                 },
1021         }
1022         resp = httptest.NewRecorder()
1023         s.handler.ServeHTTP(resp, req)
1024         c.Check(resp.Code, check.Equals, http.StatusOK)
1025         c.Check(resp.Body.String(), check.Equals, "foo")
1026         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
1027 }
1028
1029 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString string, reqHeader http.Header, reqBody string, expectStatus int, matchRespBody string) *httptest.ResponseRecorder {
1030         if reqHeader == nil {
1031                 reqHeader = http.Header{}
1032         }
1033         u, _ := url.Parse(`http://` + hostPath + queryString)
1034         c.Logf("requesting %s", u)
1035         req := &http.Request{
1036                 Method:     method,
1037                 Host:       u.Host,
1038                 URL:        u,
1039                 RequestURI: u.RequestURI(),
1040                 Header:     reqHeader,
1041                 Body:       ioutil.NopCloser(strings.NewReader(reqBody)),
1042         }
1043
1044         resp := httptest.NewRecorder()
1045         defer func() {
1046                 c.Check(resp.Code, check.Equals, expectStatus)
1047                 c.Check(resp.Body.String(), check.Matches, matchRespBody)
1048         }()
1049
1050         s.handler.ServeHTTP(resp, req)
1051         if resp.Code != http.StatusSeeOther {
1052                 attachment, _ := regexp.MatchString(`^attachment(;|$)`, resp.Header().Get("Content-Disposition"))
1053                 // Since we're not redirecting, check that any api_token in the URL is
1054                 // handled safely.
1055                 // If there is no token in the URL, then we're good.
1056                 // Otherwise, if the response code is an error, the body is expected to
1057                 // be static content, and nothing that might maliciously introspect the
1058                 // URL. It's considered safe and allowed.
1059                 // Otherwise, if the response content has attachment disposition,
1060                 // that's considered safe for all the reasons explained in the
1061                 // safeAttachment comment in handler.go.
1062                 c.Check(!u.Query().Has("api_token") || resp.Code >= 400 || attachment, check.Equals, true)
1063                 return resp
1064         }
1065
1066         loc, err := url.Parse(resp.Header().Get("Location"))
1067         c.Assert(err, check.IsNil)
1068         c.Check(loc.Scheme, check.Equals, u.Scheme)
1069         c.Check(loc.Host, check.Equals, u.Host)
1070         c.Check(loc.RawPath, check.Equals, u.RawPath)
1071         // If the response was a redirect, it should never include an API token.
1072         c.Check(loc.Query().Has("api_token"), check.Equals, false)
1073         c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
1074         cookies := (&http.Response{Header: resp.Header()}).Cookies()
1075
1076         c.Logf("following redirect to %s", u)
1077         req = &http.Request{
1078                 Method:     "GET",
1079                 Host:       loc.Host,
1080                 URL:        loc,
1081                 RequestURI: loc.RequestURI(),
1082                 Header:     reqHeader,
1083         }
1084         for _, c := range cookies {
1085                 req.AddCookie(c)
1086         }
1087
1088         resp = httptest.NewRecorder()
1089         s.handler.ServeHTTP(resp, req)
1090
1091         if resp.Code != http.StatusSeeOther {
1092                 c.Check(resp.Header().Get("Location"), check.Equals, "")
1093         }
1094         return resp
1095 }
1096
1097 func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) {
1098         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
1099         s.testDirectoryListing(c)
1100 }
1101
1102 func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) {
1103         s.handler.Cluster.Users.AnonymousUserToken = ""
1104         s.testDirectoryListing(c)
1105 }
1106
1107 func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
1108         // The "ownership cycle" test fixtures are reachable from the
1109         // "filter group without filters" group, causing webdav's
1110         // walkfs to recurse indefinitely. Avoid that by deleting one
1111         // of the bogus fixtures.
1112         arv := arvados.NewClientFromEnv()
1113         err := arv.RequestAndDecode(nil, "DELETE", "arvados/v1/groups/zzzzz-j7d0g-cx2al9cqkmsf1hs", nil, nil)
1114         if err != nil {
1115                 c.Assert(err, check.FitsTypeOf, &arvados.TransactionError{})
1116                 c.Check(err.(*arvados.TransactionError).StatusCode, check.Equals, 404)
1117         }
1118
1119         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1120         authHeader := http.Header{
1121                 "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
1122         }
1123         for _, trial := range []struct {
1124                 uri      string
1125                 header   http.Header
1126                 expect   []string
1127                 redirect string
1128                 cutDirs  int
1129         }{
1130                 {
1131                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
1132                         header:  authHeader,
1133                         expect:  []string{"dir1/foo", "dir1/bar"},
1134                         cutDirs: 0,
1135                 },
1136                 {
1137                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
1138                         header:  authHeader,
1139                         expect:  []string{"foo", "bar"},
1140                         cutDirs: 1,
1141                 },
1142                 {
1143                         // URLs of this form ignore authHeader, and
1144                         // FooAndBarFilesInDirUUID isn't public, so
1145                         // this returns 401.
1146                         uri:    "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
1147                         header: authHeader,
1148                         expect: nil,
1149                 },
1150                 {
1151                         uri:     "download.example.com/users/active/foo_file_in_dir/",
1152                         header:  authHeader,
1153                         expect:  []string{"dir1/"},
1154                         cutDirs: 3,
1155                 },
1156                 {
1157                         uri:     "download.example.com/users/active/foo_file_in_dir/dir1/",
1158                         header:  authHeader,
1159                         expect:  []string{"bar"},
1160                         cutDirs: 4,
1161                 },
1162                 {
1163                         uri:     "download.example.com/",
1164                         header:  authHeader,
1165                         expect:  []string{"users/"},
1166                         cutDirs: 0,
1167                 },
1168                 {
1169                         uri:      "download.example.com/users",
1170                         header:   authHeader,
1171                         redirect: "/users/",
1172                         expect:   []string{"active/"},
1173                         cutDirs:  1,
1174                 },
1175                 {
1176                         uri:     "download.example.com/users/",
1177                         header:  authHeader,
1178                         expect:  []string{"active/"},
1179                         cutDirs: 1,
1180                 },
1181                 {
1182                         uri:      "download.example.com/users/active",
1183                         header:   authHeader,
1184                         redirect: "/users/active/",
1185                         expect:   []string{"foo_file_in_dir/"},
1186                         cutDirs:  2,
1187                 },
1188                 {
1189                         uri:     "download.example.com/users/active/",
1190                         header:  authHeader,
1191                         expect:  []string{"foo_file_in_dir/"},
1192                         cutDirs: 2,
1193                 },
1194                 {
1195                         uri:     "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
1196                         header:  nil,
1197                         expect:  []string{"dir1/foo", "dir1/bar"},
1198                         cutDirs: 4,
1199                 },
1200                 {
1201                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
1202                         header:  nil,
1203                         expect:  []string{"dir1/foo", "dir1/bar"},
1204                         cutDirs: 2,
1205                 },
1206                 {
1207                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken,
1208                         header:  nil,
1209                         expect:  []string{"dir1/foo", "dir1/bar"},
1210                         cutDirs: 2,
1211                 },
1212                 {
1213                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID,
1214                         header:  authHeader,
1215                         expect:  []string{"dir1/foo", "dir1/bar"},
1216                         cutDirs: 1,
1217                 },
1218                 {
1219                         uri:      "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
1220                         header:   authHeader,
1221                         redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
1222                         expect:   []string{"foo", "bar"},
1223                         cutDirs:  2,
1224                 },
1225                 {
1226                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
1227                         header:  authHeader,
1228                         expect:  []string{"foo", "bar"},
1229                         cutDirs: 3,
1230                 },
1231                 {
1232                         uri:      arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
1233                         header:   authHeader,
1234                         redirect: "/dir1/",
1235                         expect:   []string{"foo", "bar"},
1236                         cutDirs:  1,
1237                 },
1238                 {
1239                         uri:    "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
1240                         header: authHeader,
1241                         expect: nil,
1242                 },
1243                 {
1244                         uri:     "download.example.com/c=" + arvadostest.WazVersion1Collection,
1245                         header:  authHeader,
1246                         expect:  []string{"waz"},
1247                         cutDirs: 1,
1248                 },
1249                 {
1250                         uri:     "download.example.com/by_id/" + arvadostest.WazVersion1Collection,
1251                         header:  authHeader,
1252                         expect:  []string{"waz"},
1253                         cutDirs: 2,
1254                 },
1255                 {
1256                         uri:     "download.example.com/users/active/This filter group/",
1257                         header:  authHeader,
1258                         expect:  []string{"A Subproject/"},
1259                         cutDirs: 3,
1260                 },
1261                 {
1262                         uri:     "download.example.com/users/active/This filter group/A Subproject",
1263                         header:  authHeader,
1264                         expect:  []string{"baz_file/"},
1265                         cutDirs: 4,
1266                 },
1267                 {
1268                         uri:     "download.example.com/by_id/" + arvadostest.AFilterGroupUUID,
1269                         header:  authHeader,
1270                         expect:  []string{"A Subproject/"},
1271                         cutDirs: 2,
1272                 },
1273                 {
1274                         uri:     "download.example.com/by_id/" + arvadostest.AFilterGroupUUID + "/A Subproject",
1275                         header:  authHeader,
1276                         expect:  []string{"baz_file/"},
1277                         cutDirs: 3,
1278                 },
1279         } {
1280                 comment := check.Commentf("HTML: %q => %q", trial.uri, trial.expect)
1281                 resp := httptest.NewRecorder()
1282                 u := mustParseURL("//" + trial.uri)
1283                 req := &http.Request{
1284                         Method:     "GET",
1285                         Host:       u.Host,
1286                         URL:        u,
1287                         RequestURI: u.RequestURI(),
1288                         Header:     copyHeader(trial.header),
1289                 }
1290                 s.handler.ServeHTTP(resp, req)
1291                 var cookies []*http.Cookie
1292                 for resp.Code == http.StatusSeeOther {
1293                         u, _ := req.URL.Parse(resp.Header().Get("Location"))
1294                         req = &http.Request{
1295                                 Method:     "GET",
1296                                 Host:       u.Host,
1297                                 URL:        u,
1298                                 RequestURI: u.RequestURI(),
1299                                 Header:     copyHeader(trial.header),
1300                         }
1301                         cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
1302                         for _, c := range cookies {
1303                                 req.AddCookie(c)
1304                         }
1305                         resp = httptest.NewRecorder()
1306                         s.handler.ServeHTTP(resp, req)
1307                 }
1308                 if trial.redirect != "" {
1309                         c.Check(req.URL.Path, check.Equals, trial.redirect, comment)
1310                 }
1311                 if trial.expect == nil {
1312                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1313                 } else {
1314                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
1315                         for _, e := range trial.expect {
1316                                 e = strings.Replace(e, " ", "%20", -1)
1317                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`, comment)
1318                         }
1319                         c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`, comment)
1320                 }
1321
1322                 comment = check.Commentf("WebDAV: %q => %q", trial.uri, trial.expect)
1323                 req = &http.Request{
1324                         Method:     "OPTIONS",
1325                         Host:       u.Host,
1326                         URL:        u,
1327                         RequestURI: u.RequestURI(),
1328                         Header:     copyHeader(trial.header),
1329                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
1330                 }
1331                 resp = httptest.NewRecorder()
1332                 s.handler.ServeHTTP(resp, req)
1333                 if trial.expect == nil {
1334                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1335                 } else {
1336                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
1337                 }
1338
1339                 req = &http.Request{
1340                         Method:     "PROPFIND",
1341                         Host:       u.Host,
1342                         URL:        u,
1343                         RequestURI: u.RequestURI(),
1344                         Header:     copyHeader(trial.header),
1345                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
1346                 }
1347                 resp = httptest.NewRecorder()
1348                 s.handler.ServeHTTP(resp, req)
1349                 if trial.expect == nil {
1350                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1351                 } else {
1352                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment)
1353                         for _, e := range trial.expect {
1354                                 if strings.HasSuffix(e, "/") {
1355                                         e = filepath.Join(u.Path, e) + "/"
1356                                 } else {
1357                                         e = filepath.Join(u.Path, e)
1358                                 }
1359                                 e = strings.Replace(e, " ", "%20", -1)
1360                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+e+`</D:href>.*`, comment)
1361                         }
1362                 }
1363         }
1364 }
1365
1366 func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
1367         arv := arvados.NewClientFromEnv()
1368         var newCollection arvados.Collection
1369         err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1370                 "collection": map[string]string{
1371                         "owner_uuid":    arvadostest.ActiveUserUUID,
1372                         "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
1373                         "name":          "keep-web test collection",
1374                 },
1375                 "ensure_unique_name": true,
1376         })
1377         c.Assert(err, check.IsNil)
1378         defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
1379
1380         var updated arvados.Collection
1381         for _, fnm := range []string{"foo.txt", "bar.txt"} {
1382                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
1383                 u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
1384                 req := &http.Request{
1385                         Method:     "DELETE",
1386                         Host:       u.Host,
1387                         URL:        u,
1388                         RequestURI: u.RequestURI(),
1389                         Header: http.Header{
1390                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
1391                         },
1392                 }
1393                 resp := httptest.NewRecorder()
1394                 s.handler.ServeHTTP(resp, req)
1395                 c.Check(resp.Code, check.Equals, http.StatusNoContent)
1396
1397                 updated = arvados.Collection{}
1398                 err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
1399                 c.Check(err, check.IsNil)
1400                 c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`)
1401                 c.Logf("updated manifest_text %q", updated.ManifestText)
1402         }
1403         c.Check(updated.ManifestText, check.Equals, "")
1404 }
1405
1406 func (s *IntegrationSuite) TestFileContentType(c *check.C) {
1407         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1408
1409         client := arvados.NewClientFromEnv()
1410         client.AuthToken = arvadostest.ActiveToken
1411         arv, err := arvadosclient.New(client)
1412         c.Assert(err, check.Equals, nil)
1413         kc, err := keepclient.MakeKeepClient(arv)
1414         c.Assert(err, check.Equals, nil)
1415
1416         fs, err := (&arvados.Collection{}).FileSystem(client, kc)
1417         c.Assert(err, check.IsNil)
1418
1419         trials := []struct {
1420                 filename    string
1421                 content     string
1422                 contentType string
1423         }{
1424                 {"picture.txt", "BMX bikes are small this year\n", "text/plain; charset=utf-8"},
1425                 {"picture.bmp", "BMX bikes are small this year\n", "image/(x-ms-)?bmp"},
1426                 {"picture.jpg", "BMX bikes are small this year\n", "image/jpeg"},
1427                 {"picture1", "BMX bikes are small this year\n", "image/bmp"},            // content sniff; "BM" is the magic signature for .bmp
1428                 {"picture2", "Cars are small this year\n", "text/plain; charset=utf-8"}, // content sniff
1429         }
1430         for _, trial := range trials {
1431                 f, err := fs.OpenFile(trial.filename, os.O_CREATE|os.O_WRONLY, 0777)
1432                 c.Assert(err, check.IsNil)
1433                 _, err = f.Write([]byte(trial.content))
1434                 c.Assert(err, check.IsNil)
1435                 c.Assert(f.Close(), check.IsNil)
1436         }
1437         mtxt, err := fs.MarshalManifest(".")
1438         c.Assert(err, check.IsNil)
1439         var coll arvados.Collection
1440         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1441                 "collection": map[string]string{
1442                         "manifest_text": mtxt,
1443                 },
1444         })
1445         c.Assert(err, check.IsNil)
1446
1447         for _, trial := range trials {
1448                 u, _ := url.Parse("http://download.example.com/by_id/" + coll.UUID + "/" + trial.filename)
1449                 req := &http.Request{
1450                         Method:     "GET",
1451                         Host:       u.Host,
1452                         URL:        u,
1453                         RequestURI: u.RequestURI(),
1454                         Header: http.Header{
1455                                 "Authorization": {"Bearer " + client.AuthToken},
1456                         },
1457                 }
1458                 resp := httptest.NewRecorder()
1459                 s.handler.ServeHTTP(resp, req)
1460                 c.Check(resp.Code, check.Equals, http.StatusOK)
1461                 c.Check(resp.Header().Get("Content-Type"), check.Matches, trial.contentType)
1462                 c.Check(resp.Body.String(), check.Equals, trial.content)
1463         }
1464 }
1465
1466 func (s *IntegrationSuite) TestKeepClientBlockCache(c *check.C) {
1467         s.handler.Cluster.Collections.WebDAVCache.MaxBlockEntries = 42
1468         c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Not(check.Equals), 42)
1469         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/t=" + arvadostest.ActiveToken + "/foo")
1470         req := &http.Request{
1471                 Method:     "GET",
1472                 Host:       u.Host,
1473                 URL:        u,
1474                 RequestURI: u.RequestURI(),
1475         }
1476         resp := httptest.NewRecorder()
1477         s.handler.ServeHTTP(resp, req)
1478         c.Check(resp.Code, check.Equals, http.StatusOK)
1479         c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Equals, 42)
1480 }
1481
1482 // Writing to a collection shouldn't affect its entry in the
1483 // PDH-to-manifest cache.
1484 func (s *IntegrationSuite) TestCacheWriteCollectionSamePDH(c *check.C) {
1485         arv, err := arvadosclient.MakeArvadosClient()
1486         c.Assert(err, check.Equals, nil)
1487         arv.ApiToken = arvadostest.ActiveToken
1488
1489         u := mustParseURL("http://x.example/testfile")
1490         req := &http.Request{
1491                 Method:     "GET",
1492                 Host:       u.Host,
1493                 URL:        u,
1494                 RequestURI: u.RequestURI(),
1495                 Header:     http.Header{"Authorization": {"Bearer " + arv.ApiToken}},
1496         }
1497
1498         checkWithID := func(id string, status int) {
1499                 req.URL.Host = strings.Replace(id, "+", "-", -1) + ".example"
1500                 req.Host = req.URL.Host
1501                 resp := httptest.NewRecorder()
1502                 s.handler.ServeHTTP(resp, req)
1503                 c.Check(resp.Code, check.Equals, status)
1504         }
1505
1506         var colls [2]arvados.Collection
1507         for i := range colls {
1508                 err := arv.Create("collections",
1509                         map[string]interface{}{
1510                                 "ensure_unique_name": true,
1511                                 "collection": map[string]interface{}{
1512                                         "name": "test collection",
1513                                 },
1514                         }, &colls[i])
1515                 c.Assert(err, check.Equals, nil)
1516         }
1517
1518         // Populate cache with empty collection
1519         checkWithID(colls[0].PortableDataHash, http.StatusNotFound)
1520
1521         // write a file to colls[0]
1522         reqPut := *req
1523         reqPut.Method = "PUT"
1524         reqPut.URL.Host = colls[0].UUID + ".example"
1525         reqPut.Host = req.URL.Host
1526         reqPut.Body = ioutil.NopCloser(bytes.NewBufferString("testdata"))
1527         resp := httptest.NewRecorder()
1528         s.handler.ServeHTTP(resp, &reqPut)
1529         c.Check(resp.Code, check.Equals, http.StatusCreated)
1530
1531         // new file should not appear in colls[1]
1532         checkWithID(colls[1].PortableDataHash, http.StatusNotFound)
1533         checkWithID(colls[1].UUID, http.StatusNotFound)
1534
1535         checkWithID(colls[0].UUID, http.StatusOK)
1536 }
1537
1538 func copyHeader(h http.Header) http.Header {
1539         hc := http.Header{}
1540         for k, v := range h {
1541                 hc[k] = append([]string(nil), v...)
1542         }
1543         return hc
1544 }
1545
1546 func (s *IntegrationSuite) checkUploadDownloadRequest(c *check.C, req *http.Request,
1547         successCode int, direction string, perm bool, userUuid, collectionUuid, collectionPDH, filepath string) {
1548
1549         client := arvados.NewClientFromEnv()
1550         client.AuthToken = arvadostest.AdminToken
1551         var logentries arvados.LogList
1552         limit1 := 1
1553         err := client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil,
1554                 arvados.ResourceListParams{
1555                         Limit: &limit1,
1556                         Order: "created_at desc"})
1557         c.Check(err, check.IsNil)
1558         c.Check(logentries.Items, check.HasLen, 1)
1559         lastLogId := logentries.Items[0].ID
1560         c.Logf("lastLogId: %d", lastLogId)
1561
1562         var logbuf bytes.Buffer
1563         logger := logrus.New()
1564         logger.Out = &logbuf
1565         resp := httptest.NewRecorder()
1566         req = req.WithContext(ctxlog.Context(context.Background(), logger))
1567         s.handler.ServeHTTP(resp, req)
1568
1569         if perm {
1570                 c.Check(resp.Result().StatusCode, check.Equals, successCode)
1571                 c.Check(logbuf.String(), check.Matches, `(?ms).*msg="File `+direction+`".*`)
1572                 c.Check(logbuf.String(), check.Not(check.Matches), `(?ms).*level=error.*`)
1573
1574                 deadline := time.Now().Add(time.Second)
1575                 for {
1576                         c.Assert(time.Now().After(deadline), check.Equals, false, check.Commentf("timed out waiting for log entry"))
1577                         logentries = arvados.LogList{}
1578                         err = client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil,
1579                                 arvados.ResourceListParams{
1580                                         Filters: []arvados.Filter{
1581                                                 {Attr: "event_type", Operator: "=", Operand: "file_" + direction},
1582                                                 {Attr: "object_uuid", Operator: "=", Operand: userUuid},
1583                                         },
1584                                         Limit: &limit1,
1585                                         Order: "created_at desc",
1586                                 })
1587                         c.Assert(err, check.IsNil)
1588                         if len(logentries.Items) > 0 &&
1589                                 logentries.Items[0].ID > lastLogId &&
1590                                 logentries.Items[0].ObjectUUID == userUuid &&
1591                                 logentries.Items[0].Properties["collection_uuid"] == collectionUuid &&
1592                                 (collectionPDH == "" || logentries.Items[0].Properties["portable_data_hash"] == collectionPDH) &&
1593                                 logentries.Items[0].Properties["collection_file_path"] == filepath {
1594                                 break
1595                         }
1596                         c.Logf("logentries.Items: %+v", logentries.Items)
1597                         time.Sleep(50 * time.Millisecond)
1598                 }
1599         } else {
1600                 c.Check(resp.Result().StatusCode, check.Equals, http.StatusForbidden)
1601                 c.Check(logbuf.String(), check.Equals, "")
1602         }
1603 }
1604
1605 func (s *IntegrationSuite) TestDownloadLoggingPermission(c *check.C) {
1606         u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
1607
1608         s.handler.Cluster.Collections.TrustAllContent = true
1609
1610         for _, adminperm := range []bool{true, false} {
1611                 for _, userperm := range []bool{true, false} {
1612                         s.handler.Cluster.Collections.WebDAVPermission.Admin.Download = adminperm
1613                         s.handler.Cluster.Collections.WebDAVPermission.User.Download = userperm
1614
1615                         // Test admin permission
1616                         req := &http.Request{
1617                                 Method:     "GET",
1618                                 Host:       u.Host,
1619                                 URL:        u,
1620                                 RequestURI: u.RequestURI(),
1621                                 Header: http.Header{
1622                                         "Authorization": {"Bearer " + arvadostest.AdminToken},
1623                                 },
1624                         }
1625                         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", adminperm,
1626                                 arvadostest.AdminUserUUID, arvadostest.FooCollection, arvadostest.FooCollectionPDH, "foo")
1627
1628                         // Test user permission
1629                         req = &http.Request{
1630                                 Method:     "GET",
1631                                 Host:       u.Host,
1632                                 URL:        u,
1633                                 RequestURI: u.RequestURI(),
1634                                 Header: http.Header{
1635                                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
1636                                 },
1637                         }
1638                         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", userperm,
1639                                 arvadostest.ActiveUserUUID, arvadostest.FooCollection, arvadostest.FooCollectionPDH, "foo")
1640                 }
1641         }
1642
1643         s.handler.Cluster.Collections.WebDAVPermission.User.Download = true
1644
1645         for _, tryurl := range []string{"http://" + arvadostest.MultilevelCollection1 + ".keep-web.example/dir1/subdir/file1",
1646                 "http://keep-web/users/active/multilevel_collection_1/dir1/subdir/file1"} {
1647
1648                 u = mustParseURL(tryurl)
1649                 req := &http.Request{
1650                         Method:     "GET",
1651                         Host:       u.Host,
1652                         URL:        u,
1653                         RequestURI: u.RequestURI(),
1654                         Header: http.Header{
1655                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
1656                         },
1657                 }
1658                 s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true,
1659                         arvadostest.ActiveUserUUID, arvadostest.MultilevelCollection1, arvadostest.MultilevelCollection1PDH, "dir1/subdir/file1")
1660         }
1661
1662         u = mustParseURL("http://" + strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + ".keep-web.example/foo")
1663         req := &http.Request{
1664                 Method:     "GET",
1665                 Host:       u.Host,
1666                 URL:        u,
1667                 RequestURI: u.RequestURI(),
1668                 Header: http.Header{
1669                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
1670                 },
1671         }
1672         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true,
1673                 arvadostest.ActiveUserUUID, "", arvadostest.FooCollectionPDH, "foo")
1674 }
1675
1676 func (s *IntegrationSuite) TestUploadLoggingPermission(c *check.C) {
1677         for _, adminperm := range []bool{true, false} {
1678                 for _, userperm := range []bool{true, false} {
1679
1680                         arv := arvados.NewClientFromEnv()
1681                         arv.AuthToken = arvadostest.ActiveToken
1682
1683                         var coll arvados.Collection
1684                         err := arv.RequestAndDecode(&coll,
1685                                 "POST",
1686                                 "/arvados/v1/collections",
1687                                 nil,
1688                                 map[string]interface{}{
1689                                         "ensure_unique_name": true,
1690                                         "collection": map[string]interface{}{
1691                                                 "name": "test collection",
1692                                         },
1693                                 })
1694                         c.Assert(err, check.Equals, nil)
1695
1696                         u := mustParseURL("http://" + coll.UUID + ".keep-web.example/bar")
1697
1698                         s.handler.Cluster.Collections.WebDAVPermission.Admin.Upload = adminperm
1699                         s.handler.Cluster.Collections.WebDAVPermission.User.Upload = userperm
1700
1701                         // Test admin permission
1702                         req := &http.Request{
1703                                 Method:     "PUT",
1704                                 Host:       u.Host,
1705                                 URL:        u,
1706                                 RequestURI: u.RequestURI(),
1707                                 Header: http.Header{
1708                                         "Authorization": {"Bearer " + arvadostest.AdminToken},
1709                                 },
1710                                 Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
1711                         }
1712                         s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", adminperm,
1713                                 arvadostest.AdminUserUUID, coll.UUID, "", "bar")
1714
1715                         // Test user permission
1716                         req = &http.Request{
1717                                 Method:     "PUT",
1718                                 Host:       u.Host,
1719                                 URL:        u,
1720                                 RequestURI: u.RequestURI(),
1721                                 Header: http.Header{
1722                                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
1723                                 },
1724                                 Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
1725                         }
1726                         s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", userperm,
1727                                 arvadostest.ActiveUserUUID, coll.UUID, "", "bar")
1728                 }
1729         }
1730 }
1731
1732 func (s *IntegrationSuite) TestConcurrentWrites(c *check.C) {
1733         s.handler.Cluster.Collections.WebDAVCache.TTL = arvados.Duration(time.Second * 2)
1734         lockTidyInterval = time.Second
1735         client := arvados.NewClientFromEnv()
1736         client.AuthToken = arvadostest.ActiveTokenV2
1737         // Start small, and increase concurrency (2^2, 4^2, ...)
1738         // only until hitting failure. Avoids unnecessarily long
1739         // failure reports.
1740         for n := 2; n < 16 && !c.Failed(); n = n * 2 {
1741                 c.Logf("%s: n=%d", c.TestName(), n)
1742
1743                 var coll arvados.Collection
1744                 err := client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, nil)
1745                 c.Assert(err, check.IsNil)
1746                 defer client.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
1747
1748                 var wg sync.WaitGroup
1749                 for i := 0; i < n && !c.Failed(); i++ {
1750                         i := i
1751                         wg.Add(1)
1752                         go func() {
1753                                 defer wg.Done()
1754                                 u := mustParseURL(fmt.Sprintf("http://%s.collections.example.com/i=%d", coll.UUID, i))
1755                                 resp := httptest.NewRecorder()
1756                                 req, err := http.NewRequest("MKCOL", u.String(), nil)
1757                                 c.Assert(err, check.IsNil)
1758                                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
1759                                 s.handler.ServeHTTP(resp, req)
1760                                 c.Assert(resp.Code, check.Equals, http.StatusCreated)
1761                                 for j := 0; j < n && !c.Failed(); j++ {
1762                                         j := j
1763                                         wg.Add(1)
1764                                         go func() {
1765                                                 defer wg.Done()
1766                                                 content := fmt.Sprintf("i=%d/j=%d", i, j)
1767                                                 u := mustParseURL("http://" + coll.UUID + ".collections.example.com/" + content)
1768
1769                                                 resp := httptest.NewRecorder()
1770                                                 req, err := http.NewRequest("PUT", u.String(), strings.NewReader(content))
1771                                                 c.Assert(err, check.IsNil)
1772                                                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
1773                                                 s.handler.ServeHTTP(resp, req)
1774                                                 c.Check(resp.Code, check.Equals, http.StatusCreated)
1775
1776                                                 time.Sleep(time.Second)
1777                                                 resp = httptest.NewRecorder()
1778                                                 req, err = http.NewRequest("GET", u.String(), nil)
1779                                                 c.Assert(err, check.IsNil)
1780                                                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
1781                                                 s.handler.ServeHTTP(resp, req)
1782                                                 c.Check(resp.Code, check.Equals, http.StatusOK)
1783                                                 c.Check(resp.Body.String(), check.Equals, content)
1784                                         }()
1785                                 }
1786                         }()
1787                 }
1788                 wg.Wait()
1789                 for i := 0; i < n; i++ {
1790                         u := mustParseURL(fmt.Sprintf("http://%s.collections.example.com/i=%d", coll.UUID, i))
1791                         resp := httptest.NewRecorder()
1792                         req, err := http.NewRequest("PROPFIND", u.String(), &bytes.Buffer{})
1793                         c.Assert(err, check.IsNil)
1794                         req.Header.Set("Authorization", "Bearer "+client.AuthToken)
1795                         s.handler.ServeHTTP(resp, req)
1796                         c.Assert(resp.Code, check.Equals, http.StatusMultiStatus)
1797                 }
1798         }
1799 }