services/keep-web/handler_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package keepweb
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "fmt"
  11         "io"
  12         "io/ioutil"
  13         "net/http"
  14         "net/http/httptest"
  15         "net/url"
  16         "os"
  17         "path/filepath"
  18         "regexp"
  19         "strings"
  20         "sync"
  21         "time"
  22
  23         "git.arvados.org/arvados.git/lib/config"
  24         "git.arvados.org/arvados.git/sdk/go/arvados"
  25         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  26         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  27         "git.arvados.org/arvados.git/sdk/go/auth"
  28         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  29         "git.arvados.org/arvados.git/sdk/go/keepclient"
  30         "github.com/prometheus/client_golang/prometheus"
  31         "github.com/sirupsen/logrus"
  32         "golang.org/x/net/html"
  33         check "gopkg.in/check.v1"
  34 )
  35
  36 var _ = check.Suite(&UnitSuite{})
  37
  38 func init() {
  39         arvados.DebugLocksPanicMode = true
  40 }
  41
  42 type UnitSuite struct {
  43         cluster *arvados.Cluster
  44         handler *handler
  45 }
  46
  47 func (s *UnitSuite) SetUpTest(c *check.C) {
  48         logger := ctxlog.TestLogger(c)
  49         ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), logger)
  50         ldr.Path = "-"
  51         cfg, err := ldr.Load()
  52         c.Assert(err, check.IsNil)
  53         cc, err := cfg.GetCluster("")
  54         c.Assert(err, check.IsNil)
  55         s.cluster = cc
  56         s.handler = &handler{
  57                 Cluster: cc,
  58                 Cache: cache{
  59                         cluster:  cc,
  60                         logger:   logger,
  61                         registry: prometheus.NewRegistry(),
  62                 },
  63                 metrics: newMetrics(prometheus.NewRegistry()),
  64         }
  65 }
  66
  67 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
  68         h := s.handler
  69         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
  70         req := &http.Request{
  71                 Method:     "OPTIONS",
  72                 Host:       u.Host,
  73                 URL:        u,
  74                 RequestURI: u.RequestURI(),
  75                 Header: http.Header{
  76                         "Origin":                        {"https://workbench.example"},
  77                         "Access-Control-Request-Method": {"POST"},
  78                 },
  79         }
  80
  81         // Check preflight for an allowed request
  82         resp := httptest.NewRecorder()
  83         h.ServeHTTP(resp, req)
  84         c.Check(resp.Code, check.Equals, http.StatusOK)
  85         c.Check(resp.Body.String(), check.Equals, "")
  86         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
  87         c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
  88         c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout, Cache-Control")
  89
  90         // Check preflight for a disallowed request
  91         resp = httptest.NewRecorder()
  92         req.Header.Set("Access-Control-Request-Method", "MAKE-COFFEE")
  93         h.ServeHTTP(resp, req)
  94         c.Check(resp.Body.String(), check.Equals, "")
  95         c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
  96 }
  97
  98 func (s *UnitSuite) TestWebdavPrefixAndSource(c *check.C) {
  99         for _, trial := range []struct {
 100                 method   string
 101                 path     string
 102                 prefix   string
 103                 source   string
 104                 notFound bool
 105                 seeOther bool
 106         }{
 107                 {
 108                         method: "PROPFIND",
 109                         path:   "/",
 110                 },
 111                 {
 112                         method: "PROPFIND",
 113                         path:   "/dir1",
 114                 },
 115                 {
 116                         method: "PROPFIND",
 117                         path:   "/dir1/",
 118                 },
 119                 {
 120                         method: "PROPFIND",
 121                         path:   "/dir1/foo",
 122                         prefix: "/dir1",
 123                         source: "/dir1",
 124                 },
 125                 {
 126                         method: "PROPFIND",
 127                         path:   "/prefix/dir1/foo",
 128                         prefix: "/prefix/",
 129                         source: "",
 130                 },
 131                 {
 132                         method: "PROPFIND",
 133                         path:   "/prefix/dir1/foo",
 134                         prefix: "/prefix",
 135                         source: "",
 136                 },
 137                 {
 138                         method: "PROPFIND",
 139                         path:   "/prefix/dir1/foo",
 140                         prefix: "/prefix/",
 141                         source: "/",
 142                 },
 143                 {
 144                         method: "PROPFIND",
 145                         path:   "/prefix/foo",
 146                         prefix: "/prefix/",
 147                         source: "/dir1/",
 148                 },
 149                 {
 150                         method: "GET",
 151                         path:   "/prefix/foo",
 152                         prefix: "/prefix/",
 153                         source: "/dir1/",
 154                 },
 155                 {
 156                         method: "PROPFIND",
 157                         path:   "/prefix/",
 158                         prefix: "/prefix",
 159                         source: "/dir1",
 160                 },
 161                 {
 162                         method: "PROPFIND",
 163                         path:   "/prefix",
 164                         prefix: "/prefix",
 165                         source: "/dir1/",
 166                 },
 167                 {
 168                         method:   "GET",
 169                         path:     "/prefix",
 170                         prefix:   "/prefix",
 171                         source:   "/dir1",
 172                         seeOther: true,
 173                 },
 174                 {
 175                         method:   "PROPFIND",
 176                         path:     "/dir1/foo",
 177                         prefix:   "",
 178                         source:   "/dir1",
 179                         notFound: true,
 180                 },
 181         } {
 182                 c.Logf("trial %+v", trial)
 183                 u := mustParseURL("http://" + arvadostest.FooBarDirCollection + ".keep-web.example" + trial.path)
 184                 req := &http.Request{
 185                         Method:     trial.method,
 186                         Host:       u.Host,
 187                         URL:        u,
 188                         RequestURI: u.RequestURI(),
 189                         Header: http.Header{
 190                                 "Authorization":   {"Bearer " + arvadostest.ActiveTokenV2},
 191                                 "X-Webdav-Prefix": {trial.prefix},
 192                                 "X-Webdav-Source": {trial.source},
 193                         },
 194                         Body: ioutil.NopCloser(bytes.NewReader(nil)),
 195                 }
 196
 197                 resp := httptest.NewRecorder()
 198                 s.handler.ServeHTTP(resp, req)
 199                 if trial.notFound {
 200                         c.Check(resp.Code, check.Equals, http.StatusNotFound)
 201                 } else if trial.method == "PROPFIND" {
 202                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus)
 203                         c.Check(resp.Body.String(), check.Matches, `(?ms).*>\n?$`)
 204                 } else if trial.seeOther {
 205                         c.Check(resp.Code, check.Equals, http.StatusSeeOther)
 206                 } else {
 207                         c.Check(resp.Code, check.Equals, http.StatusOK)
 208                 }
 209         }
 210 }
 211
 212 func (s *UnitSuite) TestEmptyResponse(c *check.C) {
 213         // Ensure we start with an empty cache
 214         defer os.Setenv("HOME", os.Getenv("HOME"))
 215         os.Setenv("HOME", c.MkDir())
 216
 217         for _, trial := range []struct {
 218                 dataExists    bool
 219                 sendIMSHeader bool
 220                 expectStatus  int
 221                 logRegexp     string
 222         }{
 223                 // If we return no content due to a Keep read error,
 224                 // we should emit a log message.
 225                 {false, false, http.StatusOK, `(?ms).*only wrote 0 bytes.*`},
 226
 227                 // If we return no content because the client sent an
 228                 // If-Modified-Since header, our response should be
 229                 // 304.  We still expect a "File download" log since it
 230                 // counts as a file access for auditing.
 231                 {true, true, http.StatusNotModified, `(?ms).*msg="File download".*`},
 232         } {
 233                 c.Logf("trial: %+v", trial)
 234                 arvadostest.StartKeep(2, true)
 235                 if trial.dataExists {
 236                         arv, err := arvadosclient.MakeArvadosClient()
 237                         c.Assert(err, check.IsNil)
 238                         arv.ApiToken = arvadostest.ActiveToken
 239                         kc, err := keepclient.MakeKeepClient(arv)
 240                         c.Assert(err, check.IsNil)
 241                         _, _, err = kc.PutB([]byte("foo"))
 242                         c.Assert(err, check.IsNil)
 243                 }
 244
 245                 u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
 246                 req := &http.Request{
 247                         Method:     "GET",
 248                         Host:       u.Host,
 249                         URL:        u,
 250                         RequestURI: u.RequestURI(),
 251                         Header: http.Header{
 252                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
 253                         },
 254                 }
 255                 if trial.sendIMSHeader {
 256                         req.Header.Set("If-Modified-Since", strings.Replace(time.Now().UTC().Format(time.RFC1123), "UTC", "GMT", -1))
 257                 }
 258
 259                 var logbuf bytes.Buffer
 260                 logger := logrus.New()
 261                 logger.Out = &logbuf
 262                 req = req.WithContext(ctxlog.Context(context.Background(), logger))
 263
 264                 resp := httptest.NewRecorder()
 265                 s.handler.ServeHTTP(resp, req)
 266                 c.Check(resp.Code, check.Equals, trial.expectStatus)
 267                 c.Check(resp.Body.String(), check.Equals, "")
 268
 269                 c.Log(logbuf.String())
 270                 c.Check(logbuf.String(), check.Matches, trial.logRegexp)
 271         }
 272 }
 273
 274 func (s *UnitSuite) TestInvalidUUID(c *check.C) {
 275         bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-"
 276         token := arvadostest.ActiveToken
 277         for _, trial := range []string{
 278                 "http://keep-web/c=" + bogusID + "/foo",
 279                 "http://keep-web/c=" + bogusID + "/t=" + token + "/foo",
 280                 "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo",
 281                 "http://keep-web/collections/" + bogusID + "/foo",
 282                 "http://" + bogusID + ".keep-web/" + bogusID + "/foo",
 283                 "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
 284         } {
 285                 c.Log(trial)
 286                 u := mustParseURL(trial)
 287                 req := &http.Request{
 288                         Method:     "GET",
 289                         Host:       u.Host,
 290                         URL:        u,
 291                         RequestURI: u.RequestURI(),
 292                 }
 293                 resp := httptest.NewRecorder()
 294                 s.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 295                 s.handler.ServeHTTP(resp, req)
 296                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 297         }
 298 }
 299
 300 func mustParseURL(s string) *url.URL {
 301         r, err := url.Parse(s)
 302         if err != nil {
 303                 panic("parse URL: " + s)
 304         }
 305         return r
 306 }
 307
 308 func (s *IntegrationSuite) TestVhost404(c *check.C) {
 309         for _, testURL := range []string{
 310                 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
 311                 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
 312         } {
 313                 resp := httptest.NewRecorder()
 314                 u := mustParseURL(testURL)
 315                 req := &http.Request{
 316                         Method:     "GET",
 317                         URL:        u,
 318                         RequestURI: u.RequestURI(),
 319                 }
 320                 s.handler.ServeHTTP(resp, req)
 321                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 322                 c.Check(resp.Body.String(), check.Equals, notFoundMessage+"\n")
 323         }
 324 }
 325
 326 // An authorizer modifies an HTTP request to make use of the given
 327 // token -- by adding it to a header, cookie, query param, or whatever
 328 // -- and returns the HTTP status code we should expect from keep-web if
 329 // the token is invalid.
 330 type authorizer func(*http.Request, string) int
 331
 332 // We still need to accept "OAuth2 ..." as equivalent to "Bearer ..."
 333 // for compatibility with older clients, including SDKs before 3.0.
 334 func (s *IntegrationSuite) TestVhostViaAuthzHeaderOAuth2(c *check.C) {
 335         s.doVhostRequests(c, authzViaAuthzHeaderOAuth2)
 336 }
 337 func authzViaAuthzHeaderOAuth2(r *http.Request, tok string) int {
 338         r.Header.Add("Authorization", "OAuth2 "+tok)
 339         return http.StatusUnauthorized
 340 }
 341
 342 func (s *IntegrationSuite) TestVhostViaAuthzHeaderBearer(c *check.C) {
 343         s.doVhostRequests(c, authzViaAuthzHeaderBearer)
 344 }
 345 func authzViaAuthzHeaderBearer(r *http.Request, tok string) int {
 346         r.Header.Add("Authorization", "Bearer "+tok)
 347         return http.StatusUnauthorized
 348 }
 349
 350 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
 351         s.doVhostRequests(c, authzViaCookieValue)
 352 }
 353 func authzViaCookieValue(r *http.Request, tok string) int {
 354         r.AddCookie(&http.Cookie{
 355                 Name:  "arvados_api_token",
 356                 Value: auth.EncodeTokenCookie([]byte(tok)),
 357         })
 358         return http.StatusUnauthorized
 359 }
 360
 361 func (s *IntegrationSuite) TestVhostViaHTTPBasicAuth(c *check.C) {
 362         s.doVhostRequests(c, authzViaHTTPBasicAuth)
 363 }
 364 func authzViaHTTPBasicAuth(r *http.Request, tok string) int {
 365         r.AddCookie(&http.Cookie{
 366                 Name:  "arvados_api_token",
 367                 Value: auth.EncodeTokenCookie([]byte(tok)),
 368         })
 369         return http.StatusUnauthorized
 370 }
 371
 372 func (s *IntegrationSuite) TestVhostViaHTTPBasicAuthWithExtraSpaceChars(c *check.C) {
 373         s.doVhostRequests(c, func(r *http.Request, tok string) int {
 374                 r.AddCookie(&http.Cookie{
 375                         Name:  "arvados_api_token",
 376                         Value: auth.EncodeTokenCookie([]byte(" " + tok + "\n")),
 377                 })
 378                 return http.StatusUnauthorized
 379         })
 380 }
 381
 382 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
 383         s.doVhostRequests(c, authzViaPath)
 384 }
 385 func authzViaPath(r *http.Request, tok string) int {
 386         r.URL.Path = "/t=" + tok + r.URL.Path
 387         return http.StatusNotFound
 388 }
 389
 390 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
 391         s.doVhostRequests(c, authzViaQueryString)
 392 }
 393 func authzViaQueryString(r *http.Request, tok string) int {
 394         r.URL.RawQuery = "api_token=" + tok
 395         return http.StatusUnauthorized
 396 }
 397
 398 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
 399         s.doVhostRequests(c, authzViaPOST)
 400 }
 401 func authzViaPOST(r *http.Request, tok string) int {
 402         r.Method = "POST"
 403         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 404         r.Body = ioutil.NopCloser(strings.NewReader(
 405                 url.Values{"api_token": {tok}}.Encode()))
 406         return http.StatusUnauthorized
 407 }
 408
 409 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
 410         s.doVhostRequests(c, authzViaPOST)
 411 }
 412 func authzViaXHRPOST(r *http.Request, tok string) int {
 413         r.Method = "POST"
 414         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 415         r.Header.Add("Origin", "https://origin.example")
 416         r.Body = ioutil.NopCloser(strings.NewReader(
 417                 url.Values{
 418                         "api_token":   {tok},
 419                         "disposition": {"attachment"},
 420                 }.Encode()))
 421         return http.StatusUnauthorized
 422 }
 423
 424 // Try some combinations of {url, token} using the given authorization
 425 // mechanism, and verify the result is correct.
 426 func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
 427         for _, hostPath := range []string{
 428                 arvadostest.FooCollection + ".example.com/foo",
 429                 arvadostest.FooCollection + "--collections.example.com/foo",
 430                 arvadostest.FooCollection + "--collections.example.com/_/foo",
 431                 arvadostest.FooCollectionPDH + ".example.com/foo",
 432                 strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + "--collections.example.com/foo",
 433                 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
 434         } {
 435                 c.Log("doRequests: ", hostPath)
 436                 s.doVhostRequestsWithHostPath(c, authz, hostPath)
 437         }
 438 }
 439
 440 func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
 441         for _, tok := range []string{
 442                 arvadostest.ActiveToken,
 443                 arvadostest.ActiveToken[:15],
 444                 arvadostest.SpectatorToken,
 445                 "bogus",
 446                 "",
 447         } {
 448                 u := mustParseURL("http://" + hostPath)
 449                 req := &http.Request{
 450                         Method:     "GET",
 451                         Host:       u.Host,
 452                         URL:        u,
 453                         RequestURI: u.RequestURI(),
 454                         Header:     http.Header{},
 455                 }
 456                 failCode := authz(req, tok)
 457                 req, resp := s.doReq(req)
 458                 code, body := resp.Code, resp.Body.String()
 459
 460                 // If the initial request had a (non-empty) token
 461                 // showing in the query string, we should have been
 462                 // redirected in order to hide it in a cookie.
 463                 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
 464
 465                 if tok == arvadostest.ActiveToken {
 466                         c.Check(code, check.Equals, http.StatusOK)
 467                         c.Check(body, check.Equals, "foo")
 468                 } else {
 469                         c.Check(code >= 400, check.Equals, true)
 470                         c.Check(code < 500, check.Equals, true)
 471                         if tok == arvadostest.SpectatorToken {
 472                                 // Valid token never offers to retry
 473                                 // with different credentials.
 474                                 c.Check(code, check.Equals, http.StatusNotFound)
 475                         } else {
 476                                 // Invalid token can ask to retry
 477                                 // depending on the authz method.
 478                                 c.Check(code, check.Equals, failCode)
 479                         }
 480                         if code == 404 {
 481                                 c.Check(body, check.Equals, notFoundMessage+"\n")
 482                         } else {
 483                                 c.Check(body, check.Equals, unauthorizedMessage+"\n")
 484                         }
 485                 }
 486         }
 487 }
 488
 489 func (s *IntegrationSuite) TestVhostPortMatch(c *check.C) {
 490         for _, host := range []string{"download.example.com", "DOWNLOAD.EXAMPLE.COM"} {
 491                 for _, port := range []string{"80", "443", "8000"} {
 492                         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = fmt.Sprintf("download.example.com:%v", port)
 493                         u := mustParseURL(fmt.Sprintf("http://%v/by_id/%v/foo", host, arvadostest.FooCollection))
 494                         req := &http.Request{
 495                                 Method:     "GET",
 496                                 Host:       u.Host,
 497                                 URL:        u,
 498                                 RequestURI: u.RequestURI(),
 499                                 Header:     http.Header{"Authorization": []string{"Bearer " + arvadostest.ActiveToken}},
 500                         }
 501                         req, resp := s.doReq(req)
 502                         code, _ := resp.Code, resp.Body.String()
 503
 504                         if port == "8000" {
 505                                 c.Check(code, check.Equals, 401)
 506                         } else {
 507                                 c.Check(code, check.Equals, 200)
 508                         }
 509                 }
 510         }
 511 }
 512
 513 func (s *IntegrationSuite) do(method string, urlstring string, token string, hdr http.Header) (*http.Request, *httptest.ResponseRecorder) {
 514         u := mustParseURL(urlstring)
 515         if hdr == nil && token != "" {
 516                 hdr = http.Header{"Authorization": {"Bearer " + token}}
 517         } else if hdr == nil {
 518                 hdr = http.Header{}
 519         } else if token != "" {
 520                 panic("must not pass both token and hdr")
 521         }
 522         return s.doReq(&http.Request{
 523                 Method:     method,
 524                 Host:       u.Host,
 525                 URL:        u,
 526                 RequestURI: u.RequestURI(),
 527                 Header:     hdr,
 528         })
 529 }
 530
 531 func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
 532         resp := httptest.NewRecorder()
 533         s.handler.ServeHTTP(resp, req)
 534         if resp.Code != http.StatusSeeOther {
 535                 return req, resp
 536         }
 537         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 538         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 539         req = &http.Request{
 540                 Method:     "GET",
 541                 Host:       u.Host,
 542                 URL:        u,
 543                 RequestURI: u.RequestURI(),
 544                 Header:     http.Header{},
 545         }
 546         for _, c := range cookies {
 547                 req.AddCookie(c)
 548         }
 549         return s.doReq(req)
 550 }
 551
 552 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
 553         s.testVhostRedirectTokenToCookie(c, "GET",
 554                 arvadostest.FooCollection+".example.com/foo",
 555                 "?api_token="+arvadostest.ActiveToken,
 556                 nil,
 557                 "",
 558                 http.StatusOK,
 559                 "foo",
 560         )
 561 }
 562
 563 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
 564         s.testVhostRedirectTokenToCookie(c, "GET",
 565                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
 566                 "",
 567                 nil,
 568                 "",
 569                 http.StatusOK,
 570                 "foo",
 571         )
 572 }
 573
 574 func (s *IntegrationSuite) TestCollectionSharingToken(c *check.C) {
 575         s.testVhostRedirectTokenToCookie(c, "GET",
 576                 "example.com/c="+arvadostest.FooFileCollectionUUID+"/t="+arvadostest.FooFileCollectionSharingToken+"/foo",
 577                 "",
 578                 nil,
 579                 "",
 580                 http.StatusOK,
 581                 "foo",
 582         )
 583         // Same valid sharing token, but requesting a different collection
 584         s.testVhostRedirectTokenToCookie(c, "GET",
 585                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.FooFileCollectionSharingToken+"/foo",
 586                 "",
 587                 nil,
 588                 "",
 589                 http.StatusNotFound,
 590                 regexp.QuoteMeta(notFoundMessage+"\n"),
 591         )
 592 }
 593
 594 // Bad token in URL is 404 Not Found because it doesn't make sense to
 595 // retry the same URL with different authorization.
 596 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
 597         s.testVhostRedirectTokenToCookie(c, "GET",
 598                 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
 599                 "",
 600                 nil,
 601                 "",
 602                 http.StatusNotFound,
 603                 regexp.QuoteMeta(notFoundMessage+"\n"),
 604         )
 605 }
 606
 607 // Bad token in a cookie (even if it got there via our own
 608 // query-string-to-cookie redirect) is, in principle, retryable via
 609 // wb2-login-and-redirect flow.
 610 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
 611         // Inline
 612         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 613                 arvadostest.FooCollection+".example.com/foo",
 614                 "?api_token=thisisabogustoken",
 615                 http.Header{"Sec-Fetch-Mode": {"navigate"}},
 616                 "",
 617                 http.StatusSeeOther,
 618                 "",
 619         )
 620         u, err := url.Parse(resp.Header().Get("Location"))
 621         c.Assert(err, check.IsNil)
 622         c.Logf("redirected to %s", u)
 623         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 624         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 625         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
 626
 627         // Download/attachment indicated by ?disposition=attachment
 628         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 629                 arvadostest.FooCollection+".example.com/foo",
 630                 "?api_token=thisisabogustoken&disposition=attachment",
 631                 http.Header{"Sec-Fetch-Mode": {"navigate"}},
 632                 "",
 633                 http.StatusSeeOther,
 634                 "",
 635         )
 636         u, err = url.Parse(resp.Header().Get("Location"))
 637         c.Assert(err, check.IsNil)
 638         c.Logf("redirected to %s", u)
 639         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 640         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "")
 641         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 642
 643         // Download/attachment indicated by vhost
 644         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 645                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
 646                 "?api_token=thisisabogustoken",
 647                 http.Header{"Sec-Fetch-Mode": {"navigate"}},
 648                 "",
 649                 http.StatusSeeOther,
 650                 "",
 651         )
 652         u, err = url.Parse(resp.Header().Get("Location"))
 653         c.Assert(err, check.IsNil)
 654         c.Logf("redirected to %s", u)
 655         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 656         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "")
 657         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 658
 659         // Without "Sec-Fetch-Mode: navigate" header, just 401.
 660         s.testVhostRedirectTokenToCookie(c, "GET",
 661                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
 662                 "?api_token=thisisabogustoken",
 663                 http.Header{"Sec-Fetch-Mode": {"cors"}},
 664                 "",
 665                 http.StatusUnauthorized,
 666                 regexp.QuoteMeta(unauthorizedMessage+"\n"),
 667         )
 668         s.testVhostRedirectTokenToCookie(c, "GET",
 669                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
 670                 "?api_token=thisisabogustoken",
 671                 nil,
 672                 "",
 673                 http.StatusUnauthorized,
 674                 regexp.QuoteMeta(unauthorizedMessage+"\n"),
 675         )
 676 }
 677
 678 func (s *IntegrationSuite) TestVhostRedirectWithNoCache(c *check.C) {
 679         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 680                 arvadostest.FooCollection+".example.com/foo",
 681                 "?api_token=thisisabogustoken",
 682                 http.Header{
 683                         "Sec-Fetch-Mode": {"navigate"},
 684                         "Cache-Control":  {"no-cache"},
 685                 },
 686                 "",
 687                 http.StatusSeeOther,
 688                 "",
 689         )
 690         u, err := url.Parse(resp.Header().Get("Location"))
 691         c.Assert(err, check.IsNil)
 692         c.Logf("redirected to %s", u)
 693         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 694         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 695         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
 696 }
 697
 698 func (s *IntegrationSuite) TestNoTokenWorkbench2LoginFlow(c *check.C) {
 699         for _, trial := range []struct {
 700                 anonToken    bool
 701                 cacheControl string
 702         }{
 703                 {},
 704                 {cacheControl: "no-cache"},
 705                 {anonToken: true},
 706                 {anonToken: true, cacheControl: "no-cache"},
 707         } {
 708                 c.Logf("trial: %+v", trial)
 709
 710                 if trial.anonToken {
 711                         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 712                 } else {
 713                         s.handler.Cluster.Users.AnonymousUserToken = ""
 714                 }
 715                 req, err := http.NewRequest("GET", "http://"+arvadostest.FooCollection+".example.com/foo", nil)
 716                 c.Assert(err, check.IsNil)
 717                 req.Header.Set("Sec-Fetch-Mode", "navigate")
 718                 if trial.cacheControl != "" {
 719                         req.Header.Set("Cache-Control", trial.cacheControl)
 720                 }
 721                 resp := httptest.NewRecorder()
 722                 s.handler.ServeHTTP(resp, req)
 723                 c.Check(resp.Code, check.Equals, http.StatusSeeOther)
 724                 u, err := url.Parse(resp.Header().Get("Location"))
 725                 c.Assert(err, check.IsNil)
 726                 c.Logf("redirected to %q", u)
 727                 c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 728                 c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 729                 c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
 730         }
 731 }
 732
 733 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
 734         s.testVhostRedirectTokenToCookie(c, "GET",
 735                 "example.com/c="+arvadostest.FooCollection+"/foo",
 736                 "?api_token="+arvadostest.ActiveToken,
 737                 nil,
 738                 "",
 739                 http.StatusBadRequest,
 740                 regexp.QuoteMeta("cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n"),
 741         )
 742 }
 743
 744 // If client requests an attachment by putting ?disposition=attachment
 745 // in the query string, and gets redirected, the redirect target
 746 // should respond with an attachment.
 747 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
 748         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 749                 arvadostest.FooCollection+".example.com/foo",
 750                 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
 751                 nil,
 752                 "",
 753                 http.StatusOK,
 754                 "foo",
 755         )
 756         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 757 }
 758
 759 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
 760         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 761         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 762                 "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
 763                 "?api_token="+arvadostest.ActiveToken,
 764                 nil,
 765                 "",
 766                 http.StatusOK,
 767                 "foo",
 768         )
 769         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 770 }
 771
 772 func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
 773         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 774         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 775                 "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
 776                 "?api_token="+arvadostest.ActiveToken,
 777                 nil,
 778                 "",
 779                 http.StatusOK,
 780                 "waz",
 781         )
 782         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 783         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 784                 "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz",
 785                 "?api_token="+arvadostest.ActiveToken,
 786                 nil,
 787                 "",
 788                 http.StatusOK,
 789                 "waz",
 790         )
 791         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 792 }
 793
 794 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
 795         s.handler.Cluster.Collections.TrustAllContent = true
 796         s.testVhostRedirectTokenToCookie(c, "GET",
 797                 "example.com/c="+arvadostest.FooCollection+"/foo",
 798                 "?api_token="+arvadostest.ActiveToken,
 799                 nil,
 800                 "",
 801                 http.StatusOK,
 802                 "foo",
 803         )
 804 }
 805
 806 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
 807         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
 808
 809         s.testVhostRedirectTokenToCookie(c, "GET",
 810                 "example.com/c="+arvadostest.FooCollection+"/foo",
 811                 "?api_token="+arvadostest.ActiveToken,
 812                 nil,
 813                 "",
 814                 http.StatusBadRequest,
 815                 regexp.QuoteMeta("cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n"),
 816         )
 817
 818         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 819                 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
 820                 "?api_token="+arvadostest.ActiveToken,
 821                 nil,
 822                 "",
 823                 http.StatusOK,
 824                 "foo",
 825         )
 826         c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
 827 }
 828
 829 func (s *IntegrationSuite) TestVhostRedirectMultipleTokens(c *check.C) {
 830         baseUrl := arvadostest.FooCollection + ".example.com/foo"
 831         query := url.Values{}
 832
 833         // The intent of these tests is to check that requests are redirected
 834         // correctly in the presence of multiple API tokens. The exact response
 835         // codes and content are not closely considered: they're just how
 836         // keep-web responded when we made the smallest possible fix. Changing
 837         // those responses may be okay, but you should still test all these
 838         // different cases and the associated redirect logic.
 839         query["api_token"] = []string{arvadostest.ActiveToken, arvadostest.AnonymousToken}
 840         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
 841         query["api_token"] = []string{arvadostest.ActiveToken, arvadostest.AnonymousToken, ""}
 842         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
 843         query["api_token"] = []string{arvadostest.ActiveToken, "", arvadostest.AnonymousToken}
 844         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
 845         query["api_token"] = []string{"", arvadostest.ActiveToken}
 846         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
 847
 848         expectContent := regexp.QuoteMeta(unauthorizedMessage + "\n")
 849         query["api_token"] = []string{arvadostest.AnonymousToken, "invalidtoo"}
 850         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusUnauthorized, expectContent)
 851         query["api_token"] = []string{arvadostest.AnonymousToken, ""}
 852         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusUnauthorized, expectContent)
 853         query["api_token"] = []string{"", arvadostest.AnonymousToken}
 854         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusUnauthorized, expectContent)
 855 }
 856
 857 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
 858         s.testVhostRedirectTokenToCookie(c, "POST",
 859                 arvadostest.FooCollection+".example.com/foo",
 860                 "",
 861                 http.Header{"Content-Type": {"application/x-www-form-urlencoded"}},
 862                 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
 863                 http.StatusOK,
 864                 "foo",
 865         )
 866 }
 867
 868 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
 869         s.testVhostRedirectTokenToCookie(c, "POST",
 870                 arvadostest.FooCollection+".example.com/foo",
 871                 "",
 872                 http.Header{"Content-Type": {"application/x-www-form-urlencoded"}},
 873                 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
 874                 http.StatusNotFound,
 875                 regexp.QuoteMeta(notFoundMessage+"\n"),
 876         )
 877 }
 878
 879 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
 880         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 881         s.testVhostRedirectTokenToCookie(c, "GET",
 882                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 883                 "",
 884                 nil,
 885                 "",
 886                 http.StatusOK,
 887                 "Hello world\n",
 888         )
 889 }
 890
 891 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
 892         s.handler.Cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
 893         s.testVhostRedirectTokenToCookie(c, "GET",
 894                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 895                 "",
 896                 nil,
 897                 "",
 898                 http.StatusUnauthorized,
 899                 "Authorization tokens are not accepted here: .*\n",
 900         )
 901 }
 902
 903 func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
 904         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 905
 906         client := arvados.NewClientFromEnv()
 907         client.AuthToken = arvadostest.ActiveToken
 908         fs, err := (&arvados.Collection{}).FileSystem(client, nil)
 909         c.Assert(err, check.IsNil)
 910         path := `https:\\"odd' path chars`
 911         f, err := fs.OpenFile(path, os.O_CREATE, 0777)
 912         c.Assert(err, check.IsNil)
 913         f.Close()
 914         mtxt, err := fs.MarshalManifest(".")
 915         c.Assert(err, check.IsNil)
 916         var coll arvados.Collection
 917         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 918                 "collection": map[string]string{
 919                         "manifest_text": mtxt,
 920                 },
 921         })
 922         c.Assert(err, check.IsNil)
 923
 924         u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
 925         req := &http.Request{
 926                 Method:     "GET",
 927                 Host:       u.Host,
 928                 URL:        u,
 929                 RequestURI: u.RequestURI(),
 930                 Header: http.Header{
 931                         "Authorization": {"Bearer " + client.AuthToken},
 932                 },
 933         }
 934         resp := httptest.NewRecorder()
 935         s.handler.ServeHTTP(resp, req)
 936         c.Check(resp.Code, check.Equals, http.StatusOK)
 937         doc, err := html.Parse(resp.Body)
 938         c.Assert(err, check.IsNil)
 939         pathHrefMap := getPathHrefMap(doc)
 940         c.Check(pathHrefMap, check.HasLen, 1) // the one leaf added to collection
 941         href, hasPath := pathHrefMap[path]
 942         c.Assert(hasPath, check.Equals, true) // the path is listed
 943         relUrl := mustParseURL(href)
 944         c.Check(relUrl.Path, check.Equals, "./"+path) // href can be decoded back to path
 945 }
 946
 947 func (s *IntegrationSuite) TestForwardSlashSubstitution(c *check.C) {
 948         arv := arvados.NewClientFromEnv()
 949         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 950         s.handler.Cluster.Collections.ForwardSlashNameSubstitution = "{SOLIDUS}"
 951         name := "foo/bar/baz"
 952         nameShown := strings.Replace(name, "/", "{SOLIDUS}", -1)
 953
 954         client := arvados.NewClientFromEnv()
 955         client.AuthToken = arvadostest.ActiveToken
 956         fs, err := (&arvados.Collection{}).FileSystem(client, nil)
 957         c.Assert(err, check.IsNil)
 958         f, err := fs.OpenFile("filename", os.O_CREATE, 0777)
 959         c.Assert(err, check.IsNil)
 960         f.Close()
 961         mtxt, err := fs.MarshalManifest(".")
 962         c.Assert(err, check.IsNil)
 963         var coll arvados.Collection
 964         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 965                 "collection": map[string]string{
 966                         "manifest_text": mtxt,
 967                         "name":          name,
 968                         "owner_uuid":    arvadostest.AProjectUUID,
 969                 },
 970         })
 971         c.Assert(err, check.IsNil)
 972         defer arv.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
 973
 974         base := "http://download.example.com/by_id/" + coll.OwnerUUID + "/"
 975         for tryURL, expectedAnchorText := range map[string]string{
 976                 base:                   nameShown + "/",
 977                 base + nameShown + "/": "filename",
 978         } {
 979                 u, _ := url.Parse(tryURL)
 980                 req := &http.Request{
 981                         Method:     "GET",
 982                         Host:       u.Host,
 983                         URL:        u,
 984                         RequestURI: u.RequestURI(),
 985                         Header: http.Header{
 986                                 "Authorization": {"Bearer " + client.AuthToken},
 987                         },
 988                 }
 989                 resp := httptest.NewRecorder()
 990                 s.handler.ServeHTTP(resp, req)
 991                 c.Check(resp.Code, check.Equals, http.StatusOK)
 992                 doc, err := html.Parse(resp.Body)
 993                 c.Assert(err, check.IsNil) // valid HTML
 994                 pathHrefMap := getPathHrefMap(doc)
 995                 href, hasExpected := pathHrefMap[expectedAnchorText]
 996                 c.Assert(hasExpected, check.Equals, true) // has expected anchor text
 997                 c.Assert(href, check.Not(check.Equals), "")
 998                 relUrl := mustParseURL(href)
 999                 c.Check(relUrl.Path, check.Equals, "./"+expectedAnchorText) // decoded href maps back to the anchor text
1000         }
1001 }
1002
1003 // XHRs can't follow redirect-with-cookie so they rely on method=POST
1004 // and disposition=attachment (telling us it's acceptable to respond
1005 // with content instead of a redirect) and an Origin header that gets
1006 // added automatically by the browser (telling us it's desirable to do
1007 // so).
1008 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
1009         u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
1010         req := &http.Request{
1011                 Method:     "POST",
1012                 Host:       u.Host,
1013                 URL:        u,
1014                 RequestURI: u.RequestURI(),
1015                 Header: http.Header{
1016                         "Origin":       {"https://origin.example"},
1017                         "Content-Type": {"application/x-www-form-urlencoded"},
1018                 },
1019                 Body: ioutil.NopCloser(strings.NewReader(url.Values{
1020                         "api_token":   {arvadostest.ActiveToken},
1021                         "disposition": {"attachment"},
1022                 }.Encode())),
1023         }
1024         resp := httptest.NewRecorder()
1025         s.handler.ServeHTTP(resp, req)
1026         c.Check(resp.Code, check.Equals, http.StatusOK)
1027         c.Check(resp.Body.String(), check.Equals, "foo")
1028         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
1029
1030         // GET + Origin header is representative of both AJAX GET
1031         // requests and inline images via <IMG crossorigin="anonymous"
1032         // src="...">.
1033         u.RawQuery = "api_token=" + url.QueryEscape(arvadostest.ActiveTokenV2)
1034         req = &http.Request{
1035                 Method:     "GET",
1036                 Host:       u.Host,
1037                 URL:        u,
1038                 RequestURI: u.RequestURI(),
1039                 Header: http.Header{
1040                         "Origin": {"https://origin.example"},
1041                 },
1042         }
1043         resp = httptest.NewRecorder()
1044         s.handler.ServeHTTP(resp, req)
1045         c.Check(resp.Code, check.Equals, http.StatusOK)
1046         c.Check(resp.Body.String(), check.Equals, "foo")
1047         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
1048 }
1049
1050 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString string, reqHeader http.Header, reqBody string, expectStatus int, matchRespBody string) *httptest.ResponseRecorder {
1051         if reqHeader == nil {
1052                 reqHeader = http.Header{}
1053         }
1054         u, _ := url.Parse(`http://` + hostPath + queryString)
1055         c.Logf("requesting %s", u)
1056         req := &http.Request{
1057                 Method:     method,
1058                 Host:       u.Host,
1059                 URL:        u,
1060                 RequestURI: u.RequestURI(),
1061                 Header:     reqHeader,
1062                 Body:       ioutil.NopCloser(strings.NewReader(reqBody)),
1063         }
1064
1065         resp := httptest.NewRecorder()
1066         defer func() {
1067                 c.Check(resp.Code, check.Equals, expectStatus)
1068                 c.Check(resp.Body.String(), check.Matches, matchRespBody)
1069         }()
1070
1071         s.handler.ServeHTTP(resp, req)
1072         if resp.Code != http.StatusSeeOther {
1073                 attachment, _ := regexp.MatchString(`^attachment(;|$)`, resp.Header().Get("Content-Disposition"))
1074                 // Since we're not redirecting, check that any api_token in the URL is
1075                 // handled safely.
1076                 // If there is no token in the URL, then we're good.
1077                 // Otherwise, if the response code is an error, the body is expected to
1078                 // be static content, and nothing that might maliciously introspect the
1079                 // URL. It's considered safe and allowed.
1080                 // Otherwise, if the response content has attachment disposition,
1081                 // that's considered safe for all the reasons explained in the
1082                 // safeAttachment comment in handler.go.
1083                 c.Check(!u.Query().Has("api_token") || resp.Code >= 400 || attachment, check.Equals, true)
1084                 return resp
1085         }
1086
1087         loc, err := url.Parse(resp.Header().Get("Location"))
1088         c.Assert(err, check.IsNil)
1089         c.Check(loc.Scheme, check.Equals, u.Scheme)
1090         c.Check(loc.Host, check.Equals, u.Host)
1091         c.Check(loc.RawPath, check.Equals, u.RawPath)
1092         // If the response was a redirect, it should never include an API token.
1093         c.Check(loc.Query().Has("api_token"), check.Equals, false)
1094         c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
1095         cookies := (&http.Response{Header: resp.Header()}).Cookies()
1096
1097         c.Logf("following redirect to %s", u)
1098         req = &http.Request{
1099                 Method:     "GET",
1100                 Host:       loc.Host,
1101                 URL:        loc,
1102                 RequestURI: loc.RequestURI(),
1103                 Header:     reqHeader,
1104         }
1105         for _, c := range cookies {
1106                 req.AddCookie(c)
1107         }
1108
1109         resp = httptest.NewRecorder()
1110         s.handler.ServeHTTP(resp, req)
1111
1112         if resp.Code != http.StatusSeeOther {
1113                 c.Check(resp.Header().Get("Location"), check.Equals, "")
1114         }
1115         return resp
1116 }
1117
1118 func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) {
1119         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
1120         s.testDirectoryListing(c)
1121 }
1122
1123 func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) {
1124         s.handler.Cluster.Users.AnonymousUserToken = ""
1125         s.testDirectoryListing(c)
1126 }
1127
1128 func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
1129         // The "ownership cycle" test fixtures are reachable from the
1130         // "filter group without filters" group, causing webdav's
1131         // walkfs to recurse indefinitely. Avoid that by deleting one
1132         // of the bogus fixtures.
1133         arv := arvados.NewClientFromEnv()
1134         err := arv.RequestAndDecode(nil, "DELETE", "arvados/v1/groups/zzzzz-j7d0g-cx2al9cqkmsf1hs", nil, nil)
1135         if err != nil {
1136                 c.Assert(err, check.FitsTypeOf, &arvados.TransactionError{})
1137                 c.Check(err.(*arvados.TransactionError).StatusCode, check.Equals, 404)
1138         }
1139
1140         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1141         authHeader := http.Header{
1142                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
1143         }
1144         for _, trial := range []struct {
1145                 uri      string
1146                 header   http.Header
1147                 expect   []string
1148                 redirect string
1149                 cutDirs  int
1150         }{
1151                 {
1152                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
1153                         header:  authHeader,
1154                         expect:  []string{"dir1/foo", "dir1/bar"},
1155                         cutDirs: 0,
1156                 },
1157                 {
1158                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
1159                         header:  authHeader,
1160                         expect:  []string{"foo", "bar"},
1161                         cutDirs: 1,
1162                 },
1163                 {
1164                         // URLs of this form ignore authHeader, and
1165                         // FooAndBarFilesInDirUUID isn't public, so
1166                         // this returns 401.
1167                         uri:    "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
1168                         header: authHeader,
1169                         expect: nil,
1170                 },
1171                 {
1172                         uri:     "download.example.com/users/active/foo_file_in_dir/",
1173                         header:  authHeader,
1174                         expect:  []string{"dir1/"},
1175                         cutDirs: 3,
1176                 },
1177                 {
1178                         uri:     "download.example.com/users/active/foo_file_in_dir/dir1/",
1179                         header:  authHeader,
1180                         expect:  []string{"bar"},
1181                         cutDirs: 4,
1182                 },
1183                 {
1184                         uri:     "download.example.com/",
1185                         header:  authHeader,
1186                         expect:  []string{"users/"},
1187                         cutDirs: 0,
1188                 },
1189                 {
1190                         uri:      "download.example.com/users",
1191                         header:   authHeader,
1192                         redirect: "/users/",
1193                         expect:   []string{"active/"},
1194                         cutDirs:  1,
1195                 },
1196                 {
1197                         uri:     "download.example.com/users/",
1198                         header:  authHeader,
1199                         expect:  []string{"active/"},
1200                         cutDirs: 1,
1201                 },
1202                 {
1203                         uri:      "download.example.com/users/active",
1204                         header:   authHeader,
1205                         redirect: "/users/active/",
1206                         expect:   []string{"foo_file_in_dir/"},
1207                         cutDirs:  2,
1208                 },
1209                 {
1210                         uri:     "download.example.com/users/active/",
1211                         header:  authHeader,
1212                         expect:  []string{"foo_file_in_dir/"},
1213                         cutDirs: 2,
1214                 },
1215                 {
1216                         uri:     "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
1217                         header:  nil,
1218                         expect:  []string{"dir1/foo", "dir1/bar"},
1219                         cutDirs: 4,
1220                 },
1221                 {
1222                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
1223                         header:  nil,
1224                         expect:  []string{"dir1/foo", "dir1/bar"},
1225                         cutDirs: 2,
1226                 },
1227                 {
1228                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken,
1229                         header:  nil,
1230                         expect:  []string{"dir1/foo", "dir1/bar"},
1231                         cutDirs: 2,
1232                 },
1233                 {
1234                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID,
1235                         header:  authHeader,
1236                         expect:  []string{"dir1/foo", "dir1/bar"},
1237                         cutDirs: 1,
1238                 },
1239                 {
1240                         uri:      "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
1241                         header:   authHeader,
1242                         redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
1243                         expect:   []string{"foo", "bar"},
1244                         cutDirs:  2,
1245                 },
1246                 {
1247                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
1248                         header:  authHeader,
1249                         expect:  []string{"foo", "bar"},
1250                         cutDirs: 3,
1251                 },
1252                 {
1253                         uri:      arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
1254                         header:   authHeader,
1255                         redirect: "/dir1/",
1256                         expect:   []string{"foo", "bar"},
1257                         cutDirs:  1,
1258                 },
1259                 {
1260                         uri:    "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
1261                         header: authHeader,
1262                         expect: nil,
1263                 },
1264                 {
1265                         uri:     "download.example.com/c=" + arvadostest.WazVersion1Collection,
1266                         header:  authHeader,
1267                         expect:  []string{"waz"},
1268                         cutDirs: 1,
1269                 },
1270                 {
1271                         uri:     "download.example.com/by_id/" + arvadostest.WazVersion1Collection,
1272                         header:  authHeader,
1273                         expect:  []string{"waz"},
1274                         cutDirs: 2,
1275                 },
1276                 {
1277                         uri:     "download.example.com/users/active/This filter group/",
1278                         header:  authHeader,
1279                         expect:  []string{"A Subproject/"},
1280                         cutDirs: 3,
1281                 },
1282                 {
1283                         uri:     "download.example.com/users/active/This filter group/A Subproject",
1284                         header:  authHeader,
1285                         expect:  []string{"baz_file/"},
1286                         cutDirs: 4,
1287                 },
1288                 {
1289                         uri:     "download.example.com/by_id/" + arvadostest.AFilterGroupUUID,
1290                         header:  authHeader,
1291                         expect:  []string{"A Subproject/"},
1292                         cutDirs: 2,
1293                 },
1294                 {
1295                         uri:     "download.example.com/by_id/" + arvadostest.AFilterGroupUUID + "/A Subproject",
1296                         header:  authHeader,
1297                         expect:  []string{"baz_file/"},
1298                         cutDirs: 3,
1299                 },
1300         } {
1301                 comment := check.Commentf("HTML: %q redir %q => %q", trial.uri, trial.redirect, trial.expect)
1302                 resp := httptest.NewRecorder()
1303                 u := mustParseURL("//" + trial.uri)
1304                 req := &http.Request{
1305                         Method:     "GET",
1306                         Host:       u.Host,
1307                         URL:        u,
1308                         RequestURI: u.RequestURI(),
1309                         Header:     copyHeader(trial.header),
1310                 }
1311                 s.handler.ServeHTTP(resp, req)
1312                 var cookies []*http.Cookie
1313                 for resp.Code == http.StatusSeeOther {
1314                         u, _ := req.URL.Parse(resp.Header().Get("Location"))
1315                         req = &http.Request{
1316                                 Method:     "GET",
1317                                 Host:       u.Host,
1318                                 URL:        u,
1319                                 RequestURI: u.RequestURI(),
1320                                 Header:     copyHeader(trial.header),
1321                         }
1322                         cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
1323                         for _, c := range cookies {
1324                                 req.AddCookie(c)
1325                         }
1326                         resp = httptest.NewRecorder()
1327                         s.handler.ServeHTTP(resp, req)
1328                 }
1329                 if trial.redirect != "" {
1330                         c.Check(req.URL.Path, check.Equals, trial.redirect, comment)
1331                 }
1332                 if trial.expect == nil {
1333                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1334                 } else {
1335                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
1336                         listingPageDoc, err := html.Parse(resp.Body)
1337                         c.Check(err, check.IsNil, comment) // valid HTML document
1338                         pathHrefMap := getPathHrefMap(listingPageDoc)
1339                         c.Assert(pathHrefMap, check.Not(check.HasLen), 0, comment)
1340                         for _, e := range trial.expect {
1341                                 href, hasE := pathHrefMap[e]
1342                                 c.Check(hasE, check.Equals, true, comment) // expected path is listed
1343                                 relUrl := mustParseURL(href)
1344                                 c.Check(relUrl.Path, check.Equals, "./"+e, comment) // href can be decoded back to path
1345                         }
1346                         wgetCommand := getWgetExamplePre(listingPageDoc)
1347                         wgetExpected := regexp.MustCompile(`^\$ wget .*--cut-dirs=(\d+) .*'(https?://[^']+)'$`)
1348                         wgetMatchGroups := wgetExpected.FindStringSubmatch(wgetCommand)
1349                         c.Assert(wgetMatchGroups, check.NotNil)                                     // wget command matches
1350                         c.Check(wgetMatchGroups[1], check.Equals, fmt.Sprintf("%d", trial.cutDirs)) // correct level of cut dirs in wget command
1351                         printedUrl := mustParseURL(wgetMatchGroups[2])
1352                         c.Check(printedUrl.Host, check.Equals, req.URL.Host)
1353                         c.Check(printedUrl.Path, check.Equals, req.URL.Path) // URL arg in wget command can be decoded to the right path
1354                 }
1355
1356                 comment = check.Commentf("WebDAV: %q => %q", trial.uri, trial.expect)
1357                 req = &http.Request{
1358                         Method:     "OPTIONS",
1359                         Host:       u.Host,
1360                         URL:        u,
1361                         RequestURI: u.RequestURI(),
1362                         Header:     copyHeader(trial.header),
1363                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
1364                 }
1365                 resp = httptest.NewRecorder()
1366                 s.handler.ServeHTTP(resp, req)
1367                 if trial.expect == nil {
1368                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1369                 } else {
1370                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
1371                 }
1372
1373                 req = &http.Request{
1374                         Method:     "PROPFIND",
1375                         Host:       u.Host,
1376                         URL:        u,
1377                         RequestURI: u.RequestURI(),
1378                         Header:     copyHeader(trial.header),
1379                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
1380                 }
1381                 resp = httptest.NewRecorder()
1382                 s.handler.ServeHTTP(resp, req)
1383                 // This check avoids logging a big XML document in the
1384                 // event webdav throws a 500 error after sending
1385                 // headers for a 207.
1386                 if !c.Check(strings.HasSuffix(resp.Body.String(), "Internal Server Error"), check.Equals, false) {
1387                         continue
1388                 }
1389                 if trial.expect == nil {
1390                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1391                 } else {
1392                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment)
1393                         for _, e := range trial.expect {
1394                                 if strings.HasSuffix(e, "/") {
1395                                         e = filepath.Join(u.Path, e) + "/"
1396                                 } else {
1397                                         e = filepath.Join(u.Path, e)
1398                                 }
1399                                 e = strings.Replace(e, " ", "%20", -1)
1400                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+e+`</D:href>.*`, comment)
1401                         }
1402                 }
1403         }
1404 }
1405
1406 // Shallow-traverse the HTML document, gathering the nodes satisfying the
1407 // predicate function in the output slice. If a node matches the predicate,
1408 // none of its children will be visited.
1409 func getNodes(document *html.Node, predicate func(*html.Node) bool) []*html.Node {
1410         var acc []*html.Node
1411         var traverse func(*html.Node, []*html.Node) []*html.Node
1412         traverse = func(root *html.Node, sofar []*html.Node) []*html.Node {
1413                 if root == nil {
1414                         return sofar
1415                 }
1416                 if predicate(root) {
1417                         return append(sofar, root)
1418                 }
1419                 for cur := root.FirstChild; cur != nil; cur = cur.NextSibling {
1420                         sofar = traverse(cur, sofar)
1421                 }
1422                 return sofar
1423         }
1424         return traverse(document, acc)
1425 }
1426
1427 // Returns true if a node has the attribute targetAttr with the given value
1428 func matchesAttributeValue(node *html.Node, targetAttr string, value string) bool {
1429         for _, attr := range node.Attr {
1430                 if attr.Key == targetAttr && attr.Val == value {
1431                         return true
1432                 }
1433         }
1434         return false
1435 }
1436
1437 // Concatenate the content of text-node children of node; only direct
1438 // children are visited, and any non-text children are skipped.
1439 func getNodeText(node *html.Node) string {
1440         var recv strings.Builder
1441         for c := node.FirstChild; c != nil; c = c.NextSibling {
1442                 if c.Type == html.TextNode {
1443                         recv.WriteString(c.Data)
1444                 }
1445         }
1446         return recv.String()
1447 }
1448
1449 // Returns a map from the directory listing item string (a path) to the href
1450 // value of its <a> tag (an encoded relative URL)
1451 func getPathHrefMap(document *html.Node) map[string]string {
1452         isItemATag := func(node *html.Node) bool {
1453                 return node.Type == html.ElementNode && node.Data == "a" && matchesAttributeValue(node, "class", "item")
1454         }
1455         aTags := getNodes(document, isItemATag)
1456         output := make(map[string]string)
1457         for _, elem := range aTags {
1458                 textContent := getNodeText(elem)
1459                 for _, attr := range elem.Attr {
1460                         if attr.Key == "href" {
1461                                 output[textContent] = attr.Val
1462                                 break
1463                         }
1464                 }
1465         }
1466         return output
1467 }
1468
1469 func getWgetExamplePre(document *html.Node) string {
1470         isWgetPre := func(node *html.Node) bool {
1471                 return node.Type == html.ElementNode && matchesAttributeValue(node, "id", "wget-example")
1472         }
1473         elements := getNodes(document, isWgetPre)
1474         if len(elements) != 1 {
1475                 return ""
1476         }
1477         return getNodeText(elements[0])
1478 }
1479
1480 func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
1481         arv := arvados.NewClientFromEnv()
1482         var newCollection arvados.Collection
1483         err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1484                 "collection": map[string]string{
1485                         "owner_uuid":    arvadostest.ActiveUserUUID,
1486                         "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
1487                         "name":          "keep-web test collection",
1488                 },
1489                 "ensure_unique_name": true,
1490         })
1491         c.Assert(err, check.IsNil)
1492         defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
1493
1494         var updated arvados.Collection
1495         for _, fnm := range []string{"foo.txt", "bar.txt"} {
1496                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
1497                 u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
1498                 req := &http.Request{
1499                         Method:     "DELETE",
1500                         Host:       u.Host,
1501                         URL:        u,
1502                         RequestURI: u.RequestURI(),
1503                         Header: http.Header{
1504                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
1505                         },
1506                 }
1507                 resp := httptest.NewRecorder()
1508                 s.handler.ServeHTTP(resp, req)
1509                 c.Check(resp.Code, check.Equals, http.StatusNoContent)
1510
1511                 updated = arvados.Collection{}
1512                 err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
1513                 c.Check(err, check.IsNil)
1514                 c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`)
1515                 c.Logf("updated manifest_text %q", updated.ManifestText)
1516         }
1517         c.Check(updated.ManifestText, check.Equals, "")
1518 }
1519
1520 func (s *IntegrationSuite) TestFileContentType(c *check.C) {
1521         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1522
1523         client := arvados.NewClientFromEnv()
1524         client.AuthToken = arvadostest.ActiveToken
1525         arv, err := arvadosclient.New(client)
1526         c.Assert(err, check.Equals, nil)
1527         kc, err := keepclient.MakeKeepClient(arv)
1528         c.Assert(err, check.Equals, nil)
1529
1530         fs, err := (&arvados.Collection{}).FileSystem(client, kc)
1531         c.Assert(err, check.IsNil)
1532
1533         trials := []struct {
1534                 filename    string
1535                 content     string
1536                 contentType string
1537         }{
1538                 {"picture.txt", "BMX bikes are small this year\n", "text/plain; charset=utf-8"},
1539                 {"picture.bmp", "BMX bikes are small this year\n", "image/(x-ms-)?bmp"},
1540                 {"picture.jpg", "BMX bikes are small this year\n", "image/jpeg"},
1541                 {"picture1", "BMX bikes are small this year\n", "image/bmp"},            // content sniff; "BM" is the magic signature for .bmp
1542                 {"picture2", "Cars are small this year\n", "text/plain; charset=utf-8"}, // content sniff
1543         }
1544         for _, trial := range trials {
1545                 f, err := fs.OpenFile(trial.filename, os.O_CREATE|os.O_WRONLY, 0777)
1546                 c.Assert(err, check.IsNil)
1547                 _, err = f.Write([]byte(trial.content))
1548                 c.Assert(err, check.IsNil)
1549                 c.Assert(f.Close(), check.IsNil)
1550         }
1551         mtxt, err := fs.MarshalManifest(".")
1552         c.Assert(err, check.IsNil)
1553         var coll arvados.Collection
1554         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1555                 "collection": map[string]string{
1556                         "manifest_text": mtxt,
1557                 },
1558         })
1559         c.Assert(err, check.IsNil)
1560
1561         for _, trial := range trials {
1562                 u, _ := url.Parse("http://download.example.com/by_id/" + coll.UUID + "/" + trial.filename)
1563                 req := &http.Request{
1564                         Method:     "GET",
1565                         Host:       u.Host,
1566                         URL:        u,
1567                         RequestURI: u.RequestURI(),
1568                         Header: http.Header{
1569                                 "Authorization": {"Bearer " + client.AuthToken},
1570                         },
1571                 }
1572                 resp := httptest.NewRecorder()
1573                 s.handler.ServeHTTP(resp, req)
1574                 c.Check(resp.Code, check.Equals, http.StatusOK)
1575                 c.Check(resp.Header().Get("Content-Type"), check.Matches, trial.contentType)
1576                 c.Check(resp.Body.String(), check.Equals, trial.content)
1577         }
1578 }
1579
1580 func (s *IntegrationSuite) TestCacheSize(c *check.C) {
1581         req, err := http.NewRequest("GET", "http://"+arvadostest.FooCollection+".example.com/foo", nil)
1582         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
1583         c.Assert(err, check.IsNil)
1584         resp := httptest.NewRecorder()
1585         s.handler.ServeHTTP(resp, req)
1586         c.Assert(resp.Code, check.Equals, http.StatusOK)
1587         c.Check(s.handler.Cache.sessions[arvadostest.ActiveTokenV2].client.DiskCacheSize.Percent(), check.Equals, int64(10))
1588 }
1589
1590 // Writing to a collection shouldn't affect its entry in the
1591 // PDH-to-manifest cache.
1592 func (s *IntegrationSuite) TestCacheWriteCollectionSamePDH(c *check.C) {
1593         arv, err := arvadosclient.MakeArvadosClient()
1594         c.Assert(err, check.Equals, nil)
1595         arv.ApiToken = arvadostest.ActiveToken
1596
1597         u := mustParseURL("http://x.example/testfile")
1598         req := &http.Request{
1599                 Method:     "GET",
1600                 Host:       u.Host,
1601                 URL:        u,
1602                 RequestURI: u.RequestURI(),
1603                 Header:     http.Header{"Authorization": {"Bearer " + arv.ApiToken}},
1604         }
1605
1606         checkWithID := func(id string, status int) {
1607                 req.URL.Host = strings.Replace(id, "+", "-", -1) + ".example"
1608                 req.Host = req.URL.Host
1609                 resp := httptest.NewRecorder()
1610                 s.handler.ServeHTTP(resp, req)
1611                 c.Check(resp.Code, check.Equals, status)
1612         }
1613
1614         var colls [2]arvados.Collection
1615         for i := range colls {
1616                 err := arv.Create("collections",
1617                         map[string]interface{}{
1618                                 "ensure_unique_name": true,
1619                                 "collection": map[string]interface{}{
1620                                         "name": "test collection",
1621                                 },
1622                         }, &colls[i])
1623                 c.Assert(err, check.Equals, nil)
1624         }
1625
1626         // Populate cache with empty collection
1627         checkWithID(colls[0].PortableDataHash, http.StatusNotFound)
1628
1629         // write a file to colls[0]
1630         reqPut := *req
1631         reqPut.Method = "PUT"
1632         reqPut.URL.Host = colls[0].UUID + ".example"
1633         reqPut.Host = req.URL.Host
1634         reqPut.Body = ioutil.NopCloser(bytes.NewBufferString("testdata"))
1635         resp := httptest.NewRecorder()
1636         s.handler.ServeHTTP(resp, &reqPut)
1637         c.Check(resp.Code, check.Equals, http.StatusCreated)
1638
1639         // new file should not appear in colls[1]
1640         checkWithID(colls[1].PortableDataHash, http.StatusNotFound)
1641         checkWithID(colls[1].UUID, http.StatusNotFound)
1642
1643         checkWithID(colls[0].UUID, http.StatusOK)
1644 }
1645
1646 func copyHeader(h http.Header) http.Header {
1647         hc := http.Header{}
1648         for k, v := range h {
1649                 hc[k] = append([]string(nil), v...)
1650         }
1651         return hc
1652 }
1653
1654 func (s *IntegrationSuite) checkUploadDownloadRequest(c *check.C, req *http.Request,
1655         successCode int, direction string, perm bool, userUuid, collectionUuid, collectionPDH, filepath string) {
1656
1657         client := arvados.NewClientFromEnv()
1658         client.AuthToken = arvadostest.AdminToken
1659         var logentries arvados.LogList
1660         limit1 := 1
1661         err := client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil,
1662                 arvados.ResourceListParams{
1663                         Limit: &limit1,
1664                         Order: "created_at desc"})
1665         c.Check(err, check.IsNil)
1666         c.Check(logentries.Items, check.HasLen, 1)
1667         lastLogId := logentries.Items[0].ID
1668         c.Logf("lastLogId: %d", lastLogId)
1669
1670         var logbuf bytes.Buffer
1671         logger := logrus.New()
1672         logger.Out = &logbuf
1673         resp := httptest.NewRecorder()
1674         req = req.WithContext(ctxlog.Context(context.Background(), logger))
1675         s.handler.ServeHTTP(resp, req)
1676
1677         if perm {
1678                 c.Check(resp.Result().StatusCode, check.Equals, successCode)
1679                 c.Check(logbuf.String(), check.Matches, `(?ms).*msg="File `+direction+`".*`)
1680                 c.Check(logbuf.String(), check.Not(check.Matches), `(?ms).*level=error.*`)
1681
1682                 deadline := time.Now().Add(time.Second)
1683                 for {
1684                         c.Assert(time.Now().After(deadline), check.Equals, false, check.Commentf("timed out waiting for log entry"))
1685                         logentries = arvados.LogList{}
1686                         err = client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil,
1687                                 arvados.ResourceListParams{
1688                                         Filters: []arvados.Filter{
1689                                                 {Attr: "event_type", Operator: "=", Operand: "file_" + direction},
1690                                                 {Attr: "object_uuid", Operator: "=", Operand: userUuid},
1691                                         },
1692                                         Limit: &limit1,
1693                                         Order: "created_at desc",
1694                                 })
1695                         c.Assert(err, check.IsNil)
1696                         if len(logentries.Items) > 0 &&
1697                                 logentries.Items[0].ID > lastLogId &&
1698                                 logentries.Items[0].ObjectUUID == userUuid &&
1699                                 logentries.Items[0].Properties["collection_uuid"] == collectionUuid &&
1700                                 (collectionPDH == "" || logentries.Items[0].Properties["portable_data_hash"] == collectionPDH) &&
1701                                 logentries.Items[0].Properties["collection_file_path"] == filepath {
1702                                 break
1703                         }
1704                         c.Logf("logentries.Items: %+v", logentries.Items)
1705                         time.Sleep(50 * time.Millisecond)
1706                 }
1707         } else {
1708                 c.Check(resp.Result().StatusCode, check.Equals, http.StatusForbidden)
1709                 c.Check(logbuf.String(), check.Equals, "")
1710         }
1711 }
1712
1713 func (s *IntegrationSuite) TestDownloadLoggingPermission(c *check.C) {
1714         u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
1715
1716         s.handler.Cluster.Collections.TrustAllContent = true
1717
1718         for _, adminperm := range []bool{true, false} {
1719                 for _, userperm := range []bool{true, false} {
1720                         s.handler.Cluster.Collections.WebDAVPermission.Admin.Download = adminperm
1721                         s.handler.Cluster.Collections.WebDAVPermission.User.Download = userperm
1722
1723                         // Test admin permission
1724                         req := &http.Request{
1725                                 Method:     "GET",
1726                                 Host:       u.Host,
1727                                 URL:        u,
1728                                 RequestURI: u.RequestURI(),
1729                                 Header: http.Header{
1730                                         "Authorization": {"Bearer " + arvadostest.AdminToken},
1731                                 },
1732                         }
1733                         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", adminperm,
1734                                 arvadostest.AdminUserUUID, arvadostest.FooCollection, arvadostest.FooCollectionPDH, "foo")
1735
1736                         // Test user permission
1737                         req = &http.Request{
1738                                 Method:     "GET",
1739                                 Host:       u.Host,
1740                                 URL:        u,
1741                                 RequestURI: u.RequestURI(),
1742                                 Header: http.Header{
1743                                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
1744                                 },
1745                         }
1746                         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", userperm,
1747                                 arvadostest.ActiveUserUUID, arvadostest.FooCollection, arvadostest.FooCollectionPDH, "foo")
1748                 }
1749         }
1750
1751         s.handler.Cluster.Collections.WebDAVPermission.User.Download = true
1752
1753         for _, tryurl := range []string{"http://" + arvadostest.MultilevelCollection1 + ".keep-web.example/dir1/subdir/file1",
1754                 "http://keep-web/users/active/multilevel_collection_1/dir1/subdir/file1"} {
1755
1756                 u = mustParseURL(tryurl)
1757                 req := &http.Request{
1758                         Method:     "GET",
1759                         Host:       u.Host,
1760                         URL:        u,
1761                         RequestURI: u.RequestURI(),
1762                         Header: http.Header{
1763                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
1764                         },
1765                 }
1766                 s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true,
1767                         arvadostest.ActiveUserUUID, arvadostest.MultilevelCollection1, arvadostest.MultilevelCollection1PDH, "dir1/subdir/file1")
1768         }
1769
1770         u = mustParseURL("http://" + strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + ".keep-web.example/foo")
1771         req := &http.Request{
1772                 Method:     "GET",
1773                 Host:       u.Host,
1774                 URL:        u,
1775                 RequestURI: u.RequestURI(),
1776                 Header: http.Header{
1777                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
1778                 },
1779         }
1780         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true,
1781                 arvadostest.ActiveUserUUID, "", arvadostest.FooCollectionPDH, "foo")
1782 }
1783
1784 func (s *IntegrationSuite) TestUploadLoggingPermission(c *check.C) {
1785         for _, adminperm := range []bool{true, false} {
1786                 for _, userperm := range []bool{true, false} {
1787
1788                         arv := arvados.NewClientFromEnv()
1789                         arv.AuthToken = arvadostest.ActiveToken
1790
1791                         var coll arvados.Collection
1792                         err := arv.RequestAndDecode(&coll,
1793                                 "POST",
1794                                 "/arvados/v1/collections",
1795                                 nil,
1796                                 map[string]interface{}{
1797                                         "ensure_unique_name": true,
1798                                         "collection": map[string]interface{}{
1799                                                 "name": "test collection",
1800                                         },
1801                                 })
1802                         c.Assert(err, check.Equals, nil)
1803
1804                         u := mustParseURL("http://" + coll.UUID + ".keep-web.example/bar")
1805
1806                         s.handler.Cluster.Collections.WebDAVPermission.Admin.Upload = adminperm
1807                         s.handler.Cluster.Collections.WebDAVPermission.User.Upload = userperm
1808
1809                         // Test admin permission
1810                         req := &http.Request{
1811                                 Method:     "PUT",
1812                                 Host:       u.Host,
1813                                 URL:        u,
1814                                 RequestURI: u.RequestURI(),
1815                                 Header: http.Header{
1816                                         "Authorization": {"Bearer " + arvadostest.AdminToken},
1817                                 },
1818                                 Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
1819                         }
1820                         s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", adminperm,
1821                                 arvadostest.AdminUserUUID, coll.UUID, "", "bar")
1822
1823                         // Test user permission
1824                         req = &http.Request{
1825                                 Method:     "PUT",
1826                                 Host:       u.Host,
1827                                 URL:        u,
1828                                 RequestURI: u.RequestURI(),
1829                                 Header: http.Header{
1830                                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
1831                                 },
1832                                 Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
1833                         }
1834                         s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", userperm,
1835                                 arvadostest.ActiveUserUUID, coll.UUID, "", "bar")
1836                 }
1837         }
1838 }
1839
1840 func (s *IntegrationSuite) TestConcurrentWrites(c *check.C) {
1841         s.handler.Cluster.Collections.WebDAVCache.TTL = arvados.Duration(time.Second * 2)
1842         lockTidyInterval = time.Second
1843         client := arvados.NewClientFromEnv()
1844         client.AuthToken = arvadostest.ActiveTokenV2
1845         // Start small, and increase concurrency (2^2, 4^2, ...)
1846         // only until hitting failure. Avoids unnecessarily long
1847         // failure reports.
1848         for n := 2; n < 16 && !c.Failed(); n = n * 2 {
1849                 c.Logf("%s: n=%d", c.TestName(), n)
1850
1851                 var coll arvados.Collection
1852                 err := client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, nil)
1853                 c.Assert(err, check.IsNil)
1854                 defer client.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
1855
1856                 var wg sync.WaitGroup
1857                 for i := 0; i < n && !c.Failed(); i++ {
1858                         i := i
1859                         wg.Add(1)
1860                         go func() {
1861                                 defer wg.Done()
1862                                 u := mustParseURL(fmt.Sprintf("http://%s.collections.example.com/i=%d", coll.UUID, i))
1863                                 resp := httptest.NewRecorder()
1864                                 req, err := http.NewRequest("MKCOL", u.String(), nil)
1865                                 c.Assert(err, check.IsNil)
1866                                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
1867                                 s.handler.ServeHTTP(resp, req)
1868                                 c.Assert(resp.Code, check.Equals, http.StatusCreated)
1869                                 for j := 0; j < n && !c.Failed(); j++ {
1870                                         j := j
1871                                         wg.Add(1)
1872                                         go func() {
1873                                                 defer wg.Done()
1874                                                 content := fmt.Sprintf("i=%d/j=%d", i, j)
1875                                                 u := mustParseURL("http://" + coll.UUID + ".collections.example.com/" + content)
1876
1877                                                 resp := httptest.NewRecorder()
1878                                                 req, err := http.NewRequest("PUT", u.String(), strings.NewReader(content))
1879                                                 c.Assert(err, check.IsNil)
1880                                                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
1881                                                 s.handler.ServeHTTP(resp, req)
1882                                                 c.Check(resp.Code, check.Equals, http.StatusCreated)
1883
1884                                                 time.Sleep(time.Second)
1885                                                 resp = httptest.NewRecorder()
1886                                                 req, err = http.NewRequest("GET", u.String(), nil)
1887                                                 c.Assert(err, check.IsNil)
1888                                                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
1889                                                 s.handler.ServeHTTP(resp, req)
1890                                                 c.Check(resp.Code, check.Equals, http.StatusOK)
1891                                                 c.Check(resp.Body.String(), check.Equals, content)
1892                                         }()
1893                                 }
1894                         }()
1895                 }
1896                 wg.Wait()
1897                 for i := 0; i < n; i++ {
1898                         u := mustParseURL(fmt.Sprintf("http://%s.collections.example.com/i=%d", coll.UUID, i))
1899                         resp := httptest.NewRecorder()
1900                         req, err := http.NewRequest("PROPFIND", u.String(), &bytes.Buffer{})
1901                         c.Assert(err, check.IsNil)
1902                         req.Header.Set("Authorization", "Bearer "+client.AuthToken)
1903                         s.handler.ServeHTTP(resp, req)
1904                         c.Assert(resp.Code, check.Equals, http.StatusMultiStatus)
1905                 }
1906         }
1907 }