services/keep-web/handler_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package keepweb
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "fmt"
  11         "html"
  12         "io"
  13         "io/ioutil"
  14         "net/http"
  15         "net/http/httptest"
  16         "net/url"
  17         "os"
  18         "path/filepath"
  19         "regexp"
  20         "strings"
  21         "sync"
  22         "time"
  23
  24         "git.arvados.org/arvados.git/lib/config"
  25         "git.arvados.org/arvados.git/sdk/go/arvados"
  26         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  27         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  28         "git.arvados.org/arvados.git/sdk/go/auth"
  29         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  30         "git.arvados.org/arvados.git/sdk/go/keepclient"
  31         "github.com/prometheus/client_golang/prometheus"
  32         "github.com/sirupsen/logrus"
  33         check "gopkg.in/check.v1"
  34 )
  35
  36 var _ = check.Suite(&UnitSuite{})
  37
  38 func init() {
  39         arvados.DebugLocksPanicMode = true
  40 }
  41
  42 type UnitSuite struct {
  43         cluster *arvados.Cluster
  44         handler *handler
  45 }
  46
  47 func (s *UnitSuite) SetUpTest(c *check.C) {
  48         logger := ctxlog.TestLogger(c)
  49         ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), logger)
  50         ldr.Path = "-"
  51         cfg, err := ldr.Load()
  52         c.Assert(err, check.IsNil)
  53         cc, err := cfg.GetCluster("")
  54         c.Assert(err, check.IsNil)
  55         s.cluster = cc
  56         s.handler = &handler{
  57                 Cluster: cc,
  58                 Cache: cache{
  59                         cluster:  cc,
  60                         logger:   logger,
  61                         registry: prometheus.NewRegistry(),
  62                 },
  63         }
  64 }
  65
  66 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
  67         h := s.handler
  68         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
  69         req := &http.Request{
  70                 Method:     "OPTIONS",
  71                 Host:       u.Host,
  72                 URL:        u,
  73                 RequestURI: u.RequestURI(),
  74                 Header: http.Header{
  75                         "Origin":                        {"https://workbench.example"},
  76                         "Access-Control-Request-Method": {"POST"},
  77                 },
  78         }
  79
  80         // Check preflight for an allowed request
  81         resp := httptest.NewRecorder()
  82         h.ServeHTTP(resp, req)
  83         c.Check(resp.Code, check.Equals, http.StatusOK)
  84         c.Check(resp.Body.String(), check.Equals, "")
  85         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
  86         c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
  87         c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout, Cache-Control")
  88
  89         // Check preflight for a disallowed request
  90         resp = httptest.NewRecorder()
  91         req.Header.Set("Access-Control-Request-Method", "MAKE-COFFEE")
  92         h.ServeHTTP(resp, req)
  93         c.Check(resp.Body.String(), check.Equals, "")
  94         c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
  95 }
  96
  97 func (s *UnitSuite) TestWebdavPrefixAndSource(c *check.C) {
  98         for _, trial := range []struct {
  99                 method   string
 100                 path     string
 101                 prefix   string
 102                 source   string
 103                 notFound bool
 104                 seeOther bool
 105         }{
 106                 {
 107                         method: "PROPFIND",
 108                         path:   "/",
 109                 },
 110                 {
 111                         method: "PROPFIND",
 112                         path:   "/dir1",
 113                 },
 114                 {
 115                         method: "PROPFIND",
 116                         path:   "/dir1/",
 117                 },
 118                 {
 119                         method: "PROPFIND",
 120                         path:   "/dir1/foo",
 121                         prefix: "/dir1",
 122                         source: "/dir1",
 123                 },
 124                 {
 125                         method: "PROPFIND",
 126                         path:   "/prefix/dir1/foo",
 127                         prefix: "/prefix/",
 128                         source: "",
 129                 },
 130                 {
 131                         method: "PROPFIND",
 132                         path:   "/prefix/dir1/foo",
 133                         prefix: "/prefix",
 134                         source: "",
 135                 },
 136                 {
 137                         method: "PROPFIND",
 138                         path:   "/prefix/dir1/foo",
 139                         prefix: "/prefix/",
 140                         source: "/",
 141                 },
 142                 {
 143                         method: "PROPFIND",
 144                         path:   "/prefix/foo",
 145                         prefix: "/prefix/",
 146                         source: "/dir1/",
 147                 },
 148                 {
 149                         method: "GET",
 150                         path:   "/prefix/foo",
 151                         prefix: "/prefix/",
 152                         source: "/dir1/",
 153                 },
 154                 {
 155                         method: "PROPFIND",
 156                         path:   "/prefix/",
 157                         prefix: "/prefix",
 158                         source: "/dir1",
 159                 },
 160                 {
 161                         method: "PROPFIND",
 162                         path:   "/prefix",
 163                         prefix: "/prefix",
 164                         source: "/dir1/",
 165                 },
 166                 {
 167                         method:   "GET",
 168                         path:     "/prefix",
 169                         prefix:   "/prefix",
 170                         source:   "/dir1",
 171                         seeOther: true,
 172                 },
 173                 {
 174                         method:   "PROPFIND",
 175                         path:     "/dir1/foo",
 176                         prefix:   "",
 177                         source:   "/dir1",
 178                         notFound: true,
 179                 },
 180         } {
 181                 c.Logf("trial %+v", trial)
 182                 u := mustParseURL("http://" + arvadostest.FooBarDirCollection + ".keep-web.example" + trial.path)
 183                 req := &http.Request{
 184                         Method:     trial.method,
 185                         Host:       u.Host,
 186                         URL:        u,
 187                         RequestURI: u.RequestURI(),
 188                         Header: http.Header{
 189                                 "Authorization":   {"Bearer " + arvadostest.ActiveTokenV2},
 190                                 "X-Webdav-Prefix": {trial.prefix},
 191                                 "X-Webdav-Source": {trial.source},
 192                         },
 193                         Body: ioutil.NopCloser(bytes.NewReader(nil)),
 194                 }
 195
 196                 resp := httptest.NewRecorder()
 197                 s.handler.ServeHTTP(resp, req)
 198                 if trial.notFound {
 199                         c.Check(resp.Code, check.Equals, http.StatusNotFound)
 200                 } else if trial.method == "PROPFIND" {
 201                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus)
 202                         c.Check(resp.Body.String(), check.Matches, `(?ms).*>\n?$`)
 203                 } else if trial.seeOther {
 204                         c.Check(resp.Code, check.Equals, http.StatusSeeOther)
 205                 } else {
 206                         c.Check(resp.Code, check.Equals, http.StatusOK)
 207                 }
 208         }
 209 }
 210
 211 func (s *UnitSuite) TestEmptyResponse(c *check.C) {
 212         // Ensure we start with an empty cache
 213         defer os.Setenv("HOME", os.Getenv("HOME"))
 214         os.Setenv("HOME", c.MkDir())
 215
 216         for _, trial := range []struct {
 217                 dataExists    bool
 218                 sendIMSHeader bool
 219                 expectStatus  int
 220                 logRegexp     string
 221         }{
 222                 // If we return no content due to a Keep read error,
 223                 // we should emit a log message.
 224                 {false, false, http.StatusOK, `(?ms).*only wrote 0 bytes.*`},
 225
 226                 // If we return no content because the client sent an
 227                 // If-Modified-Since header, our response should be
 228                 // 304.  We still expect a "File download" log since it
 229                 // counts as a file access for auditing.
 230                 {true, true, http.StatusNotModified, `(?ms).*msg="File download".*`},
 231         } {
 232                 c.Logf("trial: %+v", trial)
 233                 arvadostest.StartKeep(2, true)
 234                 if trial.dataExists {
 235                         arv, err := arvadosclient.MakeArvadosClient()
 236                         c.Assert(err, check.IsNil)
 237                         arv.ApiToken = arvadostest.ActiveToken
 238                         kc, err := keepclient.MakeKeepClient(arv)
 239                         c.Assert(err, check.IsNil)
 240                         _, _, err = kc.PutB([]byte("foo"))
 241                         c.Assert(err, check.IsNil)
 242                 }
 243
 244                 u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
 245                 req := &http.Request{
 246                         Method:     "GET",
 247                         Host:       u.Host,
 248                         URL:        u,
 249                         RequestURI: u.RequestURI(),
 250                         Header: http.Header{
 251                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
 252                         },
 253                 }
 254                 if trial.sendIMSHeader {
 255                         req.Header.Set("If-Modified-Since", strings.Replace(time.Now().UTC().Format(time.RFC1123), "UTC", "GMT", -1))
 256                 }
 257
 258                 var logbuf bytes.Buffer
 259                 logger := logrus.New()
 260                 logger.Out = &logbuf
 261                 req = req.WithContext(ctxlog.Context(context.Background(), logger))
 262
 263                 resp := httptest.NewRecorder()
 264                 s.handler.ServeHTTP(resp, req)
 265                 c.Check(resp.Code, check.Equals, trial.expectStatus)
 266                 c.Check(resp.Body.String(), check.Equals, "")
 267
 268                 c.Log(logbuf.String())
 269                 c.Check(logbuf.String(), check.Matches, trial.logRegexp)
 270         }
 271 }
 272
 273 func (s *UnitSuite) TestInvalidUUID(c *check.C) {
 274         bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-"
 275         token := arvadostest.ActiveToken
 276         for _, trial := range []string{
 277                 "http://keep-web/c=" + bogusID + "/foo",
 278                 "http://keep-web/c=" + bogusID + "/t=" + token + "/foo",
 279                 "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo",
 280                 "http://keep-web/collections/" + bogusID + "/foo",
 281                 "http://" + bogusID + ".keep-web/" + bogusID + "/foo",
 282                 "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
 283         } {
 284                 c.Log(trial)
 285                 u := mustParseURL(trial)
 286                 req := &http.Request{
 287                         Method:     "GET",
 288                         Host:       u.Host,
 289                         URL:        u,
 290                         RequestURI: u.RequestURI(),
 291                 }
 292                 resp := httptest.NewRecorder()
 293                 s.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 294                 s.handler.ServeHTTP(resp, req)
 295                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 296         }
 297 }
 298
 299 func mustParseURL(s string) *url.URL {
 300         r, err := url.Parse(s)
 301         if err != nil {
 302                 panic("parse URL: " + s)
 303         }
 304         return r
 305 }
 306
 307 func (s *IntegrationSuite) TestVhost404(c *check.C) {
 308         for _, testURL := range []string{
 309                 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
 310                 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
 311         } {
 312                 resp := httptest.NewRecorder()
 313                 u := mustParseURL(testURL)
 314                 req := &http.Request{
 315                         Method:     "GET",
 316                         URL:        u,
 317                         RequestURI: u.RequestURI(),
 318                 }
 319                 s.handler.ServeHTTP(resp, req)
 320                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 321                 c.Check(resp.Body.String(), check.Equals, notFoundMessage+"\n")
 322         }
 323 }
 324
 325 // An authorizer modifies an HTTP request to make use of the given
 326 // token -- by adding it to a header, cookie, query param, or whatever
 327 // -- and returns the HTTP status code we should expect from keep-web if
 328 // the token is invalid.
 329 type authorizer func(*http.Request, string) int
 330
 331 func (s *IntegrationSuite) TestVhostViaAuthzHeaderOAuth2(c *check.C) {
 332         s.doVhostRequests(c, authzViaAuthzHeaderOAuth2)
 333 }
 334 func authzViaAuthzHeaderOAuth2(r *http.Request, tok string) int {
 335         r.Header.Add("Authorization", "OAuth2 "+tok)
 336         return http.StatusUnauthorized
 337 }
 338
 339 func (s *IntegrationSuite) TestVhostViaAuthzHeaderBearer(c *check.C) {
 340         s.doVhostRequests(c, authzViaAuthzHeaderBearer)
 341 }
 342 func authzViaAuthzHeaderBearer(r *http.Request, tok string) int {
 343         r.Header.Add("Authorization", "Bearer "+tok)
 344         return http.StatusUnauthorized
 345 }
 346
 347 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
 348         s.doVhostRequests(c, authzViaCookieValue)
 349 }
 350 func authzViaCookieValue(r *http.Request, tok string) int {
 351         r.AddCookie(&http.Cookie{
 352                 Name:  "arvados_api_token",
 353                 Value: auth.EncodeTokenCookie([]byte(tok)),
 354         })
 355         return http.StatusUnauthorized
 356 }
 357
 358 func (s *IntegrationSuite) TestVhostViaHTTPBasicAuth(c *check.C) {
 359         s.doVhostRequests(c, authzViaHTTPBasicAuth)
 360 }
 361 func authzViaHTTPBasicAuth(r *http.Request, tok string) int {
 362         r.AddCookie(&http.Cookie{
 363                 Name:  "arvados_api_token",
 364                 Value: auth.EncodeTokenCookie([]byte(tok)),
 365         })
 366         return http.StatusUnauthorized
 367 }
 368
 369 func (s *IntegrationSuite) TestVhostViaHTTPBasicAuthWithExtraSpaceChars(c *check.C) {
 370         s.doVhostRequests(c, func(r *http.Request, tok string) int {
 371                 r.AddCookie(&http.Cookie{
 372                         Name:  "arvados_api_token",
 373                         Value: auth.EncodeTokenCookie([]byte(" " + tok + "\n")),
 374                 })
 375                 return http.StatusUnauthorized
 376         })
 377 }
 378
 379 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
 380         s.doVhostRequests(c, authzViaPath)
 381 }
 382 func authzViaPath(r *http.Request, tok string) int {
 383         r.URL.Path = "/t=" + tok + r.URL.Path
 384         return http.StatusNotFound
 385 }
 386
 387 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
 388         s.doVhostRequests(c, authzViaQueryString)
 389 }
 390 func authzViaQueryString(r *http.Request, tok string) int {
 391         r.URL.RawQuery = "api_token=" + tok
 392         return http.StatusUnauthorized
 393 }
 394
 395 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
 396         s.doVhostRequests(c, authzViaPOST)
 397 }
 398 func authzViaPOST(r *http.Request, tok string) int {
 399         r.Method = "POST"
 400         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 401         r.Body = ioutil.NopCloser(strings.NewReader(
 402                 url.Values{"api_token": {tok}}.Encode()))
 403         return http.StatusUnauthorized
 404 }
 405
 406 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
 407         s.doVhostRequests(c, authzViaPOST)
 408 }
 409 func authzViaXHRPOST(r *http.Request, tok string) int {
 410         r.Method = "POST"
 411         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 412         r.Header.Add("Origin", "https://origin.example")
 413         r.Body = ioutil.NopCloser(strings.NewReader(
 414                 url.Values{
 415                         "api_token":   {tok},
 416                         "disposition": {"attachment"},
 417                 }.Encode()))
 418         return http.StatusUnauthorized
 419 }
 420
 421 // Try some combinations of {url, token} using the given authorization
 422 // mechanism, and verify the result is correct.
 423 func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
 424         for _, hostPath := range []string{
 425                 arvadostest.FooCollection + ".example.com/foo",
 426                 arvadostest.FooCollection + "--collections.example.com/foo",
 427                 arvadostest.FooCollection + "--collections.example.com/_/foo",
 428                 arvadostest.FooCollectionPDH + ".example.com/foo",
 429                 strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + "--collections.example.com/foo",
 430                 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
 431         } {
 432                 c.Log("doRequests: ", hostPath)
 433                 s.doVhostRequestsWithHostPath(c, authz, hostPath)
 434         }
 435 }
 436
 437 func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
 438         for _, tok := range []string{
 439                 arvadostest.ActiveToken,
 440                 arvadostest.ActiveToken[:15],
 441                 arvadostest.SpectatorToken,
 442                 "bogus",
 443                 "",
 444         } {
 445                 u := mustParseURL("http://" + hostPath)
 446                 req := &http.Request{
 447                         Method:     "GET",
 448                         Host:       u.Host,
 449                         URL:        u,
 450                         RequestURI: u.RequestURI(),
 451                         Header:     http.Header{},
 452                 }
 453                 failCode := authz(req, tok)
 454                 req, resp := s.doReq(req)
 455                 code, body := resp.Code, resp.Body.String()
 456
 457                 // If the initial request had a (non-empty) token
 458                 // showing in the query string, we should have been
 459                 // redirected in order to hide it in a cookie.
 460                 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
 461
 462                 if tok == arvadostest.ActiveToken {
 463                         c.Check(code, check.Equals, http.StatusOK)
 464                         c.Check(body, check.Equals, "foo")
 465                 } else {
 466                         c.Check(code >= 400, check.Equals, true)
 467                         c.Check(code < 500, check.Equals, true)
 468                         if tok == arvadostest.SpectatorToken {
 469                                 // Valid token never offers to retry
 470                                 // with different credentials.
 471                                 c.Check(code, check.Equals, http.StatusNotFound)
 472                         } else {
 473                                 // Invalid token can ask to retry
 474                                 // depending on the authz method.
 475                                 c.Check(code, check.Equals, failCode)
 476                         }
 477                         if code == 404 {
 478                                 c.Check(body, check.Equals, notFoundMessage+"\n")
 479                         } else {
 480                                 c.Check(body, check.Equals, unauthorizedMessage+"\n")
 481                         }
 482                 }
 483         }
 484 }
 485
 486 func (s *IntegrationSuite) TestVhostPortMatch(c *check.C) {
 487         for _, host := range []string{"download.example.com", "DOWNLOAD.EXAMPLE.COM"} {
 488                 for _, port := range []string{"80", "443", "8000"} {
 489                         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = fmt.Sprintf("download.example.com:%v", port)
 490                         u := mustParseURL(fmt.Sprintf("http://%v/by_id/%v/foo", host, arvadostest.FooCollection))
 491                         req := &http.Request{
 492                                 Method:     "GET",
 493                                 Host:       u.Host,
 494                                 URL:        u,
 495                                 RequestURI: u.RequestURI(),
 496                                 Header:     http.Header{"Authorization": []string{"Bearer " + arvadostest.ActiveToken}},
 497                         }
 498                         req, resp := s.doReq(req)
 499                         code, _ := resp.Code, resp.Body.String()
 500
 501                         if port == "8000" {
 502                                 c.Check(code, check.Equals, 401)
 503                         } else {
 504                                 c.Check(code, check.Equals, 200)
 505                         }
 506                 }
 507         }
 508 }
 509
 510 func (s *IntegrationSuite) do(method string, urlstring string, token string, hdr http.Header) (*http.Request, *httptest.ResponseRecorder) {
 511         u := mustParseURL(urlstring)
 512         if hdr == nil && token != "" {
 513                 hdr = http.Header{"Authorization": {"Bearer " + token}}
 514         } else if hdr == nil {
 515                 hdr = http.Header{}
 516         } else if token != "" {
 517                 panic("must not pass both token and hdr")
 518         }
 519         return s.doReq(&http.Request{
 520                 Method:     method,
 521                 Host:       u.Host,
 522                 URL:        u,
 523                 RequestURI: u.RequestURI(),
 524                 Header:     hdr,
 525         })
 526 }
 527
 528 func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
 529         resp := httptest.NewRecorder()
 530         s.handler.ServeHTTP(resp, req)
 531         if resp.Code != http.StatusSeeOther {
 532                 return req, resp
 533         }
 534         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 535         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 536         req = &http.Request{
 537                 Method:     "GET",
 538                 Host:       u.Host,
 539                 URL:        u,
 540                 RequestURI: u.RequestURI(),
 541                 Header:     http.Header{},
 542         }
 543         for _, c := range cookies {
 544                 req.AddCookie(c)
 545         }
 546         return s.doReq(req)
 547 }
 548
 549 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
 550         s.testVhostRedirectTokenToCookie(c, "GET",
 551                 arvadostest.FooCollection+".example.com/foo",
 552                 "?api_token="+arvadostest.ActiveToken,
 553                 nil,
 554                 "",
 555                 http.StatusOK,
 556                 "foo",
 557         )
 558 }
 559
 560 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
 561         s.testVhostRedirectTokenToCookie(c, "GET",
 562                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
 563                 "",
 564                 nil,
 565                 "",
 566                 http.StatusOK,
 567                 "foo",
 568         )
 569 }
 570
 571 func (s *IntegrationSuite) TestCollectionSharingToken(c *check.C) {
 572         s.testVhostRedirectTokenToCookie(c, "GET",
 573                 "example.com/c="+arvadostest.FooFileCollectionUUID+"/t="+arvadostest.FooFileCollectionSharingToken+"/foo",
 574                 "",
 575                 nil,
 576                 "",
 577                 http.StatusOK,
 578                 "foo",
 579         )
 580         // Same valid sharing token, but requesting a different collection
 581         s.testVhostRedirectTokenToCookie(c, "GET",
 582                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.FooFileCollectionSharingToken+"/foo",
 583                 "",
 584                 nil,
 585                 "",
 586                 http.StatusNotFound,
 587                 regexp.QuoteMeta(notFoundMessage+"\n"),
 588         )
 589 }
 590
 591 // Bad token in URL is 404 Not Found because it doesn't make sense to
 592 // retry the same URL with different authorization.
 593 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
 594         s.testVhostRedirectTokenToCookie(c, "GET",
 595                 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
 596                 "",
 597                 nil,
 598                 "",
 599                 http.StatusNotFound,
 600                 regexp.QuoteMeta(notFoundMessage+"\n"),
 601         )
 602 }
 603
 604 // Bad token in a cookie (even if it got there via our own
 605 // query-string-to-cookie redirect) is, in principle, retryable via
 606 // wb2-login-and-redirect flow.
 607 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
 608         // Inline
 609         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 610                 arvadostest.FooCollection+".example.com/foo",
 611                 "?api_token=thisisabogustoken",
 612                 http.Header{"Sec-Fetch-Mode": {"navigate"}},
 613                 "",
 614                 http.StatusSeeOther,
 615                 "",
 616         )
 617         u, err := url.Parse(resp.Header().Get("Location"))
 618         c.Assert(err, check.IsNil)
 619         c.Logf("redirected to %s", u)
 620         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 621         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 622         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
 623
 624         // Download/attachment indicated by ?disposition=attachment
 625         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 626                 arvadostest.FooCollection+".example.com/foo",
 627                 "?api_token=thisisabogustoken&disposition=attachment",
 628                 http.Header{"Sec-Fetch-Mode": {"navigate"}},
 629                 "",
 630                 http.StatusSeeOther,
 631                 "",
 632         )
 633         u, err = url.Parse(resp.Header().Get("Location"))
 634         c.Assert(err, check.IsNil)
 635         c.Logf("redirected to %s", u)
 636         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 637         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "")
 638         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 639
 640         // Download/attachment indicated by vhost
 641         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 642                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
 643                 "?api_token=thisisabogustoken",
 644                 http.Header{"Sec-Fetch-Mode": {"navigate"}},
 645                 "",
 646                 http.StatusSeeOther,
 647                 "",
 648         )
 649         u, err = url.Parse(resp.Header().Get("Location"))
 650         c.Assert(err, check.IsNil)
 651         c.Logf("redirected to %s", u)
 652         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 653         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "")
 654         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 655
 656         // Without "Sec-Fetch-Mode: navigate" header, just 401.
 657         s.testVhostRedirectTokenToCookie(c, "GET",
 658                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
 659                 "?api_token=thisisabogustoken",
 660                 http.Header{"Sec-Fetch-Mode": {"cors"}},
 661                 "",
 662                 http.StatusUnauthorized,
 663                 regexp.QuoteMeta(unauthorizedMessage+"\n"),
 664         )
 665         s.testVhostRedirectTokenToCookie(c, "GET",
 666                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
 667                 "?api_token=thisisabogustoken",
 668                 nil,
 669                 "",
 670                 http.StatusUnauthorized,
 671                 regexp.QuoteMeta(unauthorizedMessage+"\n"),
 672         )
 673 }
 674
 675 func (s *IntegrationSuite) TestVhostRedirectWithNoCache(c *check.C) {
 676         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 677                 arvadostest.FooCollection+".example.com/foo",
 678                 "?api_token=thisisabogustoken",
 679                 http.Header{
 680                         "Sec-Fetch-Mode": {"navigate"},
 681                         "Cache-Control":  {"no-cache"},
 682                 },
 683                 "",
 684                 http.StatusSeeOther,
 685                 "",
 686         )
 687         u, err := url.Parse(resp.Header().Get("Location"))
 688         c.Assert(err, check.IsNil)
 689         c.Logf("redirected to %s", u)
 690         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 691         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 692         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
 693 }
 694
 695 func (s *IntegrationSuite) TestNoTokenWorkbench2LoginFlow(c *check.C) {
 696         for _, trial := range []struct {
 697                 anonToken    bool
 698                 cacheControl string
 699         }{
 700                 {},
 701                 {cacheControl: "no-cache"},
 702                 {anonToken: true},
 703                 {anonToken: true, cacheControl: "no-cache"},
 704         } {
 705                 c.Logf("trial: %+v", trial)
 706
 707                 if trial.anonToken {
 708                         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 709                 } else {
 710                         s.handler.Cluster.Users.AnonymousUserToken = ""
 711                 }
 712                 req, err := http.NewRequest("GET", "http://"+arvadostest.FooCollection+".example.com/foo", nil)
 713                 c.Assert(err, check.IsNil)
 714                 req.Header.Set("Sec-Fetch-Mode", "navigate")
 715                 if trial.cacheControl != "" {
 716                         req.Header.Set("Cache-Control", trial.cacheControl)
 717                 }
 718                 resp := httptest.NewRecorder()
 719                 s.handler.ServeHTTP(resp, req)
 720                 c.Check(resp.Code, check.Equals, http.StatusSeeOther)
 721                 u, err := url.Parse(resp.Header().Get("Location"))
 722                 c.Assert(err, check.IsNil)
 723                 c.Logf("redirected to %q", u)
 724                 c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 725                 c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 726                 c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
 727         }
 728 }
 729
 730 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
 731         s.testVhostRedirectTokenToCookie(c, "GET",
 732                 "example.com/c="+arvadostest.FooCollection+"/foo",
 733                 "?api_token="+arvadostest.ActiveToken,
 734                 nil,
 735                 "",
 736                 http.StatusBadRequest,
 737                 regexp.QuoteMeta("cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n"),
 738         )
 739 }
 740
 741 // If client requests an attachment by putting ?disposition=attachment
 742 // in the query string, and gets redirected, the redirect target
 743 // should respond with an attachment.
 744 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
 745         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 746                 arvadostest.FooCollection+".example.com/foo",
 747                 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
 748                 nil,
 749                 "",
 750                 http.StatusOK,
 751                 "foo",
 752         )
 753         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 754 }
 755
 756 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
 757         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 758         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 759                 "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
 760                 "?api_token="+arvadostest.ActiveToken,
 761                 nil,
 762                 "",
 763                 http.StatusOK,
 764                 "foo",
 765         )
 766         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 767 }
 768
 769 func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
 770         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 771         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 772                 "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
 773                 "?api_token="+arvadostest.ActiveToken,
 774                 nil,
 775                 "",
 776                 http.StatusOK,
 777                 "waz",
 778         )
 779         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 780         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 781                 "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz",
 782                 "?api_token="+arvadostest.ActiveToken,
 783                 nil,
 784                 "",
 785                 http.StatusOK,
 786                 "waz",
 787         )
 788         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 789 }
 790
 791 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
 792         s.handler.Cluster.Collections.TrustAllContent = true
 793         s.testVhostRedirectTokenToCookie(c, "GET",
 794                 "example.com/c="+arvadostest.FooCollection+"/foo",
 795                 "?api_token="+arvadostest.ActiveToken,
 796                 nil,
 797                 "",
 798                 http.StatusOK,
 799                 "foo",
 800         )
 801 }
 802
 803 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
 804         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
 805
 806         s.testVhostRedirectTokenToCookie(c, "GET",
 807                 "example.com/c="+arvadostest.FooCollection+"/foo",
 808                 "?api_token="+arvadostest.ActiveToken,
 809                 nil,
 810                 "",
 811                 http.StatusBadRequest,
 812                 regexp.QuoteMeta("cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n"),
 813         )
 814
 815         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 816                 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
 817                 "?api_token="+arvadostest.ActiveToken,
 818                 nil,
 819                 "",
 820                 http.StatusOK,
 821                 "foo",
 822         )
 823         c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
 824 }
 825
 826 func (s *IntegrationSuite) TestVhostRedirectMultipleTokens(c *check.C) {
 827         baseUrl := arvadostest.FooCollection + ".example.com/foo"
 828         query := url.Values{}
 829
 830         // The intent of these tests is to check that requests are redirected
 831         // correctly in the presence of multiple API tokens. The exact response
 832         // codes and content are not closely considered: they're just how
 833         // keep-web responded when we made the smallest possible fix. Changing
 834         // those responses may be okay, but you should still test all these
 835         // different cases and the associated redirect logic.
 836         query["api_token"] = []string{arvadostest.ActiveToken, arvadostest.AnonymousToken}
 837         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
 838         query["api_token"] = []string{arvadostest.ActiveToken, arvadostest.AnonymousToken, ""}
 839         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
 840         query["api_token"] = []string{arvadostest.ActiveToken, "", arvadostest.AnonymousToken}
 841         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
 842         query["api_token"] = []string{"", arvadostest.ActiveToken}
 843         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
 844
 845         expectContent := regexp.QuoteMeta(unauthorizedMessage + "\n")
 846         query["api_token"] = []string{arvadostest.AnonymousToken, "invalidtoo"}
 847         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusUnauthorized, expectContent)
 848         query["api_token"] = []string{arvadostest.AnonymousToken, ""}
 849         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusUnauthorized, expectContent)
 850         query["api_token"] = []string{"", arvadostest.AnonymousToken}
 851         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusUnauthorized, expectContent)
 852 }
 853
 854 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
 855         s.testVhostRedirectTokenToCookie(c, "POST",
 856                 arvadostest.FooCollection+".example.com/foo",
 857                 "",
 858                 http.Header{"Content-Type": {"application/x-www-form-urlencoded"}},
 859                 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
 860                 http.StatusOK,
 861                 "foo",
 862         )
 863 }
 864
 865 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
 866         s.testVhostRedirectTokenToCookie(c, "POST",
 867                 arvadostest.FooCollection+".example.com/foo",
 868                 "",
 869                 http.Header{"Content-Type": {"application/x-www-form-urlencoded"}},
 870                 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
 871                 http.StatusNotFound,
 872                 regexp.QuoteMeta(notFoundMessage+"\n"),
 873         )
 874 }
 875
 876 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
 877         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 878         s.testVhostRedirectTokenToCookie(c, "GET",
 879                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 880                 "",
 881                 nil,
 882                 "",
 883                 http.StatusOK,
 884                 "Hello world\n",
 885         )
 886 }
 887
 888 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
 889         s.handler.Cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
 890         s.testVhostRedirectTokenToCookie(c, "GET",
 891                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 892                 "",
 893                 nil,
 894                 "",
 895                 http.StatusUnauthorized,
 896                 "Authorization tokens are not accepted here: .*\n",
 897         )
 898 }
 899
 900 func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
 901         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 902
 903         client := arvados.NewClientFromEnv()
 904         client.AuthToken = arvadostest.ActiveToken
 905         fs, err := (&arvados.Collection{}).FileSystem(client, nil)
 906         c.Assert(err, check.IsNil)
 907         f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777)
 908         c.Assert(err, check.IsNil)
 909         f.Close()
 910         mtxt, err := fs.MarshalManifest(".")
 911         c.Assert(err, check.IsNil)
 912         var coll arvados.Collection
 913         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 914                 "collection": map[string]string{
 915                         "manifest_text": mtxt,
 916                 },
 917         })
 918         c.Assert(err, check.IsNil)
 919
 920         u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
 921         req := &http.Request{
 922                 Method:     "GET",
 923                 Host:       u.Host,
 924                 URL:        u,
 925                 RequestURI: u.RequestURI(),
 926                 Header: http.Header{
 927                         "Authorization": {"Bearer " + client.AuthToken},
 928                 },
 929         }
 930         resp := httptest.NewRecorder()
 931         s.handler.ServeHTTP(resp, req)
 932         c.Check(resp.Code, check.Equals, http.StatusOK)
 933         c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\&#34;odd&#39; path chars.*`)
 934 }
 935
 936 func (s *IntegrationSuite) TestForwardSlashSubstitution(c *check.C) {
 937         arv := arvados.NewClientFromEnv()
 938         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 939         s.handler.Cluster.Collections.ForwardSlashNameSubstitution = "{SOLIDUS}"
 940         name := "foo/bar/baz"
 941         nameShown := strings.Replace(name, "/", "{SOLIDUS}", -1)
 942         nameShownEscaped := strings.Replace(name, "/", "%7bSOLIDUS%7d", -1)
 943
 944         client := arvados.NewClientFromEnv()
 945         client.AuthToken = arvadostest.ActiveToken
 946         fs, err := (&arvados.Collection{}).FileSystem(client, nil)
 947         c.Assert(err, check.IsNil)
 948         f, err := fs.OpenFile("filename", os.O_CREATE, 0777)
 949         c.Assert(err, check.IsNil)
 950         f.Close()
 951         mtxt, err := fs.MarshalManifest(".")
 952         c.Assert(err, check.IsNil)
 953         var coll arvados.Collection
 954         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 955                 "collection": map[string]string{
 956                         "manifest_text": mtxt,
 957                         "name":          name,
 958                         "owner_uuid":    arvadostest.AProjectUUID,
 959                 },
 960         })
 961         c.Assert(err, check.IsNil)
 962         defer arv.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
 963
 964         base := "http://download.example.com/by_id/" + coll.OwnerUUID + "/"
 965         for tryURL, expectRegexp := range map[string]string{
 966                 base:                          `(?ms).*href="./` + nameShownEscaped + `/"\S+` + nameShown + `.*`,
 967                 base + nameShownEscaped + "/": `(?ms).*href="./filename"\S+filename.*`,
 968         } {
 969                 u, _ := url.Parse(tryURL)
 970                 req := &http.Request{
 971                         Method:     "GET",
 972                         Host:       u.Host,
 973                         URL:        u,
 974                         RequestURI: u.RequestURI(),
 975                         Header: http.Header{
 976                                 "Authorization": {"Bearer " + client.AuthToken},
 977                         },
 978                 }
 979                 resp := httptest.NewRecorder()
 980                 s.handler.ServeHTTP(resp, req)
 981                 c.Check(resp.Code, check.Equals, http.StatusOK)
 982                 c.Check(resp.Body.String(), check.Matches, expectRegexp)
 983         }
 984 }
 985
 986 // XHRs can't follow redirect-with-cookie so they rely on method=POST
 987 // and disposition=attachment (telling us it's acceptable to respond
 988 // with content instead of a redirect) and an Origin header that gets
 989 // added automatically by the browser (telling us it's desirable to do
 990 // so).
 991 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
 992         u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
 993         req := &http.Request{
 994                 Method:     "POST",
 995                 Host:       u.Host,
 996                 URL:        u,
 997                 RequestURI: u.RequestURI(),
 998                 Header: http.Header{
 999                         "Origin":       {"https://origin.example"},
1000                         "Content-Type": {"application/x-www-form-urlencoded"},
1001                 },
1002                 Body: ioutil.NopCloser(strings.NewReader(url.Values{
1003                         "api_token":   {arvadostest.ActiveToken},
1004                         "disposition": {"attachment"},
1005                 }.Encode())),
1006         }
1007         resp := httptest.NewRecorder()
1008         s.handler.ServeHTTP(resp, req)
1009         c.Check(resp.Code, check.Equals, http.StatusOK)
1010         c.Check(resp.Body.String(), check.Equals, "foo")
1011         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
1012
1013         // GET + Origin header is representative of both AJAX GET
1014         // requests and inline images via <IMG crossorigin="anonymous"
1015         // src="...">.
1016         u.RawQuery = "api_token=" + url.QueryEscape(arvadostest.ActiveTokenV2)
1017         req = &http.Request{
1018                 Method:     "GET",
1019                 Host:       u.Host,
1020                 URL:        u,
1021                 RequestURI: u.RequestURI(),
1022                 Header: http.Header{
1023                         "Origin": {"https://origin.example"},
1024                 },
1025         }
1026         resp = httptest.NewRecorder()
1027         s.handler.ServeHTTP(resp, req)
1028         c.Check(resp.Code, check.Equals, http.StatusOK)
1029         c.Check(resp.Body.String(), check.Equals, "foo")
1030         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
1031 }
1032
1033 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString string, reqHeader http.Header, reqBody string, expectStatus int, matchRespBody string) *httptest.ResponseRecorder {
1034         if reqHeader == nil {
1035                 reqHeader = http.Header{}
1036         }
1037         u, _ := url.Parse(`http://` + hostPath + queryString)
1038         c.Logf("requesting %s", u)
1039         req := &http.Request{
1040                 Method:     method,
1041                 Host:       u.Host,
1042                 URL:        u,
1043                 RequestURI: u.RequestURI(),
1044                 Header:     reqHeader,
1045                 Body:       ioutil.NopCloser(strings.NewReader(reqBody)),
1046         }
1047
1048         resp := httptest.NewRecorder()
1049         defer func() {
1050                 c.Check(resp.Code, check.Equals, expectStatus)
1051                 c.Check(resp.Body.String(), check.Matches, matchRespBody)
1052         }()
1053
1054         s.handler.ServeHTTP(resp, req)
1055         if resp.Code != http.StatusSeeOther {
1056                 attachment, _ := regexp.MatchString(`^attachment(;|$)`, resp.Header().Get("Content-Disposition"))
1057                 // Since we're not redirecting, check that any api_token in the URL is
1058                 // handled safely.
1059                 // If there is no token in the URL, then we're good.
1060                 // Otherwise, if the response code is an error, the body is expected to
1061                 // be static content, and nothing that might maliciously introspect the
1062                 // URL. It's considered safe and allowed.
1063                 // Otherwise, if the response content has attachment disposition,
1064                 // that's considered safe for all the reasons explained in the
1065                 // safeAttachment comment in handler.go.
1066                 c.Check(!u.Query().Has("api_token") || resp.Code >= 400 || attachment, check.Equals, true)
1067                 return resp
1068         }
1069
1070         loc, err := url.Parse(resp.Header().Get("Location"))
1071         c.Assert(err, check.IsNil)
1072         c.Check(loc.Scheme, check.Equals, u.Scheme)
1073         c.Check(loc.Host, check.Equals, u.Host)
1074         c.Check(loc.RawPath, check.Equals, u.RawPath)
1075         // If the response was a redirect, it should never include an API token.
1076         c.Check(loc.Query().Has("api_token"), check.Equals, false)
1077         c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
1078         cookies := (&http.Response{Header: resp.Header()}).Cookies()
1079
1080         c.Logf("following redirect to %s", u)
1081         req = &http.Request{
1082                 Method:     "GET",
1083                 Host:       loc.Host,
1084                 URL:        loc,
1085                 RequestURI: loc.RequestURI(),
1086                 Header:     reqHeader,
1087         }
1088         for _, c := range cookies {
1089                 req.AddCookie(c)
1090         }
1091
1092         resp = httptest.NewRecorder()
1093         s.handler.ServeHTTP(resp, req)
1094
1095         if resp.Code != http.StatusSeeOther {
1096                 c.Check(resp.Header().Get("Location"), check.Equals, "")
1097         }
1098         return resp
1099 }
1100
1101 func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) {
1102         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
1103         s.testDirectoryListing(c)
1104 }
1105
1106 func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) {
1107         s.handler.Cluster.Users.AnonymousUserToken = ""
1108         s.testDirectoryListing(c)
1109 }
1110
1111 func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
1112         // The "ownership cycle" test fixtures are reachable from the
1113         // "filter group without filters" group, causing webdav's
1114         // walkfs to recurse indefinitely. Avoid that by deleting one
1115         // of the bogus fixtures.
1116         arv := arvados.NewClientFromEnv()
1117         err := arv.RequestAndDecode(nil, "DELETE", "arvados/v1/groups/zzzzz-j7d0g-cx2al9cqkmsf1hs", nil, nil)
1118         if err != nil {
1119                 c.Assert(err, check.FitsTypeOf, &arvados.TransactionError{})
1120                 c.Check(err.(*arvados.TransactionError).StatusCode, check.Equals, 404)
1121         }
1122
1123         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1124         authHeader := http.Header{
1125                 "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
1126         }
1127         for _, trial := range []struct {
1128                 uri      string
1129                 header   http.Header
1130                 expect   []string
1131                 redirect string
1132                 cutDirs  int
1133         }{
1134                 {
1135                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
1136                         header:  authHeader,
1137                         expect:  []string{"dir1/foo", "dir1/bar"},
1138                         cutDirs: 0,
1139                 },
1140                 {
1141                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
1142                         header:  authHeader,
1143                         expect:  []string{"foo", "bar"},
1144                         cutDirs: 1,
1145                 },
1146                 {
1147                         // URLs of this form ignore authHeader, and
1148                         // FooAndBarFilesInDirUUID isn't public, so
1149                         // this returns 401.
1150                         uri:    "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
1151                         header: authHeader,
1152                         expect: nil,
1153                 },
1154                 {
1155                         uri:     "download.example.com/users/active/foo_file_in_dir/",
1156                         header:  authHeader,
1157                         expect:  []string{"dir1/"},
1158                         cutDirs: 3,
1159                 },
1160                 {
1161                         uri:     "download.example.com/users/active/foo_file_in_dir/dir1/",
1162                         header:  authHeader,
1163                         expect:  []string{"bar"},
1164                         cutDirs: 4,
1165                 },
1166                 {
1167                         uri:     "download.example.com/",
1168                         header:  authHeader,
1169                         expect:  []string{"users/"},
1170                         cutDirs: 0,
1171                 },
1172                 {
1173                         uri:      "download.example.com/users",
1174                         header:   authHeader,
1175                         redirect: "/users/",
1176                         expect:   []string{"active/"},
1177                         cutDirs:  1,
1178                 },
1179                 {
1180                         uri:     "download.example.com/users/",
1181                         header:  authHeader,
1182                         expect:  []string{"active/"},
1183                         cutDirs: 1,
1184                 },
1185                 {
1186                         uri:      "download.example.com/users/active",
1187                         header:   authHeader,
1188                         redirect: "/users/active/",
1189                         expect:   []string{"foo_file_in_dir/"},
1190                         cutDirs:  2,
1191                 },
1192                 {
1193                         uri:     "download.example.com/users/active/",
1194                         header:  authHeader,
1195                         expect:  []string{"foo_file_in_dir/"},
1196                         cutDirs: 2,
1197                 },
1198                 {
1199                         uri:     "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
1200                         header:  nil,
1201                         expect:  []string{"dir1/foo", "dir1/bar"},
1202                         cutDirs: 4,
1203                 },
1204                 {
1205                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
1206                         header:  nil,
1207                         expect:  []string{"dir1/foo", "dir1/bar"},
1208                         cutDirs: 2,
1209                 },
1210                 {
1211                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken,
1212                         header:  nil,
1213                         expect:  []string{"dir1/foo", "dir1/bar"},
1214                         cutDirs: 2,
1215                 },
1216                 {
1217                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID,
1218                         header:  authHeader,
1219                         expect:  []string{"dir1/foo", "dir1/bar"},
1220                         cutDirs: 1,
1221                 },
1222                 {
1223                         uri:      "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
1224                         header:   authHeader,
1225                         redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
1226                         expect:   []string{"foo", "bar"},
1227                         cutDirs:  2,
1228                 },
1229                 {
1230                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
1231                         header:  authHeader,
1232                         expect:  []string{"foo", "bar"},
1233                         cutDirs: 3,
1234                 },
1235                 {
1236                         uri:      arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
1237                         header:   authHeader,
1238                         redirect: "/dir1/",
1239                         expect:   []string{"foo", "bar"},
1240                         cutDirs:  1,
1241                 },
1242                 {
1243                         uri:    "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
1244                         header: authHeader,
1245                         expect: nil,
1246                 },
1247                 {
1248                         uri:     "download.example.com/c=" + arvadostest.WazVersion1Collection,
1249                         header:  authHeader,
1250                         expect:  []string{"waz"},
1251                         cutDirs: 1,
1252                 },
1253                 {
1254                         uri:     "download.example.com/by_id/" + arvadostest.WazVersion1Collection,
1255                         header:  authHeader,
1256                         expect:  []string{"waz"},
1257                         cutDirs: 2,
1258                 },
1259                 {
1260                         uri:     "download.example.com/users/active/This filter group/",
1261                         header:  authHeader,
1262                         expect:  []string{"A Subproject/"},
1263                         cutDirs: 3,
1264                 },
1265                 {
1266                         uri:     "download.example.com/users/active/This filter group/A Subproject",
1267                         header:  authHeader,
1268                         expect:  []string{"baz_file/"},
1269                         cutDirs: 4,
1270                 },
1271                 {
1272                         uri:     "download.example.com/by_id/" + arvadostest.AFilterGroupUUID,
1273                         header:  authHeader,
1274                         expect:  []string{"A Subproject/"},
1275                         cutDirs: 2,
1276                 },
1277                 {
1278                         uri:     "download.example.com/by_id/" + arvadostest.AFilterGroupUUID + "/A Subproject",
1279                         header:  authHeader,
1280                         expect:  []string{"baz_file/"},
1281                         cutDirs: 3,
1282                 },
1283         } {
1284                 comment := check.Commentf("HTML: %q redir %q => %q", trial.uri, trial.redirect, trial.expect)
1285                 resp := httptest.NewRecorder()
1286                 u := mustParseURL("//" + trial.uri)
1287                 req := &http.Request{
1288                         Method:     "GET",
1289                         Host:       u.Host,
1290                         URL:        u,
1291                         RequestURI: u.RequestURI(),
1292                         Header:     copyHeader(trial.header),
1293                 }
1294                 s.handler.ServeHTTP(resp, req)
1295                 var cookies []*http.Cookie
1296                 for resp.Code == http.StatusSeeOther {
1297                         u, _ := req.URL.Parse(resp.Header().Get("Location"))
1298                         req = &http.Request{
1299                                 Method:     "GET",
1300                                 Host:       u.Host,
1301                                 URL:        u,
1302                                 RequestURI: u.RequestURI(),
1303                                 Header:     copyHeader(trial.header),
1304                         }
1305                         cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
1306                         for _, c := range cookies {
1307                                 req.AddCookie(c)
1308                         }
1309                         resp = httptest.NewRecorder()
1310                         s.handler.ServeHTTP(resp, req)
1311                 }
1312                 if trial.redirect != "" {
1313                         c.Check(req.URL.Path, check.Equals, trial.redirect, comment)
1314                 }
1315                 if trial.expect == nil {
1316                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1317                 } else {
1318                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
1319                         for _, e := range trial.expect {
1320                                 e = strings.Replace(e, " ", "%20", -1)
1321                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`, comment)
1322                         }
1323                         c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`, comment)
1324                 }
1325
1326                 comment = check.Commentf("WebDAV: %q => %q", trial.uri, trial.expect)
1327                 req = &http.Request{
1328                         Method:     "OPTIONS",
1329                         Host:       u.Host,
1330                         URL:        u,
1331                         RequestURI: u.RequestURI(),
1332                         Header:     copyHeader(trial.header),
1333                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
1334                 }
1335                 resp = httptest.NewRecorder()
1336                 s.handler.ServeHTTP(resp, req)
1337                 if trial.expect == nil {
1338                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1339                 } else {
1340                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
1341                 }
1342
1343                 req = &http.Request{
1344                         Method:     "PROPFIND",
1345                         Host:       u.Host,
1346                         URL:        u,
1347                         RequestURI: u.RequestURI(),
1348                         Header:     copyHeader(trial.header),
1349                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
1350                 }
1351                 resp = httptest.NewRecorder()
1352                 s.handler.ServeHTTP(resp, req)
1353                 // This check avoids logging a big XML document in the
1354                 // event webdav throws a 500 error after sending
1355                 // headers for a 207.
1356                 if !c.Check(strings.HasSuffix(resp.Body.String(), "Internal Server Error"), check.Equals, false) {
1357                         continue
1358                 }
1359                 if trial.expect == nil {
1360                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1361                 } else {
1362                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment)
1363                         for _, e := range trial.expect {
1364                                 if strings.HasSuffix(e, "/") {
1365                                         e = filepath.Join(u.Path, e) + "/"
1366                                 } else {
1367                                         e = filepath.Join(u.Path, e)
1368                                 }
1369                                 e = strings.Replace(e, " ", "%20", -1)
1370                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+e+`</D:href>.*`, comment)
1371                         }
1372                 }
1373         }
1374 }
1375
1376 func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
1377         arv := arvados.NewClientFromEnv()
1378         var newCollection arvados.Collection
1379         err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1380                 "collection": map[string]string{
1381                         "owner_uuid":    arvadostest.ActiveUserUUID,
1382                         "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
1383                         "name":          "keep-web test collection",
1384                 },
1385                 "ensure_unique_name": true,
1386         })
1387         c.Assert(err, check.IsNil)
1388         defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
1389
1390         var updated arvados.Collection
1391         for _, fnm := range []string{"foo.txt", "bar.txt"} {
1392                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
1393                 u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
1394                 req := &http.Request{
1395                         Method:     "DELETE",
1396                         Host:       u.Host,
1397                         URL:        u,
1398                         RequestURI: u.RequestURI(),
1399                         Header: http.Header{
1400                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
1401                         },
1402                 }
1403                 resp := httptest.NewRecorder()
1404                 s.handler.ServeHTTP(resp, req)
1405                 c.Check(resp.Code, check.Equals, http.StatusNoContent)
1406
1407                 updated = arvados.Collection{}
1408                 err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
1409                 c.Check(err, check.IsNil)
1410                 c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`)
1411                 c.Logf("updated manifest_text %q", updated.ManifestText)
1412         }
1413         c.Check(updated.ManifestText, check.Equals, "")
1414 }
1415
1416 func (s *IntegrationSuite) TestFileContentType(c *check.C) {
1417         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1418
1419         client := arvados.NewClientFromEnv()
1420         client.AuthToken = arvadostest.ActiveToken
1421         arv, err := arvadosclient.New(client)
1422         c.Assert(err, check.Equals, nil)
1423         kc, err := keepclient.MakeKeepClient(arv)
1424         c.Assert(err, check.Equals, nil)
1425
1426         fs, err := (&arvados.Collection{}).FileSystem(client, kc)
1427         c.Assert(err, check.IsNil)
1428
1429         trials := []struct {
1430                 filename    string
1431                 content     string
1432                 contentType string
1433         }{
1434                 {"picture.txt", "BMX bikes are small this year\n", "text/plain; charset=utf-8"},
1435                 {"picture.bmp", "BMX bikes are small this year\n", "image/(x-ms-)?bmp"},
1436                 {"picture.jpg", "BMX bikes are small this year\n", "image/jpeg"},
1437                 {"picture1", "BMX bikes are small this year\n", "image/bmp"},            // content sniff; "BM" is the magic signature for .bmp
1438                 {"picture2", "Cars are small this year\n", "text/plain; charset=utf-8"}, // content sniff
1439         }
1440         for _, trial := range trials {
1441                 f, err := fs.OpenFile(trial.filename, os.O_CREATE|os.O_WRONLY, 0777)
1442                 c.Assert(err, check.IsNil)
1443                 _, err = f.Write([]byte(trial.content))
1444                 c.Assert(err, check.IsNil)
1445                 c.Assert(f.Close(), check.IsNil)
1446         }
1447         mtxt, err := fs.MarshalManifest(".")
1448         c.Assert(err, check.IsNil)
1449         var coll arvados.Collection
1450         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1451                 "collection": map[string]string{
1452                         "manifest_text": mtxt,
1453                 },
1454         })
1455         c.Assert(err, check.IsNil)
1456
1457         for _, trial := range trials {
1458                 u, _ := url.Parse("http://download.example.com/by_id/" + coll.UUID + "/" + trial.filename)
1459                 req := &http.Request{
1460                         Method:     "GET",
1461                         Host:       u.Host,
1462                         URL:        u,
1463                         RequestURI: u.RequestURI(),
1464                         Header: http.Header{
1465                                 "Authorization": {"Bearer " + client.AuthToken},
1466                         },
1467                 }
1468                 resp := httptest.NewRecorder()
1469                 s.handler.ServeHTTP(resp, req)
1470                 c.Check(resp.Code, check.Equals, http.StatusOK)
1471                 c.Check(resp.Header().Get("Content-Type"), check.Matches, trial.contentType)
1472                 c.Check(resp.Body.String(), check.Equals, trial.content)
1473         }
1474 }
1475
1476 func (s *IntegrationSuite) TestCacheSize(c *check.C) {
1477         req, err := http.NewRequest("GET", "http://"+arvadostest.FooCollection+".example.com/foo", nil)
1478         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
1479         c.Assert(err, check.IsNil)
1480         resp := httptest.NewRecorder()
1481         s.handler.ServeHTTP(resp, req)
1482         c.Assert(resp.Code, check.Equals, http.StatusOK)
1483         c.Check(s.handler.Cache.sessions[arvadostest.ActiveTokenV2].client.DiskCacheSize.Percent(), check.Equals, int64(10))
1484 }
1485
1486 // Writing to a collection shouldn't affect its entry in the
1487 // PDH-to-manifest cache.
1488 func (s *IntegrationSuite) TestCacheWriteCollectionSamePDH(c *check.C) {
1489         arv, err := arvadosclient.MakeArvadosClient()
1490         c.Assert(err, check.Equals, nil)
1491         arv.ApiToken = arvadostest.ActiveToken
1492
1493         u := mustParseURL("http://x.example/testfile")
1494         req := &http.Request{
1495                 Method:     "GET",
1496                 Host:       u.Host,
1497                 URL:        u,
1498                 RequestURI: u.RequestURI(),
1499                 Header:     http.Header{"Authorization": {"Bearer " + arv.ApiToken}},
1500         }
1501
1502         checkWithID := func(id string, status int) {
1503                 req.URL.Host = strings.Replace(id, "+", "-", -1) + ".example"
1504                 req.Host = req.URL.Host
1505                 resp := httptest.NewRecorder()
1506                 s.handler.ServeHTTP(resp, req)
1507                 c.Check(resp.Code, check.Equals, status)
1508         }
1509
1510         var colls [2]arvados.Collection
1511         for i := range colls {
1512                 err := arv.Create("collections",
1513                         map[string]interface{}{
1514                                 "ensure_unique_name": true,
1515                                 "collection": map[string]interface{}{
1516                                         "name": "test collection",
1517                                 },
1518                         }, &colls[i])
1519                 c.Assert(err, check.Equals, nil)
1520         }
1521
1522         // Populate cache with empty collection
1523         checkWithID(colls[0].PortableDataHash, http.StatusNotFound)
1524
1525         // write a file to colls[0]
1526         reqPut := *req
1527         reqPut.Method = "PUT"
1528         reqPut.URL.Host = colls[0].UUID + ".example"
1529         reqPut.Host = req.URL.Host
1530         reqPut.Body = ioutil.NopCloser(bytes.NewBufferString("testdata"))
1531         resp := httptest.NewRecorder()
1532         s.handler.ServeHTTP(resp, &reqPut)
1533         c.Check(resp.Code, check.Equals, http.StatusCreated)
1534
1535         // new file should not appear in colls[1]
1536         checkWithID(colls[1].PortableDataHash, http.StatusNotFound)
1537         checkWithID(colls[1].UUID, http.StatusNotFound)
1538
1539         checkWithID(colls[0].UUID, http.StatusOK)
1540 }
1541
1542 func copyHeader(h http.Header) http.Header {
1543         hc := http.Header{}
1544         for k, v := range h {
1545                 hc[k] = append([]string(nil), v...)
1546         }
1547         return hc
1548 }
1549
1550 func (s *IntegrationSuite) checkUploadDownloadRequest(c *check.C, req *http.Request,
1551         successCode int, direction string, perm bool, userUuid, collectionUuid, collectionPDH, filepath string) {
1552
1553         client := arvados.NewClientFromEnv()
1554         client.AuthToken = arvadostest.AdminToken
1555         var logentries arvados.LogList
1556         limit1 := 1
1557         err := client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil,
1558                 arvados.ResourceListParams{
1559                         Limit: &limit1,
1560                         Order: "created_at desc"})
1561         c.Check(err, check.IsNil)
1562         c.Check(logentries.Items, check.HasLen, 1)
1563         lastLogId := logentries.Items[0].ID
1564         c.Logf("lastLogId: %d", lastLogId)
1565
1566         var logbuf bytes.Buffer
1567         logger := logrus.New()
1568         logger.Out = &logbuf
1569         resp := httptest.NewRecorder()
1570         req = req.WithContext(ctxlog.Context(context.Background(), logger))
1571         s.handler.ServeHTTP(resp, req)
1572
1573         if perm {
1574                 c.Check(resp.Result().StatusCode, check.Equals, successCode)
1575                 c.Check(logbuf.String(), check.Matches, `(?ms).*msg="File `+direction+`".*`)
1576                 c.Check(logbuf.String(), check.Not(check.Matches), `(?ms).*level=error.*`)
1577
1578                 deadline := time.Now().Add(time.Second)
1579                 for {
1580                         c.Assert(time.Now().After(deadline), check.Equals, false, check.Commentf("timed out waiting for log entry"))
1581                         logentries = arvados.LogList{}
1582                         err = client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil,
1583                                 arvados.ResourceListParams{
1584                                         Filters: []arvados.Filter{
1585                                                 {Attr: "event_type", Operator: "=", Operand: "file_" + direction},
1586                                                 {Attr: "object_uuid", Operator: "=", Operand: userUuid},
1587                                         },
1588                                         Limit: &limit1,
1589                                         Order: "created_at desc",
1590                                 })
1591                         c.Assert(err, check.IsNil)
1592                         if len(logentries.Items) > 0 &&
1593                                 logentries.Items[0].ID > lastLogId &&
1594                                 logentries.Items[0].ObjectUUID == userUuid &&
1595                                 logentries.Items[0].Properties["collection_uuid"] == collectionUuid &&
1596                                 (collectionPDH == "" || logentries.Items[0].Properties["portable_data_hash"] == collectionPDH) &&
1597                                 logentries.Items[0].Properties["collection_file_path"] == filepath {
1598                                 break
1599                         }
1600                         c.Logf("logentries.Items: %+v", logentries.Items)
1601                         time.Sleep(50 * time.Millisecond)
1602                 }
1603         } else {
1604                 c.Check(resp.Result().StatusCode, check.Equals, http.StatusForbidden)
1605                 c.Check(logbuf.String(), check.Equals, "")
1606         }
1607 }
1608
1609 func (s *IntegrationSuite) TestDownloadLoggingPermission(c *check.C) {
1610         u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
1611
1612         s.handler.Cluster.Collections.TrustAllContent = true
1613
1614         for _, adminperm := range []bool{true, false} {
1615                 for _, userperm := range []bool{true, false} {
1616                         s.handler.Cluster.Collections.WebDAVPermission.Admin.Download = adminperm
1617                         s.handler.Cluster.Collections.WebDAVPermission.User.Download = userperm
1618
1619                         // Test admin permission
1620                         req := &http.Request{
1621                                 Method:     "GET",
1622                                 Host:       u.Host,
1623                                 URL:        u,
1624                                 RequestURI: u.RequestURI(),
1625                                 Header: http.Header{
1626                                         "Authorization": {"Bearer " + arvadostest.AdminToken},
1627                                 },
1628                         }
1629                         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", adminperm,
1630                                 arvadostest.AdminUserUUID, arvadostest.FooCollection, arvadostest.FooCollectionPDH, "foo")
1631
1632                         // Test user permission
1633                         req = &http.Request{
1634                                 Method:     "GET",
1635                                 Host:       u.Host,
1636                                 URL:        u,
1637                                 RequestURI: u.RequestURI(),
1638                                 Header: http.Header{
1639                                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
1640                                 },
1641                         }
1642                         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", userperm,
1643                                 arvadostest.ActiveUserUUID, arvadostest.FooCollection, arvadostest.FooCollectionPDH, "foo")
1644                 }
1645         }
1646
1647         s.handler.Cluster.Collections.WebDAVPermission.User.Download = true
1648
1649         for _, tryurl := range []string{"http://" + arvadostest.MultilevelCollection1 + ".keep-web.example/dir1/subdir/file1",
1650                 "http://keep-web/users/active/multilevel_collection_1/dir1/subdir/file1"} {
1651
1652                 u = mustParseURL(tryurl)
1653                 req := &http.Request{
1654                         Method:     "GET",
1655                         Host:       u.Host,
1656                         URL:        u,
1657                         RequestURI: u.RequestURI(),
1658                         Header: http.Header{
1659                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
1660                         },
1661                 }
1662                 s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true,
1663                         arvadostest.ActiveUserUUID, arvadostest.MultilevelCollection1, arvadostest.MultilevelCollection1PDH, "dir1/subdir/file1")
1664         }
1665
1666         u = mustParseURL("http://" + strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + ".keep-web.example/foo")
1667         req := &http.Request{
1668                 Method:     "GET",
1669                 Host:       u.Host,
1670                 URL:        u,
1671                 RequestURI: u.RequestURI(),
1672                 Header: http.Header{
1673                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
1674                 },
1675         }
1676         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true,
1677                 arvadostest.ActiveUserUUID, "", arvadostest.FooCollectionPDH, "foo")
1678 }
1679
1680 func (s *IntegrationSuite) TestUploadLoggingPermission(c *check.C) {
1681         for _, adminperm := range []bool{true, false} {
1682                 for _, userperm := range []bool{true, false} {
1683
1684                         arv := arvados.NewClientFromEnv()
1685                         arv.AuthToken = arvadostest.ActiveToken
1686
1687                         var coll arvados.Collection
1688                         err := arv.RequestAndDecode(&coll,
1689                                 "POST",
1690                                 "/arvados/v1/collections",
1691                                 nil,
1692                                 map[string]interface{}{
1693                                         "ensure_unique_name": true,
1694                                         "collection": map[string]interface{}{
1695                                                 "name": "test collection",
1696                                         },
1697                                 })
1698                         c.Assert(err, check.Equals, nil)
1699
1700                         u := mustParseURL("http://" + coll.UUID + ".keep-web.example/bar")
1701
1702                         s.handler.Cluster.Collections.WebDAVPermission.Admin.Upload = adminperm
1703                         s.handler.Cluster.Collections.WebDAVPermission.User.Upload = userperm
1704
1705                         // Test admin permission
1706                         req := &http.Request{
1707                                 Method:     "PUT",
1708                                 Host:       u.Host,
1709                                 URL:        u,
1710                                 RequestURI: u.RequestURI(),
1711                                 Header: http.Header{
1712                                         "Authorization": {"Bearer " + arvadostest.AdminToken},
1713                                 },
1714                                 Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
1715                         }
1716                         s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", adminperm,
1717                                 arvadostest.AdminUserUUID, coll.UUID, "", "bar")
1718
1719                         // Test user permission
1720                         req = &http.Request{
1721                                 Method:     "PUT",
1722                                 Host:       u.Host,
1723                                 URL:        u,
1724                                 RequestURI: u.RequestURI(),
1725                                 Header: http.Header{
1726                                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
1727                                 },
1728                                 Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
1729                         }
1730                         s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", userperm,
1731                                 arvadostest.ActiveUserUUID, coll.UUID, "", "bar")
1732                 }
1733         }
1734 }
1735
1736 func (s *IntegrationSuite) TestConcurrentWrites(c *check.C) {
1737         s.handler.Cluster.Collections.WebDAVCache.TTL = arvados.Duration(time.Second * 2)
1738         lockTidyInterval = time.Second
1739         client := arvados.NewClientFromEnv()
1740         client.AuthToken = arvadostest.ActiveTokenV2
1741         // Start small, and increase concurrency (2^2, 4^2, ...)
1742         // only until hitting failure. Avoids unnecessarily long
1743         // failure reports.
1744         for n := 2; n < 16 && !c.Failed(); n = n * 2 {
1745                 c.Logf("%s: n=%d", c.TestName(), n)
1746
1747                 var coll arvados.Collection
1748                 err := client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, nil)
1749                 c.Assert(err, check.IsNil)
1750                 defer client.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
1751
1752                 var wg sync.WaitGroup
1753                 for i := 0; i < n && !c.Failed(); i++ {
1754                         i := i
1755                         wg.Add(1)
1756                         go func() {
1757                                 defer wg.Done()
1758                                 u := mustParseURL(fmt.Sprintf("http://%s.collections.example.com/i=%d", coll.UUID, i))
1759                                 resp := httptest.NewRecorder()
1760                                 req, err := http.NewRequest("MKCOL", u.String(), nil)
1761                                 c.Assert(err, check.IsNil)
1762                                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
1763                                 s.handler.ServeHTTP(resp, req)
1764                                 c.Assert(resp.Code, check.Equals, http.StatusCreated)
1765                                 for j := 0; j < n && !c.Failed(); j++ {
1766                                         j := j
1767                                         wg.Add(1)
1768                                         go func() {
1769                                                 defer wg.Done()
1770                                                 content := fmt.Sprintf("i=%d/j=%d", i, j)
1771                                                 u := mustParseURL("http://" + coll.UUID + ".collections.example.com/" + content)
1772
1773                                                 resp := httptest.NewRecorder()
1774                                                 req, err := http.NewRequest("PUT", u.String(), strings.NewReader(content))
1775                                                 c.Assert(err, check.IsNil)
1776                                                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
1777                                                 s.handler.ServeHTTP(resp, req)
1778                                                 c.Check(resp.Code, check.Equals, http.StatusCreated)
1779
1780                                                 time.Sleep(time.Second)
1781                                                 resp = httptest.NewRecorder()
1782                                                 req, err = http.NewRequest("GET", u.String(), nil)
1783                                                 c.Assert(err, check.IsNil)
1784                                                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
1785                                                 s.handler.ServeHTTP(resp, req)
1786                                                 c.Check(resp.Code, check.Equals, http.StatusOK)
1787                                                 c.Check(resp.Body.String(), check.Equals, content)
1788                                         }()
1789                                 }
1790                         }()
1791                 }
1792                 wg.Wait()
1793                 for i := 0; i < n; i++ {
1794                         u := mustParseURL(fmt.Sprintf("http://%s.collections.example.com/i=%d", coll.UUID, i))
1795                         resp := httptest.NewRecorder()
1796                         req, err := http.NewRequest("PROPFIND", u.String(), &bytes.Buffer{})
1797                         c.Assert(err, check.IsNil)
1798                         req.Header.Set("Authorization", "Bearer "+client.AuthToken)
1799                         s.handler.ServeHTTP(resp, req)
1800                         c.Assert(resp.Code, check.Equals, http.StatusMultiStatus)
1801                 }
1802         }
1803 }