services/keep-web/handler_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package keepweb
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "fmt"
  11         "html"
  12         "io"
  13         "io/ioutil"
  14         "net/http"
  15         "net/http/httptest"
  16         "net/url"
  17         "os"
  18         "path/filepath"
  19         "regexp"
  20         "strings"
  21         "sync"
  22         "time"
  23
  24         "git.arvados.org/arvados.git/lib/config"
  25         "git.arvados.org/arvados.git/sdk/go/arvados"
  26         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  27         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  28         "git.arvados.org/arvados.git/sdk/go/auth"
  29         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  30         "git.arvados.org/arvados.git/sdk/go/keepclient"
  31         "github.com/prometheus/client_golang/prometheus"
  32         "github.com/sirupsen/logrus"
  33         check "gopkg.in/check.v1"
  34 )
  35
  36 var _ = check.Suite(&UnitSuite{})
  37
  38 func init() {
  39         arvados.DebugLocksPanicMode = true
  40 }
  41
  42 type UnitSuite struct {
  43         cluster *arvados.Cluster
  44         handler *handler
  45 }
  46
  47 func (s *UnitSuite) SetUpTest(c *check.C) {
  48         logger := ctxlog.TestLogger(c)
  49         ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), logger)
  50         ldr.Path = "-"
  51         cfg, err := ldr.Load()
  52         c.Assert(err, check.IsNil)
  53         cc, err := cfg.GetCluster("")
  54         c.Assert(err, check.IsNil)
  55         s.cluster = cc
  56         s.handler = &handler{
  57                 Cluster: cc,
  58                 Cache: cache{
  59                         cluster:  cc,
  60                         logger:   logger,
  61                         registry: prometheus.NewRegistry(),
  62                 },
  63                 metrics: newMetrics(prometheus.NewRegistry()),
  64         }
  65 }
  66
  67 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
  68         h := s.handler
  69         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
  70         req := &http.Request{
  71                 Method:     "OPTIONS",
  72                 Host:       u.Host,
  73                 URL:        u,
  74                 RequestURI: u.RequestURI(),
  75                 Header: http.Header{
  76                         "Origin":                        {"https://workbench.example"},
  77                         "Access-Control-Request-Method": {"POST"},
  78                 },
  79         }
  80
  81         // Check preflight for an allowed request
  82         resp := httptest.NewRecorder()
  83         h.ServeHTTP(resp, req)
  84         c.Check(resp.Code, check.Equals, http.StatusOK)
  85         c.Check(resp.Body.String(), check.Equals, "")
  86         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
  87         c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
  88         c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout, Cache-Control")
  89
  90         // Check preflight for a disallowed request
  91         resp = httptest.NewRecorder()
  92         req.Header.Set("Access-Control-Request-Method", "MAKE-COFFEE")
  93         h.ServeHTTP(resp, req)
  94         c.Check(resp.Body.String(), check.Equals, "")
  95         c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
  96 }
  97
  98 func (s *UnitSuite) TestWebdavPrefixAndSource(c *check.C) {
  99         for _, trial := range []struct {
 100                 method   string
 101                 path     string
 102                 prefix   string
 103                 source   string
 104                 notFound bool
 105                 seeOther bool
 106         }{
 107                 {
 108                         method: "PROPFIND",
 109                         path:   "/",
 110                 },
 111                 {
 112                         method: "PROPFIND",
 113                         path:   "/dir1",
 114                 },
 115                 {
 116                         method: "PROPFIND",
 117                         path:   "/dir1/",
 118                 },
 119                 {
 120                         method: "PROPFIND",
 121                         path:   "/dir1/foo",
 122                         prefix: "/dir1",
 123                         source: "/dir1",
 124                 },
 125                 {
 126                         method: "PROPFIND",
 127                         path:   "/prefix/dir1/foo",
 128                         prefix: "/prefix/",
 129                         source: "",
 130                 },
 131                 {
 132                         method: "PROPFIND",
 133                         path:   "/prefix/dir1/foo",
 134                         prefix: "/prefix",
 135                         source: "",
 136                 },
 137                 {
 138                         method: "PROPFIND",
 139                         path:   "/prefix/dir1/foo",
 140                         prefix: "/prefix/",
 141                         source: "/",
 142                 },
 143                 {
 144                         method: "PROPFIND",
 145                         path:   "/prefix/foo",
 146                         prefix: "/prefix/",
 147                         source: "/dir1/",
 148                 },
 149                 {
 150                         method: "GET",
 151                         path:   "/prefix/foo",
 152                         prefix: "/prefix/",
 153                         source: "/dir1/",
 154                 },
 155                 {
 156                         method: "PROPFIND",
 157                         path:   "/prefix/",
 158                         prefix: "/prefix",
 159                         source: "/dir1",
 160                 },
 161                 {
 162                         method: "PROPFIND",
 163                         path:   "/prefix",
 164                         prefix: "/prefix",
 165                         source: "/dir1/",
 166                 },
 167                 {
 168                         method:   "GET",
 169                         path:     "/prefix",
 170                         prefix:   "/prefix",
 171                         source:   "/dir1",
 172                         seeOther: true,
 173                 },
 174                 {
 175                         method:   "PROPFIND",
 176                         path:     "/dir1/foo",
 177                         prefix:   "",
 178                         source:   "/dir1",
 179                         notFound: true,
 180                 },
 181         } {
 182                 c.Logf("trial %+v", trial)
 183                 u := mustParseURL("http://" + arvadostest.FooBarDirCollection + ".keep-web.example" + trial.path)
 184                 req := &http.Request{
 185                         Method:     trial.method,
 186                         Host:       u.Host,
 187                         URL:        u,
 188                         RequestURI: u.RequestURI(),
 189                         Header: http.Header{
 190                                 "Authorization":   {"Bearer " + arvadostest.ActiveTokenV2},
 191                                 "X-Webdav-Prefix": {trial.prefix},
 192                                 "X-Webdav-Source": {trial.source},
 193                         },
 194                         Body: ioutil.NopCloser(bytes.NewReader(nil)),
 195                 }
 196
 197                 resp := httptest.NewRecorder()
 198                 s.handler.ServeHTTP(resp, req)
 199                 if trial.notFound {
 200                         c.Check(resp.Code, check.Equals, http.StatusNotFound)
 201                 } else if trial.method == "PROPFIND" {
 202                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus)
 203                         c.Check(resp.Body.String(), check.Matches, `(?ms).*>\n?$`)
 204                 } else if trial.seeOther {
 205                         c.Check(resp.Code, check.Equals, http.StatusSeeOther)
 206                 } else {
 207                         c.Check(resp.Code, check.Equals, http.StatusOK)
 208                 }
 209         }
 210 }
 211
 212 func (s *UnitSuite) TestEmptyResponse(c *check.C) {
 213         // Ensure we start with an empty cache
 214         defer os.Setenv("HOME", os.Getenv("HOME"))
 215         os.Setenv("HOME", c.MkDir())
 216
 217         for _, trial := range []struct {
 218                 dataExists    bool
 219                 sendIMSHeader bool
 220                 expectStatus  int
 221                 logRegexp     string
 222         }{
 223                 // If we return no content due to a Keep read error,
 224                 // we should emit a log message.
 225                 {false, false, http.StatusOK, `(?ms).*only wrote 0 bytes.*`},
 226
 227                 // If we return no content because the client sent an
 228                 // If-Modified-Since header, our response should be
 229                 // 304.  We still expect a "File download" log since it
 230                 // counts as a file access for auditing.
 231                 {true, true, http.StatusNotModified, `(?ms).*msg="File download".*`},
 232         } {
 233                 c.Logf("trial: %+v", trial)
 234                 arvadostest.StartKeep(2, true)
 235                 if trial.dataExists {
 236                         arv, err := arvadosclient.MakeArvadosClient()
 237                         c.Assert(err, check.IsNil)
 238                         arv.ApiToken = arvadostest.ActiveToken
 239                         kc, err := keepclient.MakeKeepClient(arv)
 240                         c.Assert(err, check.IsNil)
 241                         _, _, err = kc.PutB([]byte("foo"))
 242                         c.Assert(err, check.IsNil)
 243                 }
 244
 245                 u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
 246                 req := &http.Request{
 247                         Method:     "GET",
 248                         Host:       u.Host,
 249                         URL:        u,
 250                         RequestURI: u.RequestURI(),
 251                         Header: http.Header{
 252                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
 253                         },
 254                 }
 255                 if trial.sendIMSHeader {
 256                         req.Header.Set("If-Modified-Since", strings.Replace(time.Now().UTC().Format(time.RFC1123), "UTC", "GMT", -1))
 257                 }
 258
 259                 var logbuf bytes.Buffer
 260                 logger := logrus.New()
 261                 logger.Out = &logbuf
 262                 req = req.WithContext(ctxlog.Context(context.Background(), logger))
 263
 264                 resp := httptest.NewRecorder()
 265                 s.handler.ServeHTTP(resp, req)
 266                 c.Check(resp.Code, check.Equals, trial.expectStatus)
 267                 c.Check(resp.Body.String(), check.Equals, "")
 268
 269                 c.Log(logbuf.String())
 270                 c.Check(logbuf.String(), check.Matches, trial.logRegexp)
 271         }
 272 }
 273
 274 func (s *UnitSuite) TestInvalidUUID(c *check.C) {
 275         bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-"
 276         token := arvadostest.ActiveToken
 277         for _, trial := range []string{
 278                 "http://keep-web/c=" + bogusID + "/foo",
 279                 "http://keep-web/c=" + bogusID + "/t=" + token + "/foo",
 280                 "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo",
 281                 "http://keep-web/collections/" + bogusID + "/foo",
 282                 "http://" + bogusID + ".keep-web/" + bogusID + "/foo",
 283                 "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
 284         } {
 285                 c.Log(trial)
 286                 u := mustParseURL(trial)
 287                 req := &http.Request{
 288                         Method:     "GET",
 289                         Host:       u.Host,
 290                         URL:        u,
 291                         RequestURI: u.RequestURI(),
 292                 }
 293                 resp := httptest.NewRecorder()
 294                 s.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 295                 s.handler.ServeHTTP(resp, req)
 296                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 297         }
 298 }
 299
 300 func mustParseURL(s string) *url.URL {
 301         r, err := url.Parse(s)
 302         if err != nil {
 303                 panic("parse URL: " + s)
 304         }
 305         return r
 306 }
 307
 308 func (s *IntegrationSuite) TestVhost404(c *check.C) {
 309         for _, testURL := range []string{
 310                 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
 311                 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
 312         } {
 313                 resp := httptest.NewRecorder()
 314                 u := mustParseURL(testURL)
 315                 req := &http.Request{
 316                         Method:     "GET",
 317                         URL:        u,
 318                         RequestURI: u.RequestURI(),
 319                 }
 320                 s.handler.ServeHTTP(resp, req)
 321                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 322                 c.Check(resp.Body.String(), check.Equals, notFoundMessage+"\n")
 323         }
 324 }
 325
 326 // An authorizer modifies an HTTP request to make use of the given
 327 // token -- by adding it to a header, cookie, query param, or whatever
 328 // -- and returns the HTTP status code we should expect from keep-web if
 329 // the token is invalid.
 330 type authorizer func(*http.Request, string) int
 331
 332 func (s *IntegrationSuite) TestVhostViaAuthzHeaderOAuth2(c *check.C) {
 333         s.doVhostRequests(c, authzViaAuthzHeaderOAuth2)
 334 }
 335 func authzViaAuthzHeaderOAuth2(r *http.Request, tok string) int {
 336         r.Header.Add("Authorization", "OAuth2 "+tok)
 337         return http.StatusUnauthorized
 338 }
 339
 340 func (s *IntegrationSuite) TestVhostViaAuthzHeaderBearer(c *check.C) {
 341         s.doVhostRequests(c, authzViaAuthzHeaderBearer)
 342 }
 343 func authzViaAuthzHeaderBearer(r *http.Request, tok string) int {
 344         r.Header.Add("Authorization", "Bearer "+tok)
 345         return http.StatusUnauthorized
 346 }
 347
 348 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
 349         s.doVhostRequests(c, authzViaCookieValue)
 350 }
 351 func authzViaCookieValue(r *http.Request, tok string) int {
 352         r.AddCookie(&http.Cookie{
 353                 Name:  "arvados_api_token",
 354                 Value: auth.EncodeTokenCookie([]byte(tok)),
 355         })
 356         return http.StatusUnauthorized
 357 }
 358
 359 func (s *IntegrationSuite) TestVhostViaHTTPBasicAuth(c *check.C) {
 360         s.doVhostRequests(c, authzViaHTTPBasicAuth)
 361 }
 362 func authzViaHTTPBasicAuth(r *http.Request, tok string) int {
 363         r.AddCookie(&http.Cookie{
 364                 Name:  "arvados_api_token",
 365                 Value: auth.EncodeTokenCookie([]byte(tok)),
 366         })
 367         return http.StatusUnauthorized
 368 }
 369
 370 func (s *IntegrationSuite) TestVhostViaHTTPBasicAuthWithExtraSpaceChars(c *check.C) {
 371         s.doVhostRequests(c, func(r *http.Request, tok string) int {
 372                 r.AddCookie(&http.Cookie{
 373                         Name:  "arvados_api_token",
 374                         Value: auth.EncodeTokenCookie([]byte(" " + tok + "\n")),
 375                 })
 376                 return http.StatusUnauthorized
 377         })
 378 }
 379
 380 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
 381         s.doVhostRequests(c, authzViaPath)
 382 }
 383 func authzViaPath(r *http.Request, tok string) int {
 384         r.URL.Path = "/t=" + tok + r.URL.Path
 385         return http.StatusNotFound
 386 }
 387
 388 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
 389         s.doVhostRequests(c, authzViaQueryString)
 390 }
 391 func authzViaQueryString(r *http.Request, tok string) int {
 392         r.URL.RawQuery = "api_token=" + tok
 393         return http.StatusUnauthorized
 394 }
 395
 396 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
 397         s.doVhostRequests(c, authzViaPOST)
 398 }
 399 func authzViaPOST(r *http.Request, tok string) int {
 400         r.Method = "POST"
 401         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 402         r.Body = ioutil.NopCloser(strings.NewReader(
 403                 url.Values{"api_token": {tok}}.Encode()))
 404         return http.StatusUnauthorized
 405 }
 406
 407 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
 408         s.doVhostRequests(c, authzViaPOST)
 409 }
 410 func authzViaXHRPOST(r *http.Request, tok string) int {
 411         r.Method = "POST"
 412         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 413         r.Header.Add("Origin", "https://origin.example")
 414         r.Body = ioutil.NopCloser(strings.NewReader(
 415                 url.Values{
 416                         "api_token":   {tok},
 417                         "disposition": {"attachment"},
 418                 }.Encode()))
 419         return http.StatusUnauthorized
 420 }
 421
 422 // Try some combinations of {url, token} using the given authorization
 423 // mechanism, and verify the result is correct.
 424 func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
 425         for _, hostPath := range []string{
 426                 arvadostest.FooCollection + ".example.com/foo",
 427                 arvadostest.FooCollection + "--collections.example.com/foo",
 428                 arvadostest.FooCollection + "--collections.example.com/_/foo",
 429                 arvadostest.FooCollectionPDH + ".example.com/foo",
 430                 strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + "--collections.example.com/foo",
 431                 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
 432         } {
 433                 c.Log("doRequests: ", hostPath)
 434                 s.doVhostRequestsWithHostPath(c, authz, hostPath)
 435         }
 436 }
 437
 438 func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
 439         for _, tok := range []string{
 440                 arvadostest.ActiveToken,
 441                 arvadostest.ActiveToken[:15],
 442                 arvadostest.SpectatorToken,
 443                 "bogus",
 444                 "",
 445         } {
 446                 u := mustParseURL("http://" + hostPath)
 447                 req := &http.Request{
 448                         Method:     "GET",
 449                         Host:       u.Host,
 450                         URL:        u,
 451                         RequestURI: u.RequestURI(),
 452                         Header:     http.Header{},
 453                 }
 454                 failCode := authz(req, tok)
 455                 req, resp := s.doReq(req)
 456                 code, body := resp.Code, resp.Body.String()
 457
 458                 // If the initial request had a (non-empty) token
 459                 // showing in the query string, we should have been
 460                 // redirected in order to hide it in a cookie.
 461                 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
 462
 463                 if tok == arvadostest.ActiveToken {
 464                         c.Check(code, check.Equals, http.StatusOK)
 465                         c.Check(body, check.Equals, "foo")
 466                 } else {
 467                         c.Check(code >= 400, check.Equals, true)
 468                         c.Check(code < 500, check.Equals, true)
 469                         if tok == arvadostest.SpectatorToken {
 470                                 // Valid token never offers to retry
 471                                 // with different credentials.
 472                                 c.Check(code, check.Equals, http.StatusNotFound)
 473                         } else {
 474                                 // Invalid token can ask to retry
 475                                 // depending on the authz method.
 476                                 c.Check(code, check.Equals, failCode)
 477                         }
 478                         if code == 404 {
 479                                 c.Check(body, check.Equals, notFoundMessage+"\n")
 480                         } else {
 481                                 c.Check(body, check.Equals, unauthorizedMessage+"\n")
 482                         }
 483                 }
 484         }
 485 }
 486
 487 func (s *IntegrationSuite) TestVhostPortMatch(c *check.C) {
 488         for _, host := range []string{"download.example.com", "DOWNLOAD.EXAMPLE.COM"} {
 489                 for _, port := range []string{"80", "443", "8000"} {
 490                         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = fmt.Sprintf("download.example.com:%v", port)
 491                         u := mustParseURL(fmt.Sprintf("http://%v/by_id/%v/foo", host, arvadostest.FooCollection))
 492                         req := &http.Request{
 493                                 Method:     "GET",
 494                                 Host:       u.Host,
 495                                 URL:        u,
 496                                 RequestURI: u.RequestURI(),
 497                                 Header:     http.Header{"Authorization": []string{"Bearer " + arvadostest.ActiveToken}},
 498                         }
 499                         req, resp := s.doReq(req)
 500                         code, _ := resp.Code, resp.Body.String()
 501
 502                         if port == "8000" {
 503                                 c.Check(code, check.Equals, 401)
 504                         } else {
 505                                 c.Check(code, check.Equals, 200)
 506                         }
 507                 }
 508         }
 509 }
 510
 511 func (s *IntegrationSuite) do(method string, urlstring string, token string, hdr http.Header) (*http.Request, *httptest.ResponseRecorder) {
 512         u := mustParseURL(urlstring)
 513         if hdr == nil && token != "" {
 514                 hdr = http.Header{"Authorization": {"Bearer " + token}}
 515         } else if hdr == nil {
 516                 hdr = http.Header{}
 517         } else if token != "" {
 518                 panic("must not pass both token and hdr")
 519         }
 520         return s.doReq(&http.Request{
 521                 Method:     method,
 522                 Host:       u.Host,
 523                 URL:        u,
 524                 RequestURI: u.RequestURI(),
 525                 Header:     hdr,
 526         })
 527 }
 528
 529 func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
 530         resp := httptest.NewRecorder()
 531         s.handler.ServeHTTP(resp, req)
 532         if resp.Code != http.StatusSeeOther {
 533                 return req, resp
 534         }
 535         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 536         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 537         req = &http.Request{
 538                 Method:     "GET",
 539                 Host:       u.Host,
 540                 URL:        u,
 541                 RequestURI: u.RequestURI(),
 542                 Header:     http.Header{},
 543         }
 544         for _, c := range cookies {
 545                 req.AddCookie(c)
 546         }
 547         return s.doReq(req)
 548 }
 549
 550 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
 551         s.testVhostRedirectTokenToCookie(c, "GET",
 552                 arvadostest.FooCollection+".example.com/foo",
 553                 "?api_token="+arvadostest.ActiveToken,
 554                 nil,
 555                 "",
 556                 http.StatusOK,
 557                 "foo",
 558         )
 559 }
 560
 561 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
 562         s.testVhostRedirectTokenToCookie(c, "GET",
 563                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
 564                 "",
 565                 nil,
 566                 "",
 567                 http.StatusOK,
 568                 "foo",
 569         )
 570 }
 571
 572 func (s *IntegrationSuite) TestCollectionSharingToken(c *check.C) {
 573         s.testVhostRedirectTokenToCookie(c, "GET",
 574                 "example.com/c="+arvadostest.FooFileCollectionUUID+"/t="+arvadostest.FooFileCollectionSharingToken+"/foo",
 575                 "",
 576                 nil,
 577                 "",
 578                 http.StatusOK,
 579                 "foo",
 580         )
 581         // Same valid sharing token, but requesting a different collection
 582         s.testVhostRedirectTokenToCookie(c, "GET",
 583                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.FooFileCollectionSharingToken+"/foo",
 584                 "",
 585                 nil,
 586                 "",
 587                 http.StatusNotFound,
 588                 regexp.QuoteMeta(notFoundMessage+"\n"),
 589         )
 590 }
 591
 592 // Bad token in URL is 404 Not Found because it doesn't make sense to
 593 // retry the same URL with different authorization.
 594 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
 595         s.testVhostRedirectTokenToCookie(c, "GET",
 596                 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
 597                 "",
 598                 nil,
 599                 "",
 600                 http.StatusNotFound,
 601                 regexp.QuoteMeta(notFoundMessage+"\n"),
 602         )
 603 }
 604
 605 // Bad token in a cookie (even if it got there via our own
 606 // query-string-to-cookie redirect) is, in principle, retryable via
 607 // wb2-login-and-redirect flow.
 608 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
 609         // Inline
 610         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 611                 arvadostest.FooCollection+".example.com/foo",
 612                 "?api_token=thisisabogustoken",
 613                 http.Header{"Sec-Fetch-Mode": {"navigate"}},
 614                 "",
 615                 http.StatusSeeOther,
 616                 "",
 617         )
 618         u, err := url.Parse(resp.Header().Get("Location"))
 619         c.Assert(err, check.IsNil)
 620         c.Logf("redirected to %s", u)
 621         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 622         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 623         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
 624
 625         // Download/attachment indicated by ?disposition=attachment
 626         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 627                 arvadostest.FooCollection+".example.com/foo",
 628                 "?api_token=thisisabogustoken&disposition=attachment",
 629                 http.Header{"Sec-Fetch-Mode": {"navigate"}},
 630                 "",
 631                 http.StatusSeeOther,
 632                 "",
 633         )
 634         u, err = url.Parse(resp.Header().Get("Location"))
 635         c.Assert(err, check.IsNil)
 636         c.Logf("redirected to %s", u)
 637         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 638         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "")
 639         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 640
 641         // Download/attachment indicated by vhost
 642         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 643                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
 644                 "?api_token=thisisabogustoken",
 645                 http.Header{"Sec-Fetch-Mode": {"navigate"}},
 646                 "",
 647                 http.StatusSeeOther,
 648                 "",
 649         )
 650         u, err = url.Parse(resp.Header().Get("Location"))
 651         c.Assert(err, check.IsNil)
 652         c.Logf("redirected to %s", u)
 653         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 654         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "")
 655         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 656
 657         // Without "Sec-Fetch-Mode: navigate" header, just 401.
 658         s.testVhostRedirectTokenToCookie(c, "GET",
 659                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
 660                 "?api_token=thisisabogustoken",
 661                 http.Header{"Sec-Fetch-Mode": {"cors"}},
 662                 "",
 663                 http.StatusUnauthorized,
 664                 regexp.QuoteMeta(unauthorizedMessage+"\n"),
 665         )
 666         s.testVhostRedirectTokenToCookie(c, "GET",
 667                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
 668                 "?api_token=thisisabogustoken",
 669                 nil,
 670                 "",
 671                 http.StatusUnauthorized,
 672                 regexp.QuoteMeta(unauthorizedMessage+"\n"),
 673         )
 674 }
 675
 676 func (s *IntegrationSuite) TestVhostRedirectWithNoCache(c *check.C) {
 677         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 678                 arvadostest.FooCollection+".example.com/foo",
 679                 "?api_token=thisisabogustoken",
 680                 http.Header{
 681                         "Sec-Fetch-Mode": {"navigate"},
 682                         "Cache-Control":  {"no-cache"},
 683                 },
 684                 "",
 685                 http.StatusSeeOther,
 686                 "",
 687         )
 688         u, err := url.Parse(resp.Header().Get("Location"))
 689         c.Assert(err, check.IsNil)
 690         c.Logf("redirected to %s", u)
 691         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 692         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 693         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
 694 }
 695
 696 func (s *IntegrationSuite) TestNoTokenWorkbench2LoginFlow(c *check.C) {
 697         for _, trial := range []struct {
 698                 anonToken    bool
 699                 cacheControl string
 700         }{
 701                 {},
 702                 {cacheControl: "no-cache"},
 703                 {anonToken: true},
 704                 {anonToken: true, cacheControl: "no-cache"},
 705         } {
 706                 c.Logf("trial: %+v", trial)
 707
 708                 if trial.anonToken {
 709                         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 710                 } else {
 711                         s.handler.Cluster.Users.AnonymousUserToken = ""
 712                 }
 713                 req, err := http.NewRequest("GET", "http://"+arvadostest.FooCollection+".example.com/foo", nil)
 714                 c.Assert(err, check.IsNil)
 715                 req.Header.Set("Sec-Fetch-Mode", "navigate")
 716                 if trial.cacheControl != "" {
 717                         req.Header.Set("Cache-Control", trial.cacheControl)
 718                 }
 719                 resp := httptest.NewRecorder()
 720                 s.handler.ServeHTTP(resp, req)
 721                 c.Check(resp.Code, check.Equals, http.StatusSeeOther)
 722                 u, err := url.Parse(resp.Header().Get("Location"))
 723                 c.Assert(err, check.IsNil)
 724                 c.Logf("redirected to %q", u)
 725                 c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 726                 c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 727                 c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
 728         }
 729 }
 730
 731 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
 732         s.testVhostRedirectTokenToCookie(c, "GET",
 733                 "example.com/c="+arvadostest.FooCollection+"/foo",
 734                 "?api_token="+arvadostest.ActiveToken,
 735                 nil,
 736                 "",
 737                 http.StatusBadRequest,
 738                 regexp.QuoteMeta("cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n"),
 739         )
 740 }
 741
 742 // If client requests an attachment by putting ?disposition=attachment
 743 // in the query string, and gets redirected, the redirect target
 744 // should respond with an attachment.
 745 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
 746         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 747                 arvadostest.FooCollection+".example.com/foo",
 748                 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
 749                 nil,
 750                 "",
 751                 http.StatusOK,
 752                 "foo",
 753         )
 754         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 755 }
 756
 757 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
 758         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 759         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 760                 "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
 761                 "?api_token="+arvadostest.ActiveToken,
 762                 nil,
 763                 "",
 764                 http.StatusOK,
 765                 "foo",
 766         )
 767         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 768 }
 769
 770 func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
 771         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 772         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 773                 "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
 774                 "?api_token="+arvadostest.ActiveToken,
 775                 nil,
 776                 "",
 777                 http.StatusOK,
 778                 "waz",
 779         )
 780         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 781         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 782                 "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz",
 783                 "?api_token="+arvadostest.ActiveToken,
 784                 nil,
 785                 "",
 786                 http.StatusOK,
 787                 "waz",
 788         )
 789         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 790 }
 791
 792 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
 793         s.handler.Cluster.Collections.TrustAllContent = true
 794         s.testVhostRedirectTokenToCookie(c, "GET",
 795                 "example.com/c="+arvadostest.FooCollection+"/foo",
 796                 "?api_token="+arvadostest.ActiveToken,
 797                 nil,
 798                 "",
 799                 http.StatusOK,
 800                 "foo",
 801         )
 802 }
 803
 804 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
 805         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
 806
 807         s.testVhostRedirectTokenToCookie(c, "GET",
 808                 "example.com/c="+arvadostest.FooCollection+"/foo",
 809                 "?api_token="+arvadostest.ActiveToken,
 810                 nil,
 811                 "",
 812                 http.StatusBadRequest,
 813                 regexp.QuoteMeta("cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n"),
 814         )
 815
 816         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 817                 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
 818                 "?api_token="+arvadostest.ActiveToken,
 819                 nil,
 820                 "",
 821                 http.StatusOK,
 822                 "foo",
 823         )
 824         c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
 825 }
 826
 827 func (s *IntegrationSuite) TestVhostRedirectMultipleTokens(c *check.C) {
 828         baseUrl := arvadostest.FooCollection + ".example.com/foo"
 829         query := url.Values{}
 830
 831         // The intent of these tests is to check that requests are redirected
 832         // correctly in the presence of multiple API tokens. The exact response
 833         // codes and content are not closely considered: they're just how
 834         // keep-web responded when we made the smallest possible fix. Changing
 835         // those responses may be okay, but you should still test all these
 836         // different cases and the associated redirect logic.
 837         query["api_token"] = []string{arvadostest.ActiveToken, arvadostest.AnonymousToken}
 838         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
 839         query["api_token"] = []string{arvadostest.ActiveToken, arvadostest.AnonymousToken, ""}
 840         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
 841         query["api_token"] = []string{arvadostest.ActiveToken, "", arvadostest.AnonymousToken}
 842         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
 843         query["api_token"] = []string{"", arvadostest.ActiveToken}
 844         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
 845
 846         expectContent := regexp.QuoteMeta(unauthorizedMessage + "\n")
 847         query["api_token"] = []string{arvadostest.AnonymousToken, "invalidtoo"}
 848         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusUnauthorized, expectContent)
 849         query["api_token"] = []string{arvadostest.AnonymousToken, ""}
 850         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusUnauthorized, expectContent)
 851         query["api_token"] = []string{"", arvadostest.AnonymousToken}
 852         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusUnauthorized, expectContent)
 853 }
 854
 855 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
 856         s.testVhostRedirectTokenToCookie(c, "POST",
 857                 arvadostest.FooCollection+".example.com/foo",
 858                 "",
 859                 http.Header{"Content-Type": {"application/x-www-form-urlencoded"}},
 860                 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
 861                 http.StatusOK,
 862                 "foo",
 863         )
 864 }
 865
 866 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
 867         s.testVhostRedirectTokenToCookie(c, "POST",
 868                 arvadostest.FooCollection+".example.com/foo",
 869                 "",
 870                 http.Header{"Content-Type": {"application/x-www-form-urlencoded"}},
 871                 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
 872                 http.StatusNotFound,
 873                 regexp.QuoteMeta(notFoundMessage+"\n"),
 874         )
 875 }
 876
 877 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
 878         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 879         s.testVhostRedirectTokenToCookie(c, "GET",
 880                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 881                 "",
 882                 nil,
 883                 "",
 884                 http.StatusOK,
 885                 "Hello world\n",
 886         )
 887 }
 888
 889 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
 890         s.handler.Cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
 891         s.testVhostRedirectTokenToCookie(c, "GET",
 892                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 893                 "",
 894                 nil,
 895                 "",
 896                 http.StatusUnauthorized,
 897                 "Authorization tokens are not accepted here: .*\n",
 898         )
 899 }
 900
 901 func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
 902         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 903
 904         client := arvados.NewClientFromEnv()
 905         client.AuthToken = arvadostest.ActiveToken
 906         fs, err := (&arvados.Collection{}).FileSystem(client, nil)
 907         c.Assert(err, check.IsNil)
 908         f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777)
 909         c.Assert(err, check.IsNil)
 910         f.Close()
 911         mtxt, err := fs.MarshalManifest(".")
 912         c.Assert(err, check.IsNil)
 913         var coll arvados.Collection
 914         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 915                 "collection": map[string]string{
 916                         "manifest_text": mtxt,
 917                 },
 918         })
 919         c.Assert(err, check.IsNil)
 920
 921         u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
 922         req := &http.Request{
 923                 Method:     "GET",
 924                 Host:       u.Host,
 925                 URL:        u,
 926                 RequestURI: u.RequestURI(),
 927                 Header: http.Header{
 928                         "Authorization": {"Bearer " + client.AuthToken},
 929                 },
 930         }
 931         resp := httptest.NewRecorder()
 932         s.handler.ServeHTTP(resp, req)
 933         c.Check(resp.Code, check.Equals, http.StatusOK)
 934         c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\&#34;odd&#39; path chars.*`)
 935 }
 936
 937 func (s *IntegrationSuite) TestForwardSlashSubstitution(c *check.C) {
 938         arv := arvados.NewClientFromEnv()
 939         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 940         s.handler.Cluster.Collections.ForwardSlashNameSubstitution = "{SOLIDUS}"
 941         name := "foo/bar/baz"
 942         nameShown := strings.Replace(name, "/", "{SOLIDUS}", -1)
 943         nameShownEscaped := strings.Replace(name, "/", "%7bSOLIDUS%7d", -1)
 944
 945         client := arvados.NewClientFromEnv()
 946         client.AuthToken = arvadostest.ActiveToken
 947         fs, err := (&arvados.Collection{}).FileSystem(client, nil)
 948         c.Assert(err, check.IsNil)
 949         f, err := fs.OpenFile("filename", os.O_CREATE, 0777)
 950         c.Assert(err, check.IsNil)
 951         f.Close()
 952         mtxt, err := fs.MarshalManifest(".")
 953         c.Assert(err, check.IsNil)
 954         var coll arvados.Collection
 955         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 956                 "collection": map[string]string{
 957                         "manifest_text": mtxt,
 958                         "name":          name,
 959                         "owner_uuid":    arvadostest.AProjectUUID,
 960                 },
 961         })
 962         c.Assert(err, check.IsNil)
 963         defer arv.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
 964
 965         base := "http://download.example.com/by_id/" + coll.OwnerUUID + "/"
 966         for tryURL, expectRegexp := range map[string]string{
 967                 base:                          `(?ms).*href="./` + nameShownEscaped + `/"\S+` + nameShown + `.*`,
 968                 base + nameShownEscaped + "/": `(?ms).*href="./filename"\S+filename.*`,
 969         } {
 970                 u, _ := url.Parse(tryURL)
 971                 req := &http.Request{
 972                         Method:     "GET",
 973                         Host:       u.Host,
 974                         URL:        u,
 975                         RequestURI: u.RequestURI(),
 976                         Header: http.Header{
 977                                 "Authorization": {"Bearer " + client.AuthToken},
 978                         },
 979                 }
 980                 resp := httptest.NewRecorder()
 981                 s.handler.ServeHTTP(resp, req)
 982                 c.Check(resp.Code, check.Equals, http.StatusOK)
 983                 c.Check(resp.Body.String(), check.Matches, expectRegexp)
 984         }
 985 }
 986
 987 // XHRs can't follow redirect-with-cookie so they rely on method=POST
 988 // and disposition=attachment (telling us it's acceptable to respond
 989 // with content instead of a redirect) and an Origin header that gets
 990 // added automatically by the browser (telling us it's desirable to do
 991 // so).
 992 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
 993         u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
 994         req := &http.Request{
 995                 Method:     "POST",
 996                 Host:       u.Host,
 997                 URL:        u,
 998                 RequestURI: u.RequestURI(),
 999                 Header: http.Header{
1000                         "Origin":       {"https://origin.example"},
1001                         "Content-Type": {"application/x-www-form-urlencoded"},
1002                 },
1003                 Body: ioutil.NopCloser(strings.NewReader(url.Values{
1004                         "api_token":   {arvadostest.ActiveToken},
1005                         "disposition": {"attachment"},
1006                 }.Encode())),
1007         }
1008         resp := httptest.NewRecorder()
1009         s.handler.ServeHTTP(resp, req)
1010         c.Check(resp.Code, check.Equals, http.StatusOK)
1011         c.Check(resp.Body.String(), check.Equals, "foo")
1012         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
1013
1014         // GET + Origin header is representative of both AJAX GET
1015         // requests and inline images via <IMG crossorigin="anonymous"
1016         // src="...">.
1017         u.RawQuery = "api_token=" + url.QueryEscape(arvadostest.ActiveTokenV2)
1018         req = &http.Request{
1019                 Method:     "GET",
1020                 Host:       u.Host,
1021                 URL:        u,
1022                 RequestURI: u.RequestURI(),
1023                 Header: http.Header{
1024                         "Origin": {"https://origin.example"},
1025                 },
1026         }
1027         resp = httptest.NewRecorder()
1028         s.handler.ServeHTTP(resp, req)
1029         c.Check(resp.Code, check.Equals, http.StatusOK)
1030         c.Check(resp.Body.String(), check.Equals, "foo")
1031         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
1032 }
1033
1034 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString string, reqHeader http.Header, reqBody string, expectStatus int, matchRespBody string) *httptest.ResponseRecorder {
1035         if reqHeader == nil {
1036                 reqHeader = http.Header{}
1037         }
1038         u, _ := url.Parse(`http://` + hostPath + queryString)
1039         c.Logf("requesting %s", u)
1040         req := &http.Request{
1041                 Method:     method,
1042                 Host:       u.Host,
1043                 URL:        u,
1044                 RequestURI: u.RequestURI(),
1045                 Header:     reqHeader,
1046                 Body:       ioutil.NopCloser(strings.NewReader(reqBody)),
1047         }
1048
1049         resp := httptest.NewRecorder()
1050         defer func() {
1051                 c.Check(resp.Code, check.Equals, expectStatus)
1052                 c.Check(resp.Body.String(), check.Matches, matchRespBody)
1053         }()
1054
1055         s.handler.ServeHTTP(resp, req)
1056         if resp.Code != http.StatusSeeOther {
1057                 attachment, _ := regexp.MatchString(`^attachment(;|$)`, resp.Header().Get("Content-Disposition"))
1058                 // Since we're not redirecting, check that any api_token in the URL is
1059                 // handled safely.
1060                 // If there is no token in the URL, then we're good.
1061                 // Otherwise, if the response code is an error, the body is expected to
1062                 // be static content, and nothing that might maliciously introspect the
1063                 // URL. It's considered safe and allowed.
1064                 // Otherwise, if the response content has attachment disposition,
1065                 // that's considered safe for all the reasons explained in the
1066                 // safeAttachment comment in handler.go.
1067                 c.Check(!u.Query().Has("api_token") || resp.Code >= 400 || attachment, check.Equals, true)
1068                 return resp
1069         }
1070
1071         loc, err := url.Parse(resp.Header().Get("Location"))
1072         c.Assert(err, check.IsNil)
1073         c.Check(loc.Scheme, check.Equals, u.Scheme)
1074         c.Check(loc.Host, check.Equals, u.Host)
1075         c.Check(loc.RawPath, check.Equals, u.RawPath)
1076         // If the response was a redirect, it should never include an API token.
1077         c.Check(loc.Query().Has("api_token"), check.Equals, false)
1078         c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
1079         cookies := (&http.Response{Header: resp.Header()}).Cookies()
1080
1081         c.Logf("following redirect to %s", u)
1082         req = &http.Request{
1083                 Method:     "GET",
1084                 Host:       loc.Host,
1085                 URL:        loc,
1086                 RequestURI: loc.RequestURI(),
1087                 Header:     reqHeader,
1088         }
1089         for _, c := range cookies {
1090                 req.AddCookie(c)
1091         }
1092
1093         resp = httptest.NewRecorder()
1094         s.handler.ServeHTTP(resp, req)
1095
1096         if resp.Code != http.StatusSeeOther {
1097                 c.Check(resp.Header().Get("Location"), check.Equals, "")
1098         }
1099         return resp
1100 }
1101
1102 func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) {
1103         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
1104         s.testDirectoryListing(c)
1105 }
1106
1107 func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) {
1108         s.handler.Cluster.Users.AnonymousUserToken = ""
1109         s.testDirectoryListing(c)
1110 }
1111
1112 func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
1113         // The "ownership cycle" test fixtures are reachable from the
1114         // "filter group without filters" group, causing webdav's
1115         // walkfs to recurse indefinitely. Avoid that by deleting one
1116         // of the bogus fixtures.
1117         arv := arvados.NewClientFromEnv()
1118         err := arv.RequestAndDecode(nil, "DELETE", "arvados/v1/groups/zzzzz-j7d0g-cx2al9cqkmsf1hs", nil, nil)
1119         if err != nil {
1120                 c.Assert(err, check.FitsTypeOf, &arvados.TransactionError{})
1121                 c.Check(err.(*arvados.TransactionError).StatusCode, check.Equals, 404)
1122         }
1123
1124         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1125         authHeader := http.Header{
1126                 "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
1127         }
1128         for _, trial := range []struct {
1129                 uri      string
1130                 header   http.Header
1131                 expect   []string
1132                 redirect string
1133                 cutDirs  int
1134         }{
1135                 {
1136                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
1137                         header:  authHeader,
1138                         expect:  []string{"dir1/foo", "dir1/bar"},
1139                         cutDirs: 0,
1140                 },
1141                 {
1142                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
1143                         header:  authHeader,
1144                         expect:  []string{"foo", "bar"},
1145                         cutDirs: 1,
1146                 },
1147                 {
1148                         // URLs of this form ignore authHeader, and
1149                         // FooAndBarFilesInDirUUID isn't public, so
1150                         // this returns 401.
1151                         uri:    "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
1152                         header: authHeader,
1153                         expect: nil,
1154                 },
1155                 {
1156                         uri:     "download.example.com/users/active/foo_file_in_dir/",
1157                         header:  authHeader,
1158                         expect:  []string{"dir1/"},
1159                         cutDirs: 3,
1160                 },
1161                 {
1162                         uri:     "download.example.com/users/active/foo_file_in_dir/dir1/",
1163                         header:  authHeader,
1164                         expect:  []string{"bar"},
1165                         cutDirs: 4,
1166                 },
1167                 {
1168                         uri:     "download.example.com/",
1169                         header:  authHeader,
1170                         expect:  []string{"users/"},
1171                         cutDirs: 0,
1172                 },
1173                 {
1174                         uri:      "download.example.com/users",
1175                         header:   authHeader,
1176                         redirect: "/users/",
1177                         expect:   []string{"active/"},
1178                         cutDirs:  1,
1179                 },
1180                 {
1181                         uri:     "download.example.com/users/",
1182                         header:  authHeader,
1183                         expect:  []string{"active/"},
1184                         cutDirs: 1,
1185                 },
1186                 {
1187                         uri:      "download.example.com/users/active",
1188                         header:   authHeader,
1189                         redirect: "/users/active/",
1190                         expect:   []string{"foo_file_in_dir/"},
1191                         cutDirs:  2,
1192                 },
1193                 {
1194                         uri:     "download.example.com/users/active/",
1195                         header:  authHeader,
1196                         expect:  []string{"foo_file_in_dir/"},
1197                         cutDirs: 2,
1198                 },
1199                 {
1200                         uri:     "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
1201                         header:  nil,
1202                         expect:  []string{"dir1/foo", "dir1/bar"},
1203                         cutDirs: 4,
1204                 },
1205                 {
1206                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
1207                         header:  nil,
1208                         expect:  []string{"dir1/foo", "dir1/bar"},
1209                         cutDirs: 2,
1210                 },
1211                 {
1212                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken,
1213                         header:  nil,
1214                         expect:  []string{"dir1/foo", "dir1/bar"},
1215                         cutDirs: 2,
1216                 },
1217                 {
1218                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID,
1219                         header:  authHeader,
1220                         expect:  []string{"dir1/foo", "dir1/bar"},
1221                         cutDirs: 1,
1222                 },
1223                 {
1224                         uri:      "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
1225                         header:   authHeader,
1226                         redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
1227                         expect:   []string{"foo", "bar"},
1228                         cutDirs:  2,
1229                 },
1230                 {
1231                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
1232                         header:  authHeader,
1233                         expect:  []string{"foo", "bar"},
1234                         cutDirs: 3,
1235                 },
1236                 {
1237                         uri:      arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
1238                         header:   authHeader,
1239                         redirect: "/dir1/",
1240                         expect:   []string{"foo", "bar"},
1241                         cutDirs:  1,
1242                 },
1243                 {
1244                         uri:    "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
1245                         header: authHeader,
1246                         expect: nil,
1247                 },
1248                 {
1249                         uri:     "download.example.com/c=" + arvadostest.WazVersion1Collection,
1250                         header:  authHeader,
1251                         expect:  []string{"waz"},
1252                         cutDirs: 1,
1253                 },
1254                 {
1255                         uri:     "download.example.com/by_id/" + arvadostest.WazVersion1Collection,
1256                         header:  authHeader,
1257                         expect:  []string{"waz"},
1258                         cutDirs: 2,
1259                 },
1260                 {
1261                         uri:     "download.example.com/users/active/This filter group/",
1262                         header:  authHeader,
1263                         expect:  []string{"A Subproject/"},
1264                         cutDirs: 3,
1265                 },
1266                 {
1267                         uri:     "download.example.com/users/active/This filter group/A Subproject",
1268                         header:  authHeader,
1269                         expect:  []string{"baz_file/"},
1270                         cutDirs: 4,
1271                 },
1272                 {
1273                         uri:     "download.example.com/by_id/" + arvadostest.AFilterGroupUUID,
1274                         header:  authHeader,
1275                         expect:  []string{"A Subproject/"},
1276                         cutDirs: 2,
1277                 },
1278                 {
1279                         uri:     "download.example.com/by_id/" + arvadostest.AFilterGroupUUID + "/A Subproject",
1280                         header:  authHeader,
1281                         expect:  []string{"baz_file/"},
1282                         cutDirs: 3,
1283                 },
1284         } {
1285                 comment := check.Commentf("HTML: %q redir %q => %q", trial.uri, trial.redirect, trial.expect)
1286                 resp := httptest.NewRecorder()
1287                 u := mustParseURL("//" + trial.uri)
1288                 req := &http.Request{
1289                         Method:     "GET",
1290                         Host:       u.Host,
1291                         URL:        u,
1292                         RequestURI: u.RequestURI(),
1293                         Header:     copyHeader(trial.header),
1294                 }
1295                 s.handler.ServeHTTP(resp, req)
1296                 var cookies []*http.Cookie
1297                 for resp.Code == http.StatusSeeOther {
1298                         u, _ := req.URL.Parse(resp.Header().Get("Location"))
1299                         req = &http.Request{
1300                                 Method:     "GET",
1301                                 Host:       u.Host,
1302                                 URL:        u,
1303                                 RequestURI: u.RequestURI(),
1304                                 Header:     copyHeader(trial.header),
1305                         }
1306                         cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
1307                         for _, c := range cookies {
1308                                 req.AddCookie(c)
1309                         }
1310                         resp = httptest.NewRecorder()
1311                         s.handler.ServeHTTP(resp, req)
1312                 }
1313                 if trial.redirect != "" {
1314                         c.Check(req.URL.Path, check.Equals, trial.redirect, comment)
1315                 }
1316                 if trial.expect == nil {
1317                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1318                 } else {
1319                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
1320                         for _, e := range trial.expect {
1321                                 e = strings.Replace(e, " ", "%20", -1)
1322                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`, comment)
1323                         }
1324                         c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`, comment)
1325                 }
1326
1327                 comment = check.Commentf("WebDAV: %q => %q", trial.uri, trial.expect)
1328                 req = &http.Request{
1329                         Method:     "OPTIONS",
1330                         Host:       u.Host,
1331                         URL:        u,
1332                         RequestURI: u.RequestURI(),
1333                         Header:     copyHeader(trial.header),
1334                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
1335                 }
1336                 resp = httptest.NewRecorder()
1337                 s.handler.ServeHTTP(resp, req)
1338                 if trial.expect == nil {
1339                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1340                 } else {
1341                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
1342                 }
1343
1344                 req = &http.Request{
1345                         Method:     "PROPFIND",
1346                         Host:       u.Host,
1347                         URL:        u,
1348                         RequestURI: u.RequestURI(),
1349                         Header:     copyHeader(trial.header),
1350                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
1351                 }
1352                 resp = httptest.NewRecorder()
1353                 s.handler.ServeHTTP(resp, req)
1354                 // This check avoids logging a big XML document in the
1355                 // event webdav throws a 500 error after sending
1356                 // headers for a 207.
1357                 if !c.Check(strings.HasSuffix(resp.Body.String(), "Internal Server Error"), check.Equals, false) {
1358                         continue
1359                 }
1360                 if trial.expect == nil {
1361                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1362                 } else {
1363                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment)
1364                         for _, e := range trial.expect {
1365                                 if strings.HasSuffix(e, "/") {
1366                                         e = filepath.Join(u.Path, e) + "/"
1367                                 } else {
1368                                         e = filepath.Join(u.Path, e)
1369                                 }
1370                                 e = strings.Replace(e, " ", "%20", -1)
1371                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+e+`</D:href>.*`, comment)
1372                         }
1373                 }
1374         }
1375 }
1376
1377 func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
1378         arv := arvados.NewClientFromEnv()
1379         var newCollection arvados.Collection
1380         err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1381                 "collection": map[string]string{
1382                         "owner_uuid":    arvadostest.ActiveUserUUID,
1383                         "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
1384                         "name":          "keep-web test collection",
1385                 },
1386                 "ensure_unique_name": true,
1387         })
1388         c.Assert(err, check.IsNil)
1389         defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
1390
1391         var updated arvados.Collection
1392         for _, fnm := range []string{"foo.txt", "bar.txt"} {
1393                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
1394                 u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
1395                 req := &http.Request{
1396                         Method:     "DELETE",
1397                         Host:       u.Host,
1398                         URL:        u,
1399                         RequestURI: u.RequestURI(),
1400                         Header: http.Header{
1401                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
1402                         },
1403                 }
1404                 resp := httptest.NewRecorder()
1405                 s.handler.ServeHTTP(resp, req)
1406                 c.Check(resp.Code, check.Equals, http.StatusNoContent)
1407
1408                 updated = arvados.Collection{}
1409                 err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
1410                 c.Check(err, check.IsNil)
1411                 c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`)
1412                 c.Logf("updated manifest_text %q", updated.ManifestText)
1413         }
1414         c.Check(updated.ManifestText, check.Equals, "")
1415 }
1416
1417 func (s *IntegrationSuite) TestFileContentType(c *check.C) {
1418         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1419
1420         client := arvados.NewClientFromEnv()
1421         client.AuthToken = arvadostest.ActiveToken
1422         arv, err := arvadosclient.New(client)
1423         c.Assert(err, check.Equals, nil)
1424         kc, err := keepclient.MakeKeepClient(arv)
1425         c.Assert(err, check.Equals, nil)
1426
1427         fs, err := (&arvados.Collection{}).FileSystem(client, kc)
1428         c.Assert(err, check.IsNil)
1429
1430         trials := []struct {
1431                 filename    string
1432                 content     string
1433                 contentType string
1434         }{
1435                 {"picture.txt", "BMX bikes are small this year\n", "text/plain; charset=utf-8"},
1436                 {"picture.bmp", "BMX bikes are small this year\n", "image/(x-ms-)?bmp"},
1437                 {"picture.jpg", "BMX bikes are small this year\n", "image/jpeg"},
1438                 {"picture1", "BMX bikes are small this year\n", "image/bmp"},            // content sniff; "BM" is the magic signature for .bmp
1439                 {"picture2", "Cars are small this year\n", "text/plain; charset=utf-8"}, // content sniff
1440         }
1441         for _, trial := range trials {
1442                 f, err := fs.OpenFile(trial.filename, os.O_CREATE|os.O_WRONLY, 0777)
1443                 c.Assert(err, check.IsNil)
1444                 _, err = f.Write([]byte(trial.content))
1445                 c.Assert(err, check.IsNil)
1446                 c.Assert(f.Close(), check.IsNil)
1447         }
1448         mtxt, err := fs.MarshalManifest(".")
1449         c.Assert(err, check.IsNil)
1450         var coll arvados.Collection
1451         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1452                 "collection": map[string]string{
1453                         "manifest_text": mtxt,
1454                 },
1455         })
1456         c.Assert(err, check.IsNil)
1457
1458         for _, trial := range trials {
1459                 u, _ := url.Parse("http://download.example.com/by_id/" + coll.UUID + "/" + trial.filename)
1460                 req := &http.Request{
1461                         Method:     "GET",
1462                         Host:       u.Host,
1463                         URL:        u,
1464                         RequestURI: u.RequestURI(),
1465                         Header: http.Header{
1466                                 "Authorization": {"Bearer " + client.AuthToken},
1467                         },
1468                 }
1469                 resp := httptest.NewRecorder()
1470                 s.handler.ServeHTTP(resp, req)
1471                 c.Check(resp.Code, check.Equals, http.StatusOK)
1472                 c.Check(resp.Header().Get("Content-Type"), check.Matches, trial.contentType)
1473                 c.Check(resp.Body.String(), check.Equals, trial.content)
1474         }
1475 }
1476
1477 func (s *IntegrationSuite) TestCacheSize(c *check.C) {
1478         req, err := http.NewRequest("GET", "http://"+arvadostest.FooCollection+".example.com/foo", nil)
1479         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
1480         c.Assert(err, check.IsNil)
1481         resp := httptest.NewRecorder()
1482         s.handler.ServeHTTP(resp, req)
1483         c.Assert(resp.Code, check.Equals, http.StatusOK)
1484         c.Check(s.handler.Cache.sessions[arvadostest.ActiveTokenV2].client.DiskCacheSize.Percent(), check.Equals, int64(10))
1485 }
1486
1487 // Writing to a collection shouldn't affect its entry in the
1488 // PDH-to-manifest cache.
1489 func (s *IntegrationSuite) TestCacheWriteCollectionSamePDH(c *check.C) {
1490         arv, err := arvadosclient.MakeArvadosClient()
1491         c.Assert(err, check.Equals, nil)
1492         arv.ApiToken = arvadostest.ActiveToken
1493
1494         u := mustParseURL("http://x.example/testfile")
1495         req := &http.Request{
1496                 Method:     "GET",
1497                 Host:       u.Host,
1498                 URL:        u,
1499                 RequestURI: u.RequestURI(),
1500                 Header:     http.Header{"Authorization": {"Bearer " + arv.ApiToken}},
1501         }
1502
1503         checkWithID := func(id string, status int) {
1504                 req.URL.Host = strings.Replace(id, "+", "-", -1) + ".example"
1505                 req.Host = req.URL.Host
1506                 resp := httptest.NewRecorder()
1507                 s.handler.ServeHTTP(resp, req)
1508                 c.Check(resp.Code, check.Equals, status)
1509         }
1510
1511         var colls [2]arvados.Collection
1512         for i := range colls {
1513                 err := arv.Create("collections",
1514                         map[string]interface{}{
1515                                 "ensure_unique_name": true,
1516                                 "collection": map[string]interface{}{
1517                                         "name": "test collection",
1518                                 },
1519                         }, &colls[i])
1520                 c.Assert(err, check.Equals, nil)
1521         }
1522
1523         // Populate cache with empty collection
1524         checkWithID(colls[0].PortableDataHash, http.StatusNotFound)
1525
1526         // write a file to colls[0]
1527         reqPut := *req
1528         reqPut.Method = "PUT"
1529         reqPut.URL.Host = colls[0].UUID + ".example"
1530         reqPut.Host = req.URL.Host
1531         reqPut.Body = ioutil.NopCloser(bytes.NewBufferString("testdata"))
1532         resp := httptest.NewRecorder()
1533         s.handler.ServeHTTP(resp, &reqPut)
1534         c.Check(resp.Code, check.Equals, http.StatusCreated)
1535
1536         // new file should not appear in colls[1]
1537         checkWithID(colls[1].PortableDataHash, http.StatusNotFound)
1538         checkWithID(colls[1].UUID, http.StatusNotFound)
1539
1540         checkWithID(colls[0].UUID, http.StatusOK)
1541 }
1542
1543 func copyHeader(h http.Header) http.Header {
1544         hc := http.Header{}
1545         for k, v := range h {
1546                 hc[k] = append([]string(nil), v...)
1547         }
1548         return hc
1549 }
1550
1551 func (s *IntegrationSuite) checkUploadDownloadRequest(c *check.C, req *http.Request,
1552         successCode int, direction string, perm bool, userUuid, collectionUuid, collectionPDH, filepath string) {
1553
1554         client := arvados.NewClientFromEnv()
1555         client.AuthToken = arvadostest.AdminToken
1556         var logentries arvados.LogList
1557         limit1 := 1
1558         err := client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil,
1559                 arvados.ResourceListParams{
1560                         Limit: &limit1,
1561                         Order: "created_at desc"})
1562         c.Check(err, check.IsNil)
1563         c.Check(logentries.Items, check.HasLen, 1)
1564         lastLogId := logentries.Items[0].ID
1565         c.Logf("lastLogId: %d", lastLogId)
1566
1567         var logbuf bytes.Buffer
1568         logger := logrus.New()
1569         logger.Out = &logbuf
1570         resp := httptest.NewRecorder()
1571         req = req.WithContext(ctxlog.Context(context.Background(), logger))
1572         s.handler.ServeHTTP(resp, req)
1573
1574         if perm {
1575                 c.Check(resp.Result().StatusCode, check.Equals, successCode)
1576                 c.Check(logbuf.String(), check.Matches, `(?ms).*msg="File `+direction+`".*`)
1577                 c.Check(logbuf.String(), check.Not(check.Matches), `(?ms).*level=error.*`)
1578
1579                 deadline := time.Now().Add(time.Second)
1580                 for {
1581                         c.Assert(time.Now().After(deadline), check.Equals, false, check.Commentf("timed out waiting for log entry"))
1582                         logentries = arvados.LogList{}
1583                         err = client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil,
1584                                 arvados.ResourceListParams{
1585                                         Filters: []arvados.Filter{
1586                                                 {Attr: "event_type", Operator: "=", Operand: "file_" + direction},
1587                                                 {Attr: "object_uuid", Operator: "=", Operand: userUuid},
1588                                         },
1589                                         Limit: &limit1,
1590                                         Order: "created_at desc",
1591                                 })
1592                         c.Assert(err, check.IsNil)
1593                         if len(logentries.Items) > 0 &&
1594                                 logentries.Items[0].ID > lastLogId &&
1595                                 logentries.Items[0].ObjectUUID == userUuid &&
1596                                 logentries.Items[0].Properties["collection_uuid"] == collectionUuid &&
1597                                 (collectionPDH == "" || logentries.Items[0].Properties["portable_data_hash"] == collectionPDH) &&
1598                                 logentries.Items[0].Properties["collection_file_path"] == filepath {
1599                                 break
1600                         }
1601                         c.Logf("logentries.Items: %+v", logentries.Items)
1602                         time.Sleep(50 * time.Millisecond)
1603                 }
1604         } else {
1605                 c.Check(resp.Result().StatusCode, check.Equals, http.StatusForbidden)
1606                 c.Check(logbuf.String(), check.Equals, "")
1607         }
1608 }
1609
1610 func (s *IntegrationSuite) TestDownloadLoggingPermission(c *check.C) {
1611         u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
1612
1613         s.handler.Cluster.Collections.TrustAllContent = true
1614
1615         for _, adminperm := range []bool{true, false} {
1616                 for _, userperm := range []bool{true, false} {
1617                         s.handler.Cluster.Collections.WebDAVPermission.Admin.Download = adminperm
1618                         s.handler.Cluster.Collections.WebDAVPermission.User.Download = userperm
1619
1620                         // Test admin permission
1621                         req := &http.Request{
1622                                 Method:     "GET",
1623                                 Host:       u.Host,
1624                                 URL:        u,
1625                                 RequestURI: u.RequestURI(),
1626                                 Header: http.Header{
1627                                         "Authorization": {"Bearer " + arvadostest.AdminToken},
1628                                 },
1629                         }
1630                         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", adminperm,
1631                                 arvadostest.AdminUserUUID, arvadostest.FooCollection, arvadostest.FooCollectionPDH, "foo")
1632
1633                         // Test user permission
1634                         req = &http.Request{
1635                                 Method:     "GET",
1636                                 Host:       u.Host,
1637                                 URL:        u,
1638                                 RequestURI: u.RequestURI(),
1639                                 Header: http.Header{
1640                                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
1641                                 },
1642                         }
1643                         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", userperm,
1644                                 arvadostest.ActiveUserUUID, arvadostest.FooCollection, arvadostest.FooCollectionPDH, "foo")
1645                 }
1646         }
1647
1648         s.handler.Cluster.Collections.WebDAVPermission.User.Download = true
1649
1650         for _, tryurl := range []string{"http://" + arvadostest.MultilevelCollection1 + ".keep-web.example/dir1/subdir/file1",
1651                 "http://keep-web/users/active/multilevel_collection_1/dir1/subdir/file1"} {
1652
1653                 u = mustParseURL(tryurl)
1654                 req := &http.Request{
1655                         Method:     "GET",
1656                         Host:       u.Host,
1657                         URL:        u,
1658                         RequestURI: u.RequestURI(),
1659                         Header: http.Header{
1660                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
1661                         },
1662                 }
1663                 s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true,
1664                         arvadostest.ActiveUserUUID, arvadostest.MultilevelCollection1, arvadostest.MultilevelCollection1PDH, "dir1/subdir/file1")
1665         }
1666
1667         u = mustParseURL("http://" + strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + ".keep-web.example/foo")
1668         req := &http.Request{
1669                 Method:     "GET",
1670                 Host:       u.Host,
1671                 URL:        u,
1672                 RequestURI: u.RequestURI(),
1673                 Header: http.Header{
1674                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
1675                 },
1676         }
1677         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true,
1678                 arvadostest.ActiveUserUUID, "", arvadostest.FooCollectionPDH, "foo")
1679 }
1680
1681 func (s *IntegrationSuite) TestUploadLoggingPermission(c *check.C) {
1682         for _, adminperm := range []bool{true, false} {
1683                 for _, userperm := range []bool{true, false} {
1684
1685                         arv := arvados.NewClientFromEnv()
1686                         arv.AuthToken = arvadostest.ActiveToken
1687
1688                         var coll arvados.Collection
1689                         err := arv.RequestAndDecode(&coll,
1690                                 "POST",
1691                                 "/arvados/v1/collections",
1692                                 nil,
1693                                 map[string]interface{}{
1694                                         "ensure_unique_name": true,
1695                                         "collection": map[string]interface{}{
1696                                                 "name": "test collection",
1697                                         },
1698                                 })
1699                         c.Assert(err, check.Equals, nil)
1700
1701                         u := mustParseURL("http://" + coll.UUID + ".keep-web.example/bar")
1702
1703                         s.handler.Cluster.Collections.WebDAVPermission.Admin.Upload = adminperm
1704                         s.handler.Cluster.Collections.WebDAVPermission.User.Upload = userperm
1705
1706                         // Test admin permission
1707                         req := &http.Request{
1708                                 Method:     "PUT",
1709                                 Host:       u.Host,
1710                                 URL:        u,
1711                                 RequestURI: u.RequestURI(),
1712                                 Header: http.Header{
1713                                         "Authorization": {"Bearer " + arvadostest.AdminToken},
1714                                 },
1715                                 Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
1716                         }
1717                         s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", adminperm,
1718                                 arvadostest.AdminUserUUID, coll.UUID, "", "bar")
1719
1720                         // Test user permission
1721                         req = &http.Request{
1722                                 Method:     "PUT",
1723                                 Host:       u.Host,
1724                                 URL:        u,
1725                                 RequestURI: u.RequestURI(),
1726                                 Header: http.Header{
1727                                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
1728                                 },
1729                                 Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
1730                         }
1731                         s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", userperm,
1732                                 arvadostest.ActiveUserUUID, coll.UUID, "", "bar")
1733                 }
1734         }
1735 }
1736
1737 func (s *IntegrationSuite) TestConcurrentWrites(c *check.C) {
1738         s.handler.Cluster.Collections.WebDAVCache.TTL = arvados.Duration(time.Second * 2)
1739         lockTidyInterval = time.Second
1740         client := arvados.NewClientFromEnv()
1741         client.AuthToken = arvadostest.ActiveTokenV2
1742         // Start small, and increase concurrency (2^2, 4^2, ...)
1743         // only until hitting failure. Avoids unnecessarily long
1744         // failure reports.
1745         for n := 2; n < 16 && !c.Failed(); n = n * 2 {
1746                 c.Logf("%s: n=%d", c.TestName(), n)
1747
1748                 var coll arvados.Collection
1749                 err := client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, nil)
1750                 c.Assert(err, check.IsNil)
1751                 defer client.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
1752
1753                 var wg sync.WaitGroup
1754                 for i := 0; i < n && !c.Failed(); i++ {
1755                         i := i
1756                         wg.Add(1)
1757                         go func() {
1758                                 defer wg.Done()
1759                                 u := mustParseURL(fmt.Sprintf("http://%s.collections.example.com/i=%d", coll.UUID, i))
1760                                 resp := httptest.NewRecorder()
1761                                 req, err := http.NewRequest("MKCOL", u.String(), nil)
1762                                 c.Assert(err, check.IsNil)
1763                                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
1764                                 s.handler.ServeHTTP(resp, req)
1765                                 c.Assert(resp.Code, check.Equals, http.StatusCreated)
1766                                 for j := 0; j < n && !c.Failed(); j++ {
1767                                         j := j
1768                                         wg.Add(1)
1769                                         go func() {
1770                                                 defer wg.Done()
1771                                                 content := fmt.Sprintf("i=%d/j=%d", i, j)
1772                                                 u := mustParseURL("http://" + coll.UUID + ".collections.example.com/" + content)
1773
1774                                                 resp := httptest.NewRecorder()
1775                                                 req, err := http.NewRequest("PUT", u.String(), strings.NewReader(content))
1776                                                 c.Assert(err, check.IsNil)
1777                                                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
1778                                                 s.handler.ServeHTTP(resp, req)
1779                                                 c.Check(resp.Code, check.Equals, http.StatusCreated)
1780
1781                                                 time.Sleep(time.Second)
1782                                                 resp = httptest.NewRecorder()
1783                                                 req, err = http.NewRequest("GET", u.String(), nil)
1784                                                 c.Assert(err, check.IsNil)
1785                                                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
1786                                                 s.handler.ServeHTTP(resp, req)
1787                                                 c.Check(resp.Code, check.Equals, http.StatusOK)
1788                                                 c.Check(resp.Body.String(), check.Equals, content)
1789                                         }()
1790                                 }
1791                         }()
1792                 }
1793                 wg.Wait()
1794                 for i := 0; i < n; i++ {
1795                         u := mustParseURL(fmt.Sprintf("http://%s.collections.example.com/i=%d", coll.UUID, i))
1796                         resp := httptest.NewRecorder()
1797                         req, err := http.NewRequest("PROPFIND", u.String(), &bytes.Buffer{})
1798                         c.Assert(err, check.IsNil)
1799                         req.Header.Set("Authorization", "Bearer "+client.AuthToken)
1800                         s.handler.ServeHTTP(resp, req)
1801                         c.Assert(resp.Code, check.Equals, http.StatusMultiStatus)
1802                 }
1803         }
1804 }