services/keep-web/handler_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package keepweb
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "fmt"
  11         "html"
  12         "io"
  13         "io/ioutil"
  14         "net/http"
  15         "net/http/httptest"
  16         "net/url"
  17         "os"
  18         "path/filepath"
  19         "regexp"
  20         "strings"
  21         "sync"
  22         "time"
  23
  24         "git.arvados.org/arvados.git/lib/config"
  25         "git.arvados.org/arvados.git/sdk/go/arvados"
  26         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  27         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  28         "git.arvados.org/arvados.git/sdk/go/auth"
  29         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  30         "git.arvados.org/arvados.git/sdk/go/keepclient"
  31         "github.com/prometheus/client_golang/prometheus"
  32         "github.com/sirupsen/logrus"
  33         check "gopkg.in/check.v1"
  34 )
  35
  36 var _ = check.Suite(&UnitSuite{})
  37
  38 func init() {
  39         arvados.DebugLocksPanicMode = true
  40 }
  41
  42 type UnitSuite struct {
  43         cluster *arvados.Cluster
  44         handler *handler
  45 }
  46
  47 func (s *UnitSuite) SetUpTest(c *check.C) {
  48         logger := ctxlog.TestLogger(c)
  49         ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), logger)
  50         ldr.Path = "-"
  51         cfg, err := ldr.Load()
  52         c.Assert(err, check.IsNil)
  53         cc, err := cfg.GetCluster("")
  54         c.Assert(err, check.IsNil)
  55         s.cluster = cc
  56         s.handler = &handler{
  57                 Cluster: cc,
  58                 Cache: cache{
  59                         cluster:  cc,
  60                         logger:   logger,
  61                         registry: prometheus.NewRegistry(),
  62                 },
  63         }
  64 }
  65
  66 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
  67         h := s.handler
  68         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
  69         req := &http.Request{
  70                 Method:     "OPTIONS",
  71                 Host:       u.Host,
  72                 URL:        u,
  73                 RequestURI: u.RequestURI(),
  74                 Header: http.Header{
  75                         "Origin":                        {"https://workbench.example"},
  76                         "Access-Control-Request-Method": {"POST"},
  77                 },
  78         }
  79
  80         // Check preflight for an allowed request
  81         resp := httptest.NewRecorder()
  82         h.ServeHTTP(resp, req)
  83         c.Check(resp.Code, check.Equals, http.StatusOK)
  84         c.Check(resp.Body.String(), check.Equals, "")
  85         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
  86         c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
  87         c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout, Cache-Control")
  88
  89         // Check preflight for a disallowed request
  90         resp = httptest.NewRecorder()
  91         req.Header.Set("Access-Control-Request-Method", "MAKE-COFFEE")
  92         h.ServeHTTP(resp, req)
  93         c.Check(resp.Body.String(), check.Equals, "")
  94         c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
  95 }
  96
  97 func (s *UnitSuite) TestWebdavPrefixAndSource(c *check.C) {
  98         for _, trial := range []struct {
  99                 method   string
 100                 path     string
 101                 prefix   string
 102                 source   string
 103                 notFound bool
 104                 seeOther bool
 105         }{
 106                 {
 107                         method: "PROPFIND",
 108                         path:   "/",
 109                 },
 110                 {
 111                         method: "PROPFIND",
 112                         path:   "/dir1",
 113                 },
 114                 {
 115                         method: "PROPFIND",
 116                         path:   "/dir1/",
 117                 },
 118                 {
 119                         method: "PROPFIND",
 120                         path:   "/dir1/foo",
 121                         prefix: "/dir1",
 122                         source: "/dir1",
 123                 },
 124                 {
 125                         method: "PROPFIND",
 126                         path:   "/prefix/dir1/foo",
 127                         prefix: "/prefix/",
 128                         source: "",
 129                 },
 130                 {
 131                         method: "PROPFIND",
 132                         path:   "/prefix/dir1/foo",
 133                         prefix: "/prefix",
 134                         source: "",
 135                 },
 136                 {
 137                         method: "PROPFIND",
 138                         path:   "/prefix/dir1/foo",
 139                         prefix: "/prefix/",
 140                         source: "/",
 141                 },
 142                 {
 143                         method: "PROPFIND",
 144                         path:   "/prefix/foo",
 145                         prefix: "/prefix/",
 146                         source: "/dir1/",
 147                 },
 148                 {
 149                         method: "GET",
 150                         path:   "/prefix/foo",
 151                         prefix: "/prefix/",
 152                         source: "/dir1/",
 153                 },
 154                 {
 155                         method: "PROPFIND",
 156                         path:   "/prefix/",
 157                         prefix: "/prefix",
 158                         source: "/dir1",
 159                 },
 160                 {
 161                         method: "PROPFIND",
 162                         path:   "/prefix",
 163                         prefix: "/prefix",
 164                         source: "/dir1/",
 165                 },
 166                 {
 167                         method:   "GET",
 168                         path:     "/prefix",
 169                         prefix:   "/prefix",
 170                         source:   "/dir1",
 171                         seeOther: true,
 172                 },
 173                 {
 174                         method:   "PROPFIND",
 175                         path:     "/dir1/foo",
 176                         prefix:   "",
 177                         source:   "/dir1",
 178                         notFound: true,
 179                 },
 180         } {
 181                 c.Logf("trial %+v", trial)
 182                 u := mustParseURL("http://" + arvadostest.FooBarDirCollection + ".keep-web.example" + trial.path)
 183                 req := &http.Request{
 184                         Method:     trial.method,
 185                         Host:       u.Host,
 186                         URL:        u,
 187                         RequestURI: u.RequestURI(),
 188                         Header: http.Header{
 189                                 "Authorization":   {"Bearer " + arvadostest.ActiveTokenV2},
 190                                 "X-Webdav-Prefix": {trial.prefix},
 191                                 "X-Webdav-Source": {trial.source},
 192                         },
 193                         Body: ioutil.NopCloser(bytes.NewReader(nil)),
 194                 }
 195
 196                 resp := httptest.NewRecorder()
 197                 s.handler.ServeHTTP(resp, req)
 198                 if trial.notFound {
 199                         c.Check(resp.Code, check.Equals, http.StatusNotFound)
 200                 } else if trial.method == "PROPFIND" {
 201                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus)
 202                         c.Check(resp.Body.String(), check.Matches, `(?ms).*>\n?$`)
 203                 } else if trial.seeOther {
 204                         c.Check(resp.Code, check.Equals, http.StatusSeeOther)
 205                 } else {
 206                         c.Check(resp.Code, check.Equals, http.StatusOK)
 207                 }
 208         }
 209 }
 210
 211 func (s *UnitSuite) TestEmptyResponse(c *check.C) {
 212         for _, trial := range []struct {
 213                 dataExists    bool
 214                 sendIMSHeader bool
 215                 expectStatus  int
 216                 logRegexp     string
 217         }{
 218                 // If we return no content due to a Keep read error,
 219                 // we should emit a log message.
 220                 {false, false, http.StatusOK, `(?ms).*only wrote 0 bytes.*`},
 221
 222                 // If we return no content because the client sent an
 223                 // If-Modified-Since header, our response should be
 224                 // 304.  We still expect a "File download" log since it
 225                 // counts as a file access for auditing.
 226                 {true, true, http.StatusNotModified, `(?ms).*msg="File download".*`},
 227         } {
 228                 c.Logf("trial: %+v", trial)
 229                 arvadostest.StartKeep(2, true)
 230                 if trial.dataExists {
 231                         arv, err := arvadosclient.MakeArvadosClient()
 232                         c.Assert(err, check.IsNil)
 233                         arv.ApiToken = arvadostest.ActiveToken
 234                         kc, err := keepclient.MakeKeepClient(arv)
 235                         c.Assert(err, check.IsNil)
 236                         _, _, err = kc.PutB([]byte("foo"))
 237                         c.Assert(err, check.IsNil)
 238                 }
 239
 240                 u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
 241                 req := &http.Request{
 242                         Method:     "GET",
 243                         Host:       u.Host,
 244                         URL:        u,
 245                         RequestURI: u.RequestURI(),
 246                         Header: http.Header{
 247                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
 248                         },
 249                 }
 250                 if trial.sendIMSHeader {
 251                         req.Header.Set("If-Modified-Since", strings.Replace(time.Now().UTC().Format(time.RFC1123), "UTC", "GMT", -1))
 252                 }
 253
 254                 var logbuf bytes.Buffer
 255                 logger := logrus.New()
 256                 logger.Out = &logbuf
 257                 req = req.WithContext(ctxlog.Context(context.Background(), logger))
 258
 259                 resp := httptest.NewRecorder()
 260                 s.handler.ServeHTTP(resp, req)
 261                 c.Check(resp.Code, check.Equals, trial.expectStatus)
 262                 c.Check(resp.Body.String(), check.Equals, "")
 263
 264                 c.Log(logbuf.String())
 265                 c.Check(logbuf.String(), check.Matches, trial.logRegexp)
 266         }
 267 }
 268
 269 func (s *UnitSuite) TestInvalidUUID(c *check.C) {
 270         bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-"
 271         token := arvadostest.ActiveToken
 272         for _, trial := range []string{
 273                 "http://keep-web/c=" + bogusID + "/foo",
 274                 "http://keep-web/c=" + bogusID + "/t=" + token + "/foo",
 275                 "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo",
 276                 "http://keep-web/collections/" + bogusID + "/foo",
 277                 "http://" + bogusID + ".keep-web/" + bogusID + "/foo",
 278                 "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
 279         } {
 280                 c.Log(trial)
 281                 u := mustParseURL(trial)
 282                 req := &http.Request{
 283                         Method:     "GET",
 284                         Host:       u.Host,
 285                         URL:        u,
 286                         RequestURI: u.RequestURI(),
 287                 }
 288                 resp := httptest.NewRecorder()
 289                 s.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 290                 s.handler.ServeHTTP(resp, req)
 291                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 292         }
 293 }
 294
 295 func mustParseURL(s string) *url.URL {
 296         r, err := url.Parse(s)
 297         if err != nil {
 298                 panic("parse URL: " + s)
 299         }
 300         return r
 301 }
 302
 303 func (s *IntegrationSuite) TestVhost404(c *check.C) {
 304         for _, testURL := range []string{
 305                 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
 306                 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
 307         } {
 308                 resp := httptest.NewRecorder()
 309                 u := mustParseURL(testURL)
 310                 req := &http.Request{
 311                         Method:     "GET",
 312                         URL:        u,
 313                         RequestURI: u.RequestURI(),
 314                 }
 315                 s.handler.ServeHTTP(resp, req)
 316                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 317                 c.Check(resp.Body.String(), check.Equals, notFoundMessage+"\n")
 318         }
 319 }
 320
 321 // An authorizer modifies an HTTP request to make use of the given
 322 // token -- by adding it to a header, cookie, query param, or whatever
 323 // -- and returns the HTTP status code we should expect from keep-web if
 324 // the token is invalid.
 325 type authorizer func(*http.Request, string) int
 326
 327 func (s *IntegrationSuite) TestVhostViaAuthzHeaderOAuth2(c *check.C) {
 328         s.doVhostRequests(c, authzViaAuthzHeaderOAuth2)
 329 }
 330 func authzViaAuthzHeaderOAuth2(r *http.Request, tok string) int {
 331         r.Header.Add("Authorization", "Bearer "+tok)
 332         return http.StatusUnauthorized
 333 }
 334 func (s *IntegrationSuite) TestVhostViaAuthzHeaderBearer(c *check.C) {
 335         s.doVhostRequests(c, authzViaAuthzHeaderBearer)
 336 }
 337 func authzViaAuthzHeaderBearer(r *http.Request, tok string) int {
 338         r.Header.Add("Authorization", "Bearer "+tok)
 339         return http.StatusUnauthorized
 340 }
 341
 342 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
 343         s.doVhostRequests(c, authzViaCookieValue)
 344 }
 345 func authzViaCookieValue(r *http.Request, tok string) int {
 346         r.AddCookie(&http.Cookie{
 347                 Name:  "arvados_api_token",
 348                 Value: auth.EncodeTokenCookie([]byte(tok)),
 349         })
 350         return http.StatusUnauthorized
 351 }
 352
 353 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
 354         s.doVhostRequests(c, authzViaPath)
 355 }
 356 func authzViaPath(r *http.Request, tok string) int {
 357         r.URL.Path = "/t=" + tok + r.URL.Path
 358         return http.StatusNotFound
 359 }
 360
 361 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
 362         s.doVhostRequests(c, authzViaQueryString)
 363 }
 364 func authzViaQueryString(r *http.Request, tok string) int {
 365         r.URL.RawQuery = "api_token=" + tok
 366         return http.StatusUnauthorized
 367 }
 368
 369 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
 370         s.doVhostRequests(c, authzViaPOST)
 371 }
 372 func authzViaPOST(r *http.Request, tok string) int {
 373         r.Method = "POST"
 374         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 375         r.Body = ioutil.NopCloser(strings.NewReader(
 376                 url.Values{"api_token": {tok}}.Encode()))
 377         return http.StatusUnauthorized
 378 }
 379
 380 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
 381         s.doVhostRequests(c, authzViaPOST)
 382 }
 383 func authzViaXHRPOST(r *http.Request, tok string) int {
 384         r.Method = "POST"
 385         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 386         r.Header.Add("Origin", "https://origin.example")
 387         r.Body = ioutil.NopCloser(strings.NewReader(
 388                 url.Values{
 389                         "api_token":   {tok},
 390                         "disposition": {"attachment"},
 391                 }.Encode()))
 392         return http.StatusUnauthorized
 393 }
 394
 395 // Try some combinations of {url, token} using the given authorization
 396 // mechanism, and verify the result is correct.
 397 func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
 398         for _, hostPath := range []string{
 399                 arvadostest.FooCollection + ".example.com/foo",
 400                 arvadostest.FooCollection + "--collections.example.com/foo",
 401                 arvadostest.FooCollection + "--collections.example.com/_/foo",
 402                 arvadostest.FooCollectionPDH + ".example.com/foo",
 403                 strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + "--collections.example.com/foo",
 404                 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
 405         } {
 406                 c.Log("doRequests: ", hostPath)
 407                 s.doVhostRequestsWithHostPath(c, authz, hostPath)
 408         }
 409 }
 410
 411 func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
 412         for _, tok := range []string{
 413                 arvadostest.ActiveToken,
 414                 arvadostest.ActiveToken[:15],
 415                 arvadostest.SpectatorToken,
 416                 "bogus",
 417                 "",
 418         } {
 419                 u := mustParseURL("http://" + hostPath)
 420                 req := &http.Request{
 421                         Method:     "GET",
 422                         Host:       u.Host,
 423                         URL:        u,
 424                         RequestURI: u.RequestURI(),
 425                         Header:     http.Header{},
 426                 }
 427                 failCode := authz(req, tok)
 428                 req, resp := s.doReq(req)
 429                 code, body := resp.Code, resp.Body.String()
 430
 431                 // If the initial request had a (non-empty) token
 432                 // showing in the query string, we should have been
 433                 // redirected in order to hide it in a cookie.
 434                 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
 435
 436                 if tok == arvadostest.ActiveToken {
 437                         c.Check(code, check.Equals, http.StatusOK)
 438                         c.Check(body, check.Equals, "foo")
 439                 } else {
 440                         c.Check(code >= 400, check.Equals, true)
 441                         c.Check(code < 500, check.Equals, true)
 442                         if tok == arvadostest.SpectatorToken {
 443                                 // Valid token never offers to retry
 444                                 // with different credentials.
 445                                 c.Check(code, check.Equals, http.StatusNotFound)
 446                         } else {
 447                                 // Invalid token can ask to retry
 448                                 // depending on the authz method.
 449                                 c.Check(code, check.Equals, failCode)
 450                         }
 451                         if code == 404 {
 452                                 c.Check(body, check.Equals, notFoundMessage+"\n")
 453                         } else {
 454                                 c.Check(body, check.Equals, unauthorizedMessage+"\n")
 455                         }
 456                 }
 457         }
 458 }
 459
 460 func (s *IntegrationSuite) TestVhostPortMatch(c *check.C) {
 461         for _, host := range []string{"download.example.com", "DOWNLOAD.EXAMPLE.COM"} {
 462                 for _, port := range []string{"80", "443", "8000"} {
 463                         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = fmt.Sprintf("download.example.com:%v", port)
 464                         u := mustParseURL(fmt.Sprintf("http://%v/by_id/%v/foo", host, arvadostest.FooCollection))
 465                         req := &http.Request{
 466                                 Method:     "GET",
 467                                 Host:       u.Host,
 468                                 URL:        u,
 469                                 RequestURI: u.RequestURI(),
 470                                 Header:     http.Header{"Authorization": []string{"Bearer " + arvadostest.ActiveToken}},
 471                         }
 472                         req, resp := s.doReq(req)
 473                         code, _ := resp.Code, resp.Body.String()
 474
 475                         if port == "8000" {
 476                                 c.Check(code, check.Equals, 401)
 477                         } else {
 478                                 c.Check(code, check.Equals, 200)
 479                         }
 480                 }
 481         }
 482 }
 483
 484 func (s *IntegrationSuite) do(method string, urlstring string, token string, hdr http.Header) (*http.Request, *httptest.ResponseRecorder) {
 485         u := mustParseURL(urlstring)
 486         if hdr == nil && token != "" {
 487                 hdr = http.Header{"Authorization": {"Bearer " + token}}
 488         } else if hdr == nil {
 489                 hdr = http.Header{}
 490         } else if token != "" {
 491                 panic("must not pass both token and hdr")
 492         }
 493         return s.doReq(&http.Request{
 494                 Method:     method,
 495                 Host:       u.Host,
 496                 URL:        u,
 497                 RequestURI: u.RequestURI(),
 498                 Header:     hdr,
 499         })
 500 }
 501
 502 func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
 503         resp := httptest.NewRecorder()
 504         s.handler.ServeHTTP(resp, req)
 505         if resp.Code != http.StatusSeeOther {
 506                 return req, resp
 507         }
 508         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 509         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 510         req = &http.Request{
 511                 Method:     "GET",
 512                 Host:       u.Host,
 513                 URL:        u,
 514                 RequestURI: u.RequestURI(),
 515                 Header:     http.Header{},
 516         }
 517         for _, c := range cookies {
 518                 req.AddCookie(c)
 519         }
 520         return s.doReq(req)
 521 }
 522
 523 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
 524         s.testVhostRedirectTokenToCookie(c, "GET",
 525                 arvadostest.FooCollection+".example.com/foo",
 526                 "?api_token="+arvadostest.ActiveToken,
 527                 nil,
 528                 "",
 529                 http.StatusOK,
 530                 "foo",
 531         )
 532 }
 533
 534 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
 535         s.testVhostRedirectTokenToCookie(c, "GET",
 536                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
 537                 "",
 538                 nil,
 539                 "",
 540                 http.StatusOK,
 541                 "foo",
 542         )
 543 }
 544
 545 func (s *IntegrationSuite) TestCollectionSharingToken(c *check.C) {
 546         s.testVhostRedirectTokenToCookie(c, "GET",
 547                 "example.com/c="+arvadostest.FooFileCollectionUUID+"/t="+arvadostest.FooFileCollectionSharingToken+"/foo",
 548                 "",
 549                 nil,
 550                 "",
 551                 http.StatusOK,
 552                 "foo",
 553         )
 554         // Same valid sharing token, but requesting a different collection
 555         s.testVhostRedirectTokenToCookie(c, "GET",
 556                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.FooFileCollectionSharingToken+"/foo",
 557                 "",
 558                 nil,
 559                 "",
 560                 http.StatusNotFound,
 561                 regexp.QuoteMeta(notFoundMessage+"\n"),
 562         )
 563 }
 564
 565 // Bad token in URL is 404 Not Found because it doesn't make sense to
 566 // retry the same URL with different authorization.
 567 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
 568         s.testVhostRedirectTokenToCookie(c, "GET",
 569                 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
 570                 "",
 571                 nil,
 572                 "",
 573                 http.StatusNotFound,
 574                 regexp.QuoteMeta(notFoundMessage+"\n"),
 575         )
 576 }
 577
 578 // Bad token in a cookie (even if it got there via our own
 579 // query-string-to-cookie redirect) is, in principle, retryable via
 580 // wb2-login-and-redirect flow.
 581 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
 582         // Inline
 583         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 584                 arvadostest.FooCollection+".example.com/foo",
 585                 "?api_token=thisisabogustoken",
 586                 http.Header{"Sec-Fetch-Mode": {"navigate"}},
 587                 "",
 588                 http.StatusSeeOther,
 589                 "",
 590         )
 591         u, err := url.Parse(resp.Header().Get("Location"))
 592         c.Assert(err, check.IsNil)
 593         c.Logf("redirected to %s", u)
 594         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 595         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 596         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
 597
 598         // Download/attachment indicated by ?disposition=attachment
 599         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 600                 arvadostest.FooCollection+".example.com/foo",
 601                 "?api_token=thisisabogustoken&disposition=attachment",
 602                 http.Header{"Sec-Fetch-Mode": {"navigate"}},
 603                 "",
 604                 http.StatusSeeOther,
 605                 "",
 606         )
 607         u, err = url.Parse(resp.Header().Get("Location"))
 608         c.Assert(err, check.IsNil)
 609         c.Logf("redirected to %s", u)
 610         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 611         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "")
 612         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 613
 614         // Download/attachment indicated by vhost
 615         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 616                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
 617                 "?api_token=thisisabogustoken",
 618                 http.Header{"Sec-Fetch-Mode": {"navigate"}},
 619                 "",
 620                 http.StatusSeeOther,
 621                 "",
 622         )
 623         u, err = url.Parse(resp.Header().Get("Location"))
 624         c.Assert(err, check.IsNil)
 625         c.Logf("redirected to %s", u)
 626         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 627         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "")
 628         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 629
 630         // Without "Sec-Fetch-Mode: navigate" header, just 401.
 631         s.testVhostRedirectTokenToCookie(c, "GET",
 632                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
 633                 "?api_token=thisisabogustoken",
 634                 http.Header{"Sec-Fetch-Mode": {"cors"}},
 635                 "",
 636                 http.StatusUnauthorized,
 637                 regexp.QuoteMeta(unauthorizedMessage+"\n"),
 638         )
 639         s.testVhostRedirectTokenToCookie(c, "GET",
 640                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
 641                 "?api_token=thisisabogustoken",
 642                 nil,
 643                 "",
 644                 http.StatusUnauthorized,
 645                 regexp.QuoteMeta(unauthorizedMessage+"\n"),
 646         )
 647 }
 648
 649 func (s *IntegrationSuite) TestVhostRedirectWithNoCache(c *check.C) {
 650         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 651                 arvadostest.FooCollection+".example.com/foo",
 652                 "?api_token=thisisabogustoken",
 653                 http.Header{
 654                         "Sec-Fetch-Mode": {"navigate"},
 655                         "Cache-Control":  {"no-cache"},
 656                 },
 657                 "",
 658                 http.StatusSeeOther,
 659                 "",
 660         )
 661         u, err := url.Parse(resp.Header().Get("Location"))
 662         c.Assert(err, check.IsNil)
 663         c.Logf("redirected to %s", u)
 664         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 665         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 666         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
 667 }
 668
 669 func (s *IntegrationSuite) TestNoTokenWorkbench2LoginFlow(c *check.C) {
 670         for _, trial := range []struct {
 671                 anonToken    bool
 672                 cacheControl string
 673         }{
 674                 {},
 675                 {cacheControl: "no-cache"},
 676                 {anonToken: true},
 677                 {anonToken: true, cacheControl: "no-cache"},
 678         } {
 679                 c.Logf("trial: %+v", trial)
 680
 681                 if trial.anonToken {
 682                         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 683                 } else {
 684                         s.handler.Cluster.Users.AnonymousUserToken = ""
 685                 }
 686                 req, err := http.NewRequest("GET", "http://"+arvadostest.FooCollection+".example.com/foo", nil)
 687                 c.Assert(err, check.IsNil)
 688                 req.Header.Set("Sec-Fetch-Mode", "navigate")
 689                 if trial.cacheControl != "" {
 690                         req.Header.Set("Cache-Control", trial.cacheControl)
 691                 }
 692                 resp := httptest.NewRecorder()
 693                 s.handler.ServeHTTP(resp, req)
 694                 c.Check(resp.Code, check.Equals, http.StatusSeeOther)
 695                 u, err := url.Parse(resp.Header().Get("Location"))
 696                 c.Assert(err, check.IsNil)
 697                 c.Logf("redirected to %q", u)
 698                 c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 699                 c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 700                 c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
 701         }
 702 }
 703
 704 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
 705         s.testVhostRedirectTokenToCookie(c, "GET",
 706                 "example.com/c="+arvadostest.FooCollection+"/foo",
 707                 "?api_token="+arvadostest.ActiveToken,
 708                 nil,
 709                 "",
 710                 http.StatusBadRequest,
 711                 regexp.QuoteMeta("cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n"),
 712         )
 713 }
 714
 715 // If client requests an attachment by putting ?disposition=attachment
 716 // in the query string, and gets redirected, the redirect target
 717 // should respond with an attachment.
 718 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
 719         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 720                 arvadostest.FooCollection+".example.com/foo",
 721                 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
 722                 nil,
 723                 "",
 724                 http.StatusOK,
 725                 "foo",
 726         )
 727         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 728 }
 729
 730 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
 731         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 732         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 733                 "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
 734                 "?api_token="+arvadostest.ActiveToken,
 735                 nil,
 736                 "",
 737                 http.StatusOK,
 738                 "foo",
 739         )
 740         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 741 }
 742
 743 func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
 744         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 745         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 746                 "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
 747                 "?api_token="+arvadostest.ActiveToken,
 748                 nil,
 749                 "",
 750                 http.StatusOK,
 751                 "waz",
 752         )
 753         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 754         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 755                 "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz",
 756                 "?api_token="+arvadostest.ActiveToken,
 757                 nil,
 758                 "",
 759                 http.StatusOK,
 760                 "waz",
 761         )
 762         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 763 }
 764
 765 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
 766         s.handler.Cluster.Collections.TrustAllContent = true
 767         s.testVhostRedirectTokenToCookie(c, "GET",
 768                 "example.com/c="+arvadostest.FooCollection+"/foo",
 769                 "?api_token="+arvadostest.ActiveToken,
 770                 nil,
 771                 "",
 772                 http.StatusOK,
 773                 "foo",
 774         )
 775 }
 776
 777 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
 778         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
 779
 780         s.testVhostRedirectTokenToCookie(c, "GET",
 781                 "example.com/c="+arvadostest.FooCollection+"/foo",
 782                 "?api_token="+arvadostest.ActiveToken,
 783                 nil,
 784                 "",
 785                 http.StatusBadRequest,
 786                 regexp.QuoteMeta("cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n"),
 787         )
 788
 789         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 790                 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
 791                 "?api_token="+arvadostest.ActiveToken,
 792                 nil,
 793                 "",
 794                 http.StatusOK,
 795                 "foo",
 796         )
 797         c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
 798 }
 799
 800 func (s *IntegrationSuite) TestVhostRedirectMultipleTokens(c *check.C) {
 801         baseUrl := arvadostest.FooCollection + ".example.com/foo"
 802         query := url.Values{}
 803
 804         // The intent of these tests is to check that requests are redirected
 805         // correctly in the presence of multiple API tokens. The exact response
 806         // codes and content are not closely considered: they're just how
 807         // keep-web responded when we made the smallest possible fix. Changing
 808         // those responses may be okay, but you should still test all these
 809         // different cases and the associated redirect logic.
 810         query["api_token"] = []string{arvadostest.ActiveToken, arvadostest.AnonymousToken}
 811         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
 812         query["api_token"] = []string{arvadostest.ActiveToken, arvadostest.AnonymousToken, ""}
 813         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
 814         query["api_token"] = []string{arvadostest.ActiveToken, "", arvadostest.AnonymousToken}
 815         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
 816         query["api_token"] = []string{"", arvadostest.ActiveToken}
 817         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
 818
 819         expectContent := regexp.QuoteMeta(unauthorizedMessage + "\n")
 820         query["api_token"] = []string{arvadostest.AnonymousToken, "invalidtoo"}
 821         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusUnauthorized, expectContent)
 822         query["api_token"] = []string{arvadostest.AnonymousToken, ""}
 823         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusUnauthorized, expectContent)
 824         query["api_token"] = []string{"", arvadostest.AnonymousToken}
 825         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusUnauthorized, expectContent)
 826 }
 827
 828 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
 829         s.testVhostRedirectTokenToCookie(c, "POST",
 830                 arvadostest.FooCollection+".example.com/foo",
 831                 "",
 832                 http.Header{"Content-Type": {"application/x-www-form-urlencoded"}},
 833                 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
 834                 http.StatusOK,
 835                 "foo",
 836         )
 837 }
 838
 839 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
 840         s.testVhostRedirectTokenToCookie(c, "POST",
 841                 arvadostest.FooCollection+".example.com/foo",
 842                 "",
 843                 http.Header{"Content-Type": {"application/x-www-form-urlencoded"}},
 844                 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
 845                 http.StatusNotFound,
 846                 regexp.QuoteMeta(notFoundMessage+"\n"),
 847         )
 848 }
 849
 850 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
 851         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 852         s.testVhostRedirectTokenToCookie(c, "GET",
 853                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 854                 "",
 855                 nil,
 856                 "",
 857                 http.StatusOK,
 858                 "Hello world\n",
 859         )
 860 }
 861
 862 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
 863         s.handler.Cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
 864         s.testVhostRedirectTokenToCookie(c, "GET",
 865                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 866                 "",
 867                 nil,
 868                 "",
 869                 http.StatusUnauthorized,
 870                 "Authorization tokens are not accepted here: .*\n",
 871         )
 872 }
 873
 874 func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
 875         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 876
 877         client := arvados.NewClientFromEnv()
 878         client.AuthToken = arvadostest.ActiveToken
 879         fs, err := (&arvados.Collection{}).FileSystem(client, nil)
 880         c.Assert(err, check.IsNil)
 881         f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777)
 882         c.Assert(err, check.IsNil)
 883         f.Close()
 884         mtxt, err := fs.MarshalManifest(".")
 885         c.Assert(err, check.IsNil)
 886         var coll arvados.Collection
 887         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 888                 "collection": map[string]string{
 889                         "manifest_text": mtxt,
 890                 },
 891         })
 892         c.Assert(err, check.IsNil)
 893
 894         u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
 895         req := &http.Request{
 896                 Method:     "GET",
 897                 Host:       u.Host,
 898                 URL:        u,
 899                 RequestURI: u.RequestURI(),
 900                 Header: http.Header{
 901                         "Authorization": {"Bearer " + client.AuthToken},
 902                 },
 903         }
 904         resp := httptest.NewRecorder()
 905         s.handler.ServeHTTP(resp, req)
 906         c.Check(resp.Code, check.Equals, http.StatusOK)
 907         c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\&#34;odd&#39; path chars.*`)
 908 }
 909
 910 func (s *IntegrationSuite) TestForwardSlashSubstitution(c *check.C) {
 911         arv := arvados.NewClientFromEnv()
 912         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 913         s.handler.Cluster.Collections.ForwardSlashNameSubstitution = "{SOLIDUS}"
 914         name := "foo/bar/baz"
 915         nameShown := strings.Replace(name, "/", "{SOLIDUS}", -1)
 916         nameShownEscaped := strings.Replace(name, "/", "%7bSOLIDUS%7d", -1)
 917
 918         client := arvados.NewClientFromEnv()
 919         client.AuthToken = arvadostest.ActiveToken
 920         fs, err := (&arvados.Collection{}).FileSystem(client, nil)
 921         c.Assert(err, check.IsNil)
 922         f, err := fs.OpenFile("filename", os.O_CREATE, 0777)
 923         c.Assert(err, check.IsNil)
 924         f.Close()
 925         mtxt, err := fs.MarshalManifest(".")
 926         c.Assert(err, check.IsNil)
 927         var coll arvados.Collection
 928         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 929                 "collection": map[string]string{
 930                         "manifest_text": mtxt,
 931                         "name":          name,
 932                         "owner_uuid":    arvadostest.AProjectUUID,
 933                 },
 934         })
 935         c.Assert(err, check.IsNil)
 936         defer arv.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
 937
 938         base := "http://download.example.com/by_id/" + coll.OwnerUUID + "/"
 939         for tryURL, expectRegexp := range map[string]string{
 940                 base:                          `(?ms).*href="./` + nameShownEscaped + `/"\S+` + nameShown + `.*`,
 941                 base + nameShownEscaped + "/": `(?ms).*href="./filename"\S+filename.*`,
 942         } {
 943                 u, _ := url.Parse(tryURL)
 944                 req := &http.Request{
 945                         Method:     "GET",
 946                         Host:       u.Host,
 947                         URL:        u,
 948                         RequestURI: u.RequestURI(),
 949                         Header: http.Header{
 950                                 "Authorization": {"Bearer " + client.AuthToken},
 951                         },
 952                 }
 953                 resp := httptest.NewRecorder()
 954                 s.handler.ServeHTTP(resp, req)
 955                 c.Check(resp.Code, check.Equals, http.StatusOK)
 956                 c.Check(resp.Body.String(), check.Matches, expectRegexp)
 957         }
 958 }
 959
 960 // XHRs can't follow redirect-with-cookie so they rely on method=POST
 961 // and disposition=attachment (telling us it's acceptable to respond
 962 // with content instead of a redirect) and an Origin header that gets
 963 // added automatically by the browser (telling us it's desirable to do
 964 // so).
 965 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
 966         u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
 967         req := &http.Request{
 968                 Method:     "POST",
 969                 Host:       u.Host,
 970                 URL:        u,
 971                 RequestURI: u.RequestURI(),
 972                 Header: http.Header{
 973                         "Origin":       {"https://origin.example"},
 974                         "Content-Type": {"application/x-www-form-urlencoded"},
 975                 },
 976                 Body: ioutil.NopCloser(strings.NewReader(url.Values{
 977                         "api_token":   {arvadostest.ActiveToken},
 978                         "disposition": {"attachment"},
 979                 }.Encode())),
 980         }
 981         resp := httptest.NewRecorder()
 982         s.handler.ServeHTTP(resp, req)
 983         c.Check(resp.Code, check.Equals, http.StatusOK)
 984         c.Check(resp.Body.String(), check.Equals, "foo")
 985         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
 986
 987         // GET + Origin header is representative of both AJAX GET
 988         // requests and inline images via <IMG crossorigin="anonymous"
 989         // src="...">.
 990         u.RawQuery = "api_token=" + url.QueryEscape(arvadostest.ActiveTokenV2)
 991         req = &http.Request{
 992                 Method:     "GET",
 993                 Host:       u.Host,
 994                 URL:        u,
 995                 RequestURI: u.RequestURI(),
 996                 Header: http.Header{
 997                         "Origin": {"https://origin.example"},
 998                 },
 999         }
1000         resp = httptest.NewRecorder()
1001         s.handler.ServeHTTP(resp, req)
1002         c.Check(resp.Code, check.Equals, http.StatusOK)
1003         c.Check(resp.Body.String(), check.Equals, "foo")
1004         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
1005 }
1006
1007 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString string, reqHeader http.Header, reqBody string, expectStatus int, matchRespBody string) *httptest.ResponseRecorder {
1008         if reqHeader == nil {
1009                 reqHeader = http.Header{}
1010         }
1011         u, _ := url.Parse(`http://` + hostPath + queryString)
1012         c.Logf("requesting %s", u)
1013         req := &http.Request{
1014                 Method:     method,
1015                 Host:       u.Host,
1016                 URL:        u,
1017                 RequestURI: u.RequestURI(),
1018                 Header:     reqHeader,
1019                 Body:       ioutil.NopCloser(strings.NewReader(reqBody)),
1020         }
1021
1022         resp := httptest.NewRecorder()
1023         defer func() {
1024                 c.Check(resp.Code, check.Equals, expectStatus)
1025                 c.Check(resp.Body.String(), check.Matches, matchRespBody)
1026         }()
1027
1028         s.handler.ServeHTTP(resp, req)
1029         if resp.Code != http.StatusSeeOther {
1030                 attachment, _ := regexp.MatchString(`^attachment(;|$)`, resp.Header().Get("Content-Disposition"))
1031                 // Since we're not redirecting, check that any api_token in the URL is
1032                 // handled safely.
1033                 // If there is no token in the URL, then we're good.
1034                 // Otherwise, if the response code is an error, the body is expected to
1035                 // be static content, and nothing that might maliciously introspect the
1036                 // URL. It's considered safe and allowed.
1037                 // Otherwise, if the response content has attachment disposition,
1038                 // that's considered safe for all the reasons explained in the
1039                 // safeAttachment comment in handler.go.
1040                 c.Check(!u.Query().Has("api_token") || resp.Code >= 400 || attachment, check.Equals, true)
1041                 return resp
1042         }
1043
1044         loc, err := url.Parse(resp.Header().Get("Location"))
1045         c.Assert(err, check.IsNil)
1046         c.Check(loc.Scheme, check.Equals, u.Scheme)
1047         c.Check(loc.Host, check.Equals, u.Host)
1048         c.Check(loc.RawPath, check.Equals, u.RawPath)
1049         // If the response was a redirect, it should never include an API token.
1050         c.Check(loc.Query().Has("api_token"), check.Equals, false)
1051         c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
1052         cookies := (&http.Response{Header: resp.Header()}).Cookies()
1053
1054         c.Logf("following redirect to %s", u)
1055         req = &http.Request{
1056                 Method:     "GET",
1057                 Host:       loc.Host,
1058                 URL:        loc,
1059                 RequestURI: loc.RequestURI(),
1060                 Header:     reqHeader,
1061         }
1062         for _, c := range cookies {
1063                 req.AddCookie(c)
1064         }
1065
1066         resp = httptest.NewRecorder()
1067         s.handler.ServeHTTP(resp, req)
1068
1069         if resp.Code != http.StatusSeeOther {
1070                 c.Check(resp.Header().Get("Location"), check.Equals, "")
1071         }
1072         return resp
1073 }
1074
1075 func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) {
1076         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
1077         s.testDirectoryListing(c)
1078 }
1079
1080 func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) {
1081         s.handler.Cluster.Users.AnonymousUserToken = ""
1082         s.testDirectoryListing(c)
1083 }
1084
1085 func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
1086         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1087         authHeader := http.Header{
1088                 "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
1089         }
1090         for _, trial := range []struct {
1091                 uri      string
1092                 header   http.Header
1093                 expect   []string
1094                 redirect string
1095                 cutDirs  int
1096         }{
1097                 {
1098                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
1099                         header:  authHeader,
1100                         expect:  []string{"dir1/foo", "dir1/bar"},
1101                         cutDirs: 0,
1102                 },
1103                 {
1104                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
1105                         header:  authHeader,
1106                         expect:  []string{"foo", "bar"},
1107                         cutDirs: 1,
1108                 },
1109                 {
1110                         // URLs of this form ignore authHeader, and
1111                         // FooAndBarFilesInDirUUID isn't public, so
1112                         // this returns 401.
1113                         uri:    "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
1114                         header: authHeader,
1115                         expect: nil,
1116                 },
1117                 {
1118                         uri:     "download.example.com/users/active/foo_file_in_dir/",
1119                         header:  authHeader,
1120                         expect:  []string{"dir1/"},
1121                         cutDirs: 3,
1122                 },
1123                 {
1124                         uri:     "download.example.com/users/active/foo_file_in_dir/dir1/",
1125                         header:  authHeader,
1126                         expect:  []string{"bar"},
1127                         cutDirs: 4,
1128                 },
1129                 {
1130                         uri:     "download.example.com/",
1131                         header:  authHeader,
1132                         expect:  []string{"users/"},
1133                         cutDirs: 0,
1134                 },
1135                 {
1136                         uri:      "download.example.com/users",
1137                         header:   authHeader,
1138                         redirect: "/users/",
1139                         expect:   []string{"active/"},
1140                         cutDirs:  1,
1141                 },
1142                 {
1143                         uri:     "download.example.com/users/",
1144                         header:  authHeader,
1145                         expect:  []string{"active/"},
1146                         cutDirs: 1,
1147                 },
1148                 {
1149                         uri:      "download.example.com/users/active",
1150                         header:   authHeader,
1151                         redirect: "/users/active/",
1152                         expect:   []string{"foo_file_in_dir/"},
1153                         cutDirs:  2,
1154                 },
1155                 {
1156                         uri:     "download.example.com/users/active/",
1157                         header:  authHeader,
1158                         expect:  []string{"foo_file_in_dir/"},
1159                         cutDirs: 2,
1160                 },
1161                 {
1162                         uri:     "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
1163                         header:  nil,
1164                         expect:  []string{"dir1/foo", "dir1/bar"},
1165                         cutDirs: 4,
1166                 },
1167                 {
1168                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
1169                         header:  nil,
1170                         expect:  []string{"dir1/foo", "dir1/bar"},
1171                         cutDirs: 2,
1172                 },
1173                 {
1174                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken,
1175                         header:  nil,
1176                         expect:  []string{"dir1/foo", "dir1/bar"},
1177                         cutDirs: 2,
1178                 },
1179                 {
1180                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID,
1181                         header:  authHeader,
1182                         expect:  []string{"dir1/foo", "dir1/bar"},
1183                         cutDirs: 1,
1184                 },
1185                 {
1186                         uri:      "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
1187                         header:   authHeader,
1188                         redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
1189                         expect:   []string{"foo", "bar"},
1190                         cutDirs:  2,
1191                 },
1192                 {
1193                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
1194                         header:  authHeader,
1195                         expect:  []string{"foo", "bar"},
1196                         cutDirs: 3,
1197                 },
1198                 {
1199                         uri:      arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
1200                         header:   authHeader,
1201                         redirect: "/dir1/",
1202                         expect:   []string{"foo", "bar"},
1203                         cutDirs:  1,
1204                 },
1205                 {
1206                         uri:    "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
1207                         header: authHeader,
1208                         expect: nil,
1209                 },
1210                 {
1211                         uri:     "download.example.com/c=" + arvadostest.WazVersion1Collection,
1212                         header:  authHeader,
1213                         expect:  []string{"waz"},
1214                         cutDirs: 1,
1215                 },
1216                 {
1217                         uri:     "download.example.com/by_id/" + arvadostest.WazVersion1Collection,
1218                         header:  authHeader,
1219                         expect:  []string{"waz"},
1220                         cutDirs: 2,
1221                 },
1222         } {
1223                 comment := check.Commentf("HTML: %q => %q", trial.uri, trial.expect)
1224                 resp := httptest.NewRecorder()
1225                 u := mustParseURL("//" + trial.uri)
1226                 req := &http.Request{
1227                         Method:     "GET",
1228                         Host:       u.Host,
1229                         URL:        u,
1230                         RequestURI: u.RequestURI(),
1231                         Header:     copyHeader(trial.header),
1232                 }
1233                 s.handler.ServeHTTP(resp, req)
1234                 var cookies []*http.Cookie
1235                 for resp.Code == http.StatusSeeOther {
1236                         u, _ := req.URL.Parse(resp.Header().Get("Location"))
1237                         req = &http.Request{
1238                                 Method:     "GET",
1239                                 Host:       u.Host,
1240                                 URL:        u,
1241                                 RequestURI: u.RequestURI(),
1242                                 Header:     copyHeader(trial.header),
1243                         }
1244                         cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
1245                         for _, c := range cookies {
1246                                 req.AddCookie(c)
1247                         }
1248                         resp = httptest.NewRecorder()
1249                         s.handler.ServeHTTP(resp, req)
1250                 }
1251                 if trial.redirect != "" {
1252                         c.Check(req.URL.Path, check.Equals, trial.redirect, comment)
1253                 }
1254                 if trial.expect == nil {
1255                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1256                 } else {
1257                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
1258                         for _, e := range trial.expect {
1259                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`, comment)
1260                         }
1261                         c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`, comment)
1262                 }
1263
1264                 comment = check.Commentf("WebDAV: %q => %q", trial.uri, trial.expect)
1265                 req = &http.Request{
1266                         Method:     "OPTIONS",
1267                         Host:       u.Host,
1268                         URL:        u,
1269                         RequestURI: u.RequestURI(),
1270                         Header:     copyHeader(trial.header),
1271                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
1272                 }
1273                 resp = httptest.NewRecorder()
1274                 s.handler.ServeHTTP(resp, req)
1275                 if trial.expect == nil {
1276                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1277                 } else {
1278                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
1279                 }
1280
1281                 req = &http.Request{
1282                         Method:     "PROPFIND",
1283                         Host:       u.Host,
1284                         URL:        u,
1285                         RequestURI: u.RequestURI(),
1286                         Header:     copyHeader(trial.header),
1287                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
1288                 }
1289                 resp = httptest.NewRecorder()
1290                 s.handler.ServeHTTP(resp, req)
1291                 if trial.expect == nil {
1292                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1293                 } else {
1294                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment)
1295                         for _, e := range trial.expect {
1296                                 if strings.HasSuffix(e, "/") {
1297                                         e = filepath.Join(u.Path, e) + "/"
1298                                 } else {
1299                                         e = filepath.Join(u.Path, e)
1300                                 }
1301                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+e+`</D:href>.*`, comment)
1302                         }
1303                 }
1304         }
1305 }
1306
1307 func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
1308         arv := arvados.NewClientFromEnv()
1309         var newCollection arvados.Collection
1310         err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1311                 "collection": map[string]string{
1312                         "owner_uuid":    arvadostest.ActiveUserUUID,
1313                         "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
1314                         "name":          "keep-web test collection",
1315                 },
1316                 "ensure_unique_name": true,
1317         })
1318         c.Assert(err, check.IsNil)
1319         defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
1320
1321         var updated arvados.Collection
1322         for _, fnm := range []string{"foo.txt", "bar.txt"} {
1323                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
1324                 u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
1325                 req := &http.Request{
1326                         Method:     "DELETE",
1327                         Host:       u.Host,
1328                         URL:        u,
1329                         RequestURI: u.RequestURI(),
1330                         Header: http.Header{
1331                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
1332                         },
1333                 }
1334                 resp := httptest.NewRecorder()
1335                 s.handler.ServeHTTP(resp, req)
1336                 c.Check(resp.Code, check.Equals, http.StatusNoContent)
1337
1338                 updated = arvados.Collection{}
1339                 err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
1340                 c.Check(err, check.IsNil)
1341                 c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`)
1342                 c.Logf("updated manifest_text %q", updated.ManifestText)
1343         }
1344         c.Check(updated.ManifestText, check.Equals, "")
1345 }
1346
1347 func (s *IntegrationSuite) TestFileContentType(c *check.C) {
1348         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1349
1350         client := arvados.NewClientFromEnv()
1351         client.AuthToken = arvadostest.ActiveToken
1352         arv, err := arvadosclient.New(client)
1353         c.Assert(err, check.Equals, nil)
1354         kc, err := keepclient.MakeKeepClient(arv)
1355         c.Assert(err, check.Equals, nil)
1356
1357         fs, err := (&arvados.Collection{}).FileSystem(client, kc)
1358         c.Assert(err, check.IsNil)
1359
1360         trials := []struct {
1361                 filename    string
1362                 content     string
1363                 contentType string
1364         }{
1365                 {"picture.txt", "BMX bikes are small this year\n", "text/plain; charset=utf-8"},
1366                 {"picture.bmp", "BMX bikes are small this year\n", "image/(x-ms-)?bmp"},
1367                 {"picture.jpg", "BMX bikes are small this year\n", "image/jpeg"},
1368                 {"picture1", "BMX bikes are small this year\n", "image/bmp"},            // content sniff; "BM" is the magic signature for .bmp
1369                 {"picture2", "Cars are small this year\n", "text/plain; charset=utf-8"}, // content sniff
1370         }
1371         for _, trial := range trials {
1372                 f, err := fs.OpenFile(trial.filename, os.O_CREATE|os.O_WRONLY, 0777)
1373                 c.Assert(err, check.IsNil)
1374                 _, err = f.Write([]byte(trial.content))
1375                 c.Assert(err, check.IsNil)
1376                 c.Assert(f.Close(), check.IsNil)
1377         }
1378         mtxt, err := fs.MarshalManifest(".")
1379         c.Assert(err, check.IsNil)
1380         var coll arvados.Collection
1381         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1382                 "collection": map[string]string{
1383                         "manifest_text": mtxt,
1384                 },
1385         })
1386         c.Assert(err, check.IsNil)
1387
1388         for _, trial := range trials {
1389                 u, _ := url.Parse("http://download.example.com/by_id/" + coll.UUID + "/" + trial.filename)
1390                 req := &http.Request{
1391                         Method:     "GET",
1392                         Host:       u.Host,
1393                         URL:        u,
1394                         RequestURI: u.RequestURI(),
1395                         Header: http.Header{
1396                                 "Authorization": {"Bearer " + client.AuthToken},
1397                         },
1398                 }
1399                 resp := httptest.NewRecorder()
1400                 s.handler.ServeHTTP(resp, req)
1401                 c.Check(resp.Code, check.Equals, http.StatusOK)
1402                 c.Check(resp.Header().Get("Content-Type"), check.Matches, trial.contentType)
1403                 c.Check(resp.Body.String(), check.Equals, trial.content)
1404         }
1405 }
1406
1407 func (s *IntegrationSuite) TestKeepClientBlockCache(c *check.C) {
1408         s.handler.Cluster.Collections.WebDAVCache.MaxBlockEntries = 42
1409         c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Not(check.Equals), 42)
1410         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/t=" + arvadostest.ActiveToken + "/foo")
1411         req := &http.Request{
1412                 Method:     "GET",
1413                 Host:       u.Host,
1414                 URL:        u,
1415                 RequestURI: u.RequestURI(),
1416         }
1417         resp := httptest.NewRecorder()
1418         s.handler.ServeHTTP(resp, req)
1419         c.Check(resp.Code, check.Equals, http.StatusOK)
1420         c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Equals, 42)
1421 }
1422
1423 // Writing to a collection shouldn't affect its entry in the
1424 // PDH-to-manifest cache.
1425 func (s *IntegrationSuite) TestCacheWriteCollectionSamePDH(c *check.C) {
1426         arv, err := arvadosclient.MakeArvadosClient()
1427         c.Assert(err, check.Equals, nil)
1428         arv.ApiToken = arvadostest.ActiveToken
1429
1430         u := mustParseURL("http://x.example/testfile")
1431         req := &http.Request{
1432                 Method:     "GET",
1433                 Host:       u.Host,
1434                 URL:        u,
1435                 RequestURI: u.RequestURI(),
1436                 Header:     http.Header{"Authorization": {"Bearer " + arv.ApiToken}},
1437         }
1438
1439         checkWithID := func(id string, status int) {
1440                 req.URL.Host = strings.Replace(id, "+", "-", -1) + ".example"
1441                 req.Host = req.URL.Host
1442                 resp := httptest.NewRecorder()
1443                 s.handler.ServeHTTP(resp, req)
1444                 c.Check(resp.Code, check.Equals, status)
1445         }
1446
1447         var colls [2]arvados.Collection
1448         for i := range colls {
1449                 err := arv.Create("collections",
1450                         map[string]interface{}{
1451                                 "ensure_unique_name": true,
1452                                 "collection": map[string]interface{}{
1453                                         "name": "test collection",
1454                                 },
1455                         }, &colls[i])
1456                 c.Assert(err, check.Equals, nil)
1457         }
1458
1459         // Populate cache with empty collection
1460         checkWithID(colls[0].PortableDataHash, http.StatusNotFound)
1461
1462         // write a file to colls[0]
1463         reqPut := *req
1464         reqPut.Method = "PUT"
1465         reqPut.URL.Host = colls[0].UUID + ".example"
1466         reqPut.Host = req.URL.Host
1467         reqPut.Body = ioutil.NopCloser(bytes.NewBufferString("testdata"))
1468         resp := httptest.NewRecorder()
1469         s.handler.ServeHTTP(resp, &reqPut)
1470         c.Check(resp.Code, check.Equals, http.StatusCreated)
1471
1472         // new file should not appear in colls[1]
1473         checkWithID(colls[1].PortableDataHash, http.StatusNotFound)
1474         checkWithID(colls[1].UUID, http.StatusNotFound)
1475
1476         checkWithID(colls[0].UUID, http.StatusOK)
1477 }
1478
1479 func copyHeader(h http.Header) http.Header {
1480         hc := http.Header{}
1481         for k, v := range h {
1482                 hc[k] = append([]string(nil), v...)
1483         }
1484         return hc
1485 }
1486
1487 func (s *IntegrationSuite) checkUploadDownloadRequest(c *check.C, req *http.Request,
1488         successCode int, direction string, perm bool, userUuid, collectionUuid, collectionPDH, filepath string) {
1489
1490         client := arvados.NewClientFromEnv()
1491         client.AuthToken = arvadostest.AdminToken
1492         var logentries arvados.LogList
1493         limit1 := 1
1494         err := client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil,
1495                 arvados.ResourceListParams{
1496                         Limit: &limit1,
1497                         Order: "created_at desc"})
1498         c.Check(err, check.IsNil)
1499         c.Check(logentries.Items, check.HasLen, 1)
1500         lastLogId := logentries.Items[0].ID
1501         c.Logf("lastLogId: %d", lastLogId)
1502
1503         var logbuf bytes.Buffer
1504         logger := logrus.New()
1505         logger.Out = &logbuf
1506         resp := httptest.NewRecorder()
1507         req = req.WithContext(ctxlog.Context(context.Background(), logger))
1508         s.handler.ServeHTTP(resp, req)
1509
1510         if perm {
1511                 c.Check(resp.Result().StatusCode, check.Equals, successCode)
1512                 c.Check(logbuf.String(), check.Matches, `(?ms).*msg="File `+direction+`".*`)
1513                 c.Check(logbuf.String(), check.Not(check.Matches), `(?ms).*level=error.*`)
1514
1515                 deadline := time.Now().Add(time.Second)
1516                 for {
1517                         c.Assert(time.Now().After(deadline), check.Equals, false, check.Commentf("timed out waiting for log entry"))
1518                         logentries = arvados.LogList{}
1519                         err = client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil,
1520                                 arvados.ResourceListParams{
1521                                         Filters: []arvados.Filter{
1522                                                 {Attr: "event_type", Operator: "=", Operand: "file_" + direction},
1523                                                 {Attr: "object_uuid", Operator: "=", Operand: userUuid},
1524                                         },
1525                                         Limit: &limit1,
1526                                         Order: "created_at desc",
1527                                 })
1528                         c.Assert(err, check.IsNil)
1529                         if len(logentries.Items) > 0 &&
1530                                 logentries.Items[0].ID > lastLogId &&
1531                                 logentries.Items[0].ObjectUUID == userUuid &&
1532                                 logentries.Items[0].Properties["collection_uuid"] == collectionUuid &&
1533                                 (collectionPDH == "" || logentries.Items[0].Properties["portable_data_hash"] == collectionPDH) &&
1534                                 logentries.Items[0].Properties["collection_file_path"] == filepath {
1535                                 break
1536                         }
1537                         c.Logf("logentries.Items: %+v", logentries.Items)
1538                         time.Sleep(50 * time.Millisecond)
1539                 }
1540         } else {
1541                 c.Check(resp.Result().StatusCode, check.Equals, http.StatusForbidden)
1542                 c.Check(logbuf.String(), check.Equals, "")
1543         }
1544 }
1545
1546 func (s *IntegrationSuite) TestDownloadLoggingPermission(c *check.C) {
1547         u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
1548
1549         s.handler.Cluster.Collections.TrustAllContent = true
1550
1551         for _, adminperm := range []bool{true, false} {
1552                 for _, userperm := range []bool{true, false} {
1553                         s.handler.Cluster.Collections.WebDAVPermission.Admin.Download = adminperm
1554                         s.handler.Cluster.Collections.WebDAVPermission.User.Download = userperm
1555
1556                         // Test admin permission
1557                         req := &http.Request{
1558                                 Method:     "GET",
1559                                 Host:       u.Host,
1560                                 URL:        u,
1561                                 RequestURI: u.RequestURI(),
1562                                 Header: http.Header{
1563                                         "Authorization": {"Bearer " + arvadostest.AdminToken},
1564                                 },
1565                         }
1566                         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", adminperm,
1567                                 arvadostest.AdminUserUUID, arvadostest.FooCollection, arvadostest.FooCollectionPDH, "foo")
1568
1569                         // Test user permission
1570                         req = &http.Request{
1571                                 Method:     "GET",
1572                                 Host:       u.Host,
1573                                 URL:        u,
1574                                 RequestURI: u.RequestURI(),
1575                                 Header: http.Header{
1576                                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
1577                                 },
1578                         }
1579                         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", userperm,
1580                                 arvadostest.ActiveUserUUID, arvadostest.FooCollection, arvadostest.FooCollectionPDH, "foo")
1581                 }
1582         }
1583
1584         s.handler.Cluster.Collections.WebDAVPermission.User.Download = true
1585
1586         for _, tryurl := range []string{"http://" + arvadostest.MultilevelCollection1 + ".keep-web.example/dir1/subdir/file1",
1587                 "http://keep-web/users/active/multilevel_collection_1/dir1/subdir/file1"} {
1588
1589                 u = mustParseURL(tryurl)
1590                 req := &http.Request{
1591                         Method:     "GET",
1592                         Host:       u.Host,
1593                         URL:        u,
1594                         RequestURI: u.RequestURI(),
1595                         Header: http.Header{
1596                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
1597                         },
1598                 }
1599                 s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true,
1600                         arvadostest.ActiveUserUUID, arvadostest.MultilevelCollection1, arvadostest.MultilevelCollection1PDH, "dir1/subdir/file1")
1601         }
1602
1603         u = mustParseURL("http://" + strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + ".keep-web.example/foo")
1604         req := &http.Request{
1605                 Method:     "GET",
1606                 Host:       u.Host,
1607                 URL:        u,
1608                 RequestURI: u.RequestURI(),
1609                 Header: http.Header{
1610                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
1611                 },
1612         }
1613         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true,
1614                 arvadostest.ActiveUserUUID, "", arvadostest.FooCollectionPDH, "foo")
1615 }
1616
1617 func (s *IntegrationSuite) TestUploadLoggingPermission(c *check.C) {
1618         for _, adminperm := range []bool{true, false} {
1619                 for _, userperm := range []bool{true, false} {
1620
1621                         arv := arvados.NewClientFromEnv()
1622                         arv.AuthToken = arvadostest.ActiveToken
1623
1624                         var coll arvados.Collection
1625                         err := arv.RequestAndDecode(&coll,
1626                                 "POST",
1627                                 "/arvados/v1/collections",
1628                                 nil,
1629                                 map[string]interface{}{
1630                                         "ensure_unique_name": true,
1631                                         "collection": map[string]interface{}{
1632                                                 "name": "test collection",
1633                                         },
1634                                 })
1635                         c.Assert(err, check.Equals, nil)
1636
1637                         u := mustParseURL("http://" + coll.UUID + ".keep-web.example/bar")
1638
1639                         s.handler.Cluster.Collections.WebDAVPermission.Admin.Upload = adminperm
1640                         s.handler.Cluster.Collections.WebDAVPermission.User.Upload = userperm
1641
1642                         // Test admin permission
1643                         req := &http.Request{
1644                                 Method:     "PUT",
1645                                 Host:       u.Host,
1646                                 URL:        u,
1647                                 RequestURI: u.RequestURI(),
1648                                 Header: http.Header{
1649                                         "Authorization": {"Bearer " + arvadostest.AdminToken},
1650                                 },
1651                                 Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
1652                         }
1653                         s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", adminperm,
1654                                 arvadostest.AdminUserUUID, coll.UUID, "", "bar")
1655
1656                         // Test user permission
1657                         req = &http.Request{
1658                                 Method:     "PUT",
1659                                 Host:       u.Host,
1660                                 URL:        u,
1661                                 RequestURI: u.RequestURI(),
1662                                 Header: http.Header{
1663                                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
1664                                 },
1665                                 Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
1666                         }
1667                         s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", userperm,
1668                                 arvadostest.ActiveUserUUID, coll.UUID, "", "bar")
1669                 }
1670         }
1671 }
1672
1673 func (s *IntegrationSuite) TestConcurrentWrites(c *check.C) {
1674         s.handler.Cluster.Collections.WebDAVCache.TTL = arvados.Duration(time.Second * 2)
1675         lockTidyInterval = time.Second
1676         client := arvados.NewClientFromEnv()
1677         client.AuthToken = arvadostest.ActiveTokenV2
1678         // Start small, and increase concurrency (2^2, 4^2, ...)
1679         // only until hitting failure. Avoids unnecessarily long
1680         // failure reports.
1681         for n := 2; n < 16 && !c.Failed(); n = n * 2 {
1682                 c.Logf("%s: n=%d", c.TestName(), n)
1683
1684                 var coll arvados.Collection
1685                 err := client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, nil)
1686                 c.Assert(err, check.IsNil)
1687                 defer client.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
1688
1689                 var wg sync.WaitGroup
1690                 for i := 0; i < n && !c.Failed(); i++ {
1691                         i := i
1692                         wg.Add(1)
1693                         go func() {
1694                                 defer wg.Done()
1695                                 u := mustParseURL(fmt.Sprintf("http://%s.collections.example.com/i=%d", coll.UUID, i))
1696                                 resp := httptest.NewRecorder()
1697                                 req, err := http.NewRequest("MKCOL", u.String(), nil)
1698                                 c.Assert(err, check.IsNil)
1699                                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
1700                                 s.handler.ServeHTTP(resp, req)
1701                                 c.Assert(resp.Code, check.Equals, http.StatusCreated)
1702                                 for j := 0; j < n && !c.Failed(); j++ {
1703                                         j := j
1704                                         wg.Add(1)
1705                                         go func() {
1706                                                 defer wg.Done()
1707                                                 content := fmt.Sprintf("i=%d/j=%d", i, j)
1708                                                 u := mustParseURL("http://" + coll.UUID + ".collections.example.com/" + content)
1709
1710                                                 resp := httptest.NewRecorder()
1711                                                 req, err := http.NewRequest("PUT", u.String(), strings.NewReader(content))
1712                                                 c.Assert(err, check.IsNil)
1713                                                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
1714                                                 s.handler.ServeHTTP(resp, req)
1715                                                 c.Check(resp.Code, check.Equals, http.StatusCreated)
1716
1717                                                 time.Sleep(time.Second)
1718                                                 resp = httptest.NewRecorder()
1719                                                 req, err = http.NewRequest("GET", u.String(), nil)
1720                                                 c.Assert(err, check.IsNil)
1721                                                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
1722                                                 s.handler.ServeHTTP(resp, req)
1723                                                 c.Check(resp.Code, check.Equals, http.StatusOK)
1724                                                 c.Check(resp.Body.String(), check.Equals, content)
1725                                         }()
1726                                 }
1727                         }()
1728                 }
1729                 wg.Wait()
1730                 for i := 0; i < n; i++ {
1731                         u := mustParseURL(fmt.Sprintf("http://%s.collections.example.com/i=%d", coll.UUID, i))
1732                         resp := httptest.NewRecorder()
1733                         req, err := http.NewRequest("PROPFIND", u.String(), &bytes.Buffer{})
1734                         c.Assert(err, check.IsNil)
1735                         req.Header.Set("Authorization", "Bearer "+client.AuthToken)
1736                         s.handler.ServeHTTP(resp, req)
1737                         c.Assert(resp.Code, check.Equals, http.StatusMultiStatus)
1738                 }
1739         }
1740 }