services/keep-web/handler_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package keepweb
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "fmt"
  11         "html"
  12         "io"
  13         "io/ioutil"
  14         "net/http"
  15         "net/http/httptest"
  16         "net/url"
  17         "os"
  18         "path/filepath"
  19         "regexp"
  20         "strings"
  21         "time"
  22
  23         "git.arvados.org/arvados.git/lib/config"
  24         "git.arvados.org/arvados.git/sdk/go/arvados"
  25         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  26         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  27         "git.arvados.org/arvados.git/sdk/go/auth"
  28         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  29         "git.arvados.org/arvados.git/sdk/go/keepclient"
  30         "github.com/prometheus/client_golang/prometheus"
  31         "github.com/sirupsen/logrus"
  32         check "gopkg.in/check.v1"
  33 )
  34
  35 var _ = check.Suite(&UnitSuite{})
  36
  37 func init() {
  38         arvados.DebugLocksPanicMode = true
  39 }
  40
  41 type UnitSuite struct {
  42         cluster *arvados.Cluster
  43         handler *handler
  44 }
  45
  46 func (s *UnitSuite) SetUpTest(c *check.C) {
  47         logger := ctxlog.TestLogger(c)
  48         ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), logger)
  49         ldr.Path = "-"
  50         cfg, err := ldr.Load()
  51         c.Assert(err, check.IsNil)
  52         cc, err := cfg.GetCluster("")
  53         c.Assert(err, check.IsNil)
  54         s.cluster = cc
  55         s.handler = &handler{
  56                 Cluster: cc,
  57                 Cache: cache{
  58                         cluster:  cc,
  59                         logger:   logger,
  60                         registry: prometheus.NewRegistry(),
  61                 },
  62         }
  63 }
  64
  65 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
  66         h := s.handler
  67         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
  68         req := &http.Request{
  69                 Method:     "OPTIONS",
  70                 Host:       u.Host,
  71                 URL:        u,
  72                 RequestURI: u.RequestURI(),
  73                 Header: http.Header{
  74                         "Origin":                        {"https://workbench.example"},
  75                         "Access-Control-Request-Method": {"POST"},
  76                 },
  77         }
  78
  79         // Check preflight for an allowed request
  80         resp := httptest.NewRecorder()
  81         h.ServeHTTP(resp, req)
  82         c.Check(resp.Code, check.Equals, http.StatusOK)
  83         c.Check(resp.Body.String(), check.Equals, "")
  84         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
  85         c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
  86         c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout, Cache-Control")
  87
  88         // Check preflight for a disallowed request
  89         resp = httptest.NewRecorder()
  90         req.Header.Set("Access-Control-Request-Method", "MAKE-COFFEE")
  91         h.ServeHTTP(resp, req)
  92         c.Check(resp.Body.String(), check.Equals, "")
  93         c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
  94 }
  95
  96 func (s *UnitSuite) TestWebdavPrefixAndSource(c *check.C) {
  97         for _, trial := range []struct {
  98                 method   string
  99                 path     string
 100                 prefix   string
 101                 source   string
 102                 notFound bool
 103                 seeOther bool
 104         }{
 105                 {
 106                         method: "PROPFIND",
 107                         path:   "/",
 108                 },
 109                 {
 110                         method: "PROPFIND",
 111                         path:   "/dir1",
 112                 },
 113                 {
 114                         method: "PROPFIND",
 115                         path:   "/dir1/",
 116                 },
 117                 {
 118                         method: "PROPFIND",
 119                         path:   "/dir1/foo",
 120                         prefix: "/dir1",
 121                         source: "/dir1",
 122                 },
 123                 {
 124                         method: "PROPFIND",
 125                         path:   "/prefix/dir1/foo",
 126                         prefix: "/prefix/",
 127                         source: "",
 128                 },
 129                 {
 130                         method: "PROPFIND",
 131                         path:   "/prefix/dir1/foo",
 132                         prefix: "/prefix",
 133                         source: "",
 134                 },
 135                 {
 136                         method: "PROPFIND",
 137                         path:   "/prefix/dir1/foo",
 138                         prefix: "/prefix/",
 139                         source: "/",
 140                 },
 141                 {
 142                         method: "PROPFIND",
 143                         path:   "/prefix/foo",
 144                         prefix: "/prefix/",
 145                         source: "/dir1/",
 146                 },
 147                 {
 148                         method: "GET",
 149                         path:   "/prefix/foo",
 150                         prefix: "/prefix/",
 151                         source: "/dir1/",
 152                 },
 153                 {
 154                         method: "PROPFIND",
 155                         path:   "/prefix/",
 156                         prefix: "/prefix",
 157                         source: "/dir1",
 158                 },
 159                 {
 160                         method: "PROPFIND",
 161                         path:   "/prefix",
 162                         prefix: "/prefix",
 163                         source: "/dir1/",
 164                 },
 165                 {
 166                         method:   "GET",
 167                         path:     "/prefix",
 168                         prefix:   "/prefix",
 169                         source:   "/dir1",
 170                         seeOther: true,
 171                 },
 172                 {
 173                         method:   "PROPFIND",
 174                         path:     "/dir1/foo",
 175                         prefix:   "",
 176                         source:   "/dir1",
 177                         notFound: true,
 178                 },
 179         } {
 180                 c.Logf("trial %+v", trial)
 181                 u := mustParseURL("http://" + arvadostest.FooBarDirCollection + ".keep-web.example" + trial.path)
 182                 req := &http.Request{
 183                         Method:     trial.method,
 184                         Host:       u.Host,
 185                         URL:        u,
 186                         RequestURI: u.RequestURI(),
 187                         Header: http.Header{
 188                                 "Authorization":   {"Bearer " + arvadostest.ActiveTokenV2},
 189                                 "X-Webdav-Prefix": {trial.prefix},
 190                                 "X-Webdav-Source": {trial.source},
 191                         },
 192                         Body: ioutil.NopCloser(bytes.NewReader(nil)),
 193                 }
 194
 195                 resp := httptest.NewRecorder()
 196                 s.handler.ServeHTTP(resp, req)
 197                 if trial.notFound {
 198                         c.Check(resp.Code, check.Equals, http.StatusNotFound)
 199                 } else if trial.method == "PROPFIND" {
 200                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus)
 201                         c.Check(resp.Body.String(), check.Matches, `(?ms).*>\n?$`)
 202                 } else if trial.seeOther {
 203                         c.Check(resp.Code, check.Equals, http.StatusSeeOther)
 204                 } else {
 205                         c.Check(resp.Code, check.Equals, http.StatusOK)
 206                 }
 207         }
 208 }
 209
 210 func (s *UnitSuite) TestEmptyResponse(c *check.C) {
 211         for _, trial := range []struct {
 212                 dataExists    bool
 213                 sendIMSHeader bool
 214                 expectStatus  int
 215                 logRegexp     string
 216         }{
 217                 // If we return no content due to a Keep read error,
 218                 // we should emit a log message.
 219                 {false, false, http.StatusOK, `(?ms).*only wrote 0 bytes.*`},
 220
 221                 // If we return no content because the client sent an
 222                 // If-Modified-Since header, our response should be
 223                 // 304.  We still expect a "File download" log since it
 224                 // counts as a file access for auditing.
 225                 {true, true, http.StatusNotModified, `(?ms).*msg="File download".*`},
 226         } {
 227                 c.Logf("trial: %+v", trial)
 228                 arvadostest.StartKeep(2, true)
 229                 if trial.dataExists {
 230                         arv, err := arvadosclient.MakeArvadosClient()
 231                         c.Assert(err, check.IsNil)
 232                         arv.ApiToken = arvadostest.ActiveToken
 233                         kc, err := keepclient.MakeKeepClient(arv)
 234                         c.Assert(err, check.IsNil)
 235                         _, _, err = kc.PutB([]byte("foo"))
 236                         c.Assert(err, check.IsNil)
 237                 }
 238
 239                 u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
 240                 req := &http.Request{
 241                         Method:     "GET",
 242                         Host:       u.Host,
 243                         URL:        u,
 244                         RequestURI: u.RequestURI(),
 245                         Header: http.Header{
 246                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
 247                         },
 248                 }
 249                 if trial.sendIMSHeader {
 250                         req.Header.Set("If-Modified-Since", strings.Replace(time.Now().UTC().Format(time.RFC1123), "UTC", "GMT", -1))
 251                 }
 252
 253                 var logbuf bytes.Buffer
 254                 logger := logrus.New()
 255                 logger.Out = &logbuf
 256                 req = req.WithContext(ctxlog.Context(context.Background(), logger))
 257
 258                 resp := httptest.NewRecorder()
 259                 s.handler.ServeHTTP(resp, req)
 260                 c.Check(resp.Code, check.Equals, trial.expectStatus)
 261                 c.Check(resp.Body.String(), check.Equals, "")
 262
 263                 c.Log(logbuf.String())
 264                 c.Check(logbuf.String(), check.Matches, trial.logRegexp)
 265         }
 266 }
 267
 268 func (s *UnitSuite) TestInvalidUUID(c *check.C) {
 269         bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-"
 270         token := arvadostest.ActiveToken
 271         for _, trial := range []string{
 272                 "http://keep-web/c=" + bogusID + "/foo",
 273                 "http://keep-web/c=" + bogusID + "/t=" + token + "/foo",
 274                 "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo",
 275                 "http://keep-web/collections/" + bogusID + "/foo",
 276                 "http://" + bogusID + ".keep-web/" + bogusID + "/foo",
 277                 "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
 278         } {
 279                 c.Log(trial)
 280                 u := mustParseURL(trial)
 281                 req := &http.Request{
 282                         Method:     "GET",
 283                         Host:       u.Host,
 284                         URL:        u,
 285                         RequestURI: u.RequestURI(),
 286                 }
 287                 resp := httptest.NewRecorder()
 288                 s.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 289                 s.handler.ServeHTTP(resp, req)
 290                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 291         }
 292 }
 293
 294 func mustParseURL(s string) *url.URL {
 295         r, err := url.Parse(s)
 296         if err != nil {
 297                 panic("parse URL: " + s)
 298         }
 299         return r
 300 }
 301
 302 func (s *IntegrationSuite) TestVhost404(c *check.C) {
 303         for _, testURL := range []string{
 304                 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
 305                 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
 306         } {
 307                 resp := httptest.NewRecorder()
 308                 u := mustParseURL(testURL)
 309                 req := &http.Request{
 310                         Method:     "GET",
 311                         URL:        u,
 312                         RequestURI: u.RequestURI(),
 313                 }
 314                 s.handler.ServeHTTP(resp, req)
 315                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 316                 c.Check(resp.Body.String(), check.Equals, notFoundMessage+"\n")
 317         }
 318 }
 319
 320 // An authorizer modifies an HTTP request to make use of the given
 321 // token -- by adding it to a header, cookie, query param, or whatever
 322 // -- and returns the HTTP status code we should expect from keep-web if
 323 // the token is invalid.
 324 type authorizer func(*http.Request, string) int
 325
 326 func (s *IntegrationSuite) TestVhostViaAuthzHeaderOAuth2(c *check.C) {
 327         s.doVhostRequests(c, authzViaAuthzHeaderOAuth2)
 328 }
 329 func authzViaAuthzHeaderOAuth2(r *http.Request, tok string) int {
 330         r.Header.Add("Authorization", "Bearer "+tok)
 331         return http.StatusUnauthorized
 332 }
 333 func (s *IntegrationSuite) TestVhostViaAuthzHeaderBearer(c *check.C) {
 334         s.doVhostRequests(c, authzViaAuthzHeaderBearer)
 335 }
 336 func authzViaAuthzHeaderBearer(r *http.Request, tok string) int {
 337         r.Header.Add("Authorization", "Bearer "+tok)
 338         return http.StatusUnauthorized
 339 }
 340
 341 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
 342         s.doVhostRequests(c, authzViaCookieValue)
 343 }
 344 func authzViaCookieValue(r *http.Request, tok string) int {
 345         r.AddCookie(&http.Cookie{
 346                 Name:  "arvados_api_token",
 347                 Value: auth.EncodeTokenCookie([]byte(tok)),
 348         })
 349         return http.StatusUnauthorized
 350 }
 351
 352 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
 353         s.doVhostRequests(c, authzViaPath)
 354 }
 355 func authzViaPath(r *http.Request, tok string) int {
 356         r.URL.Path = "/t=" + tok + r.URL.Path
 357         return http.StatusNotFound
 358 }
 359
 360 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
 361         s.doVhostRequests(c, authzViaQueryString)
 362 }
 363 func authzViaQueryString(r *http.Request, tok string) int {
 364         r.URL.RawQuery = "api_token=" + tok
 365         return http.StatusUnauthorized
 366 }
 367
 368 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
 369         s.doVhostRequests(c, authzViaPOST)
 370 }
 371 func authzViaPOST(r *http.Request, tok string) int {
 372         r.Method = "POST"
 373         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 374         r.Body = ioutil.NopCloser(strings.NewReader(
 375                 url.Values{"api_token": {tok}}.Encode()))
 376         return http.StatusUnauthorized
 377 }
 378
 379 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
 380         s.doVhostRequests(c, authzViaPOST)
 381 }
 382 func authzViaXHRPOST(r *http.Request, tok string) int {
 383         r.Method = "POST"
 384         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 385         r.Header.Add("Origin", "https://origin.example")
 386         r.Body = ioutil.NopCloser(strings.NewReader(
 387                 url.Values{
 388                         "api_token":   {tok},
 389                         "disposition": {"attachment"},
 390                 }.Encode()))
 391         return http.StatusUnauthorized
 392 }
 393
 394 // Try some combinations of {url, token} using the given authorization
 395 // mechanism, and verify the result is correct.
 396 func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
 397         for _, hostPath := range []string{
 398                 arvadostest.FooCollection + ".example.com/foo",
 399                 arvadostest.FooCollection + "--collections.example.com/foo",
 400                 arvadostest.FooCollection + "--collections.example.com/_/foo",
 401                 arvadostest.FooCollectionPDH + ".example.com/foo",
 402                 strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + "--collections.example.com/foo",
 403                 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
 404         } {
 405                 c.Log("doRequests: ", hostPath)
 406                 s.doVhostRequestsWithHostPath(c, authz, hostPath)
 407         }
 408 }
 409
 410 func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
 411         for _, tok := range []string{
 412                 arvadostest.ActiveToken,
 413                 arvadostest.ActiveToken[:15],
 414                 arvadostest.SpectatorToken,
 415                 "bogus",
 416                 "",
 417         } {
 418                 u := mustParseURL("http://" + hostPath)
 419                 req := &http.Request{
 420                         Method:     "GET",
 421                         Host:       u.Host,
 422                         URL:        u,
 423                         RequestURI: u.RequestURI(),
 424                         Header:     http.Header{},
 425                 }
 426                 failCode := authz(req, tok)
 427                 req, resp := s.doReq(req)
 428                 code, body := resp.Code, resp.Body.String()
 429
 430                 // If the initial request had a (non-empty) token
 431                 // showing in the query string, we should have been
 432                 // redirected in order to hide it in a cookie.
 433                 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
 434
 435                 if tok == arvadostest.ActiveToken {
 436                         c.Check(code, check.Equals, http.StatusOK)
 437                         c.Check(body, check.Equals, "foo")
 438                 } else {
 439                         c.Check(code >= 400, check.Equals, true)
 440                         c.Check(code < 500, check.Equals, true)
 441                         if tok == arvadostest.SpectatorToken {
 442                                 // Valid token never offers to retry
 443                                 // with different credentials.
 444                                 c.Check(code, check.Equals, http.StatusNotFound)
 445                         } else {
 446                                 // Invalid token can ask to retry
 447                                 // depending on the authz method.
 448                                 c.Check(code, check.Equals, failCode)
 449                         }
 450                         if code == 404 {
 451                                 c.Check(body, check.Equals, notFoundMessage+"\n")
 452                         } else {
 453                                 c.Check(body, check.Equals, unauthorizedMessage+"\n")
 454                         }
 455                 }
 456         }
 457 }
 458
 459 func (s *IntegrationSuite) TestVhostPortMatch(c *check.C) {
 460         for _, host := range []string{"download.example.com", "DOWNLOAD.EXAMPLE.COM"} {
 461                 for _, port := range []string{"80", "443", "8000"} {
 462                         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = fmt.Sprintf("download.example.com:%v", port)
 463                         u := mustParseURL(fmt.Sprintf("http://%v/by_id/%v/foo", host, arvadostest.FooCollection))
 464                         req := &http.Request{
 465                                 Method:     "GET",
 466                                 Host:       u.Host,
 467                                 URL:        u,
 468                                 RequestURI: u.RequestURI(),
 469                                 Header:     http.Header{"Authorization": []string{"Bearer " + arvadostest.ActiveToken}},
 470                         }
 471                         req, resp := s.doReq(req)
 472                         code, _ := resp.Code, resp.Body.String()
 473
 474                         if port == "8000" {
 475                                 c.Check(code, check.Equals, 401)
 476                         } else {
 477                                 c.Check(code, check.Equals, 200)
 478                         }
 479                 }
 480         }
 481 }
 482
 483 func (s *IntegrationSuite) do(method string, urlstring string, token string, hdr http.Header) (*http.Request, *httptest.ResponseRecorder) {
 484         u := mustParseURL(urlstring)
 485         if hdr == nil && token != "" {
 486                 hdr = http.Header{"Authorization": {"Bearer " + token}}
 487         } else if hdr == nil {
 488                 hdr = http.Header{}
 489         } else if token != "" {
 490                 panic("must not pass both token and hdr")
 491         }
 492         return s.doReq(&http.Request{
 493                 Method:     method,
 494                 Host:       u.Host,
 495                 URL:        u,
 496                 RequestURI: u.RequestURI(),
 497                 Header:     hdr,
 498         })
 499 }
 500
 501 func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
 502         resp := httptest.NewRecorder()
 503         s.handler.ServeHTTP(resp, req)
 504         if resp.Code != http.StatusSeeOther {
 505                 return req, resp
 506         }
 507         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 508         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 509         req = &http.Request{
 510                 Method:     "GET",
 511                 Host:       u.Host,
 512                 URL:        u,
 513                 RequestURI: u.RequestURI(),
 514                 Header:     http.Header{},
 515         }
 516         for _, c := range cookies {
 517                 req.AddCookie(c)
 518         }
 519         return s.doReq(req)
 520 }
 521
 522 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
 523         s.testVhostRedirectTokenToCookie(c, "GET",
 524                 arvadostest.FooCollection+".example.com/foo",
 525                 "?api_token="+arvadostest.ActiveToken,
 526                 nil,
 527                 "",
 528                 http.StatusOK,
 529                 "foo",
 530         )
 531 }
 532
 533 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
 534         s.testVhostRedirectTokenToCookie(c, "GET",
 535                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
 536                 "",
 537                 nil,
 538                 "",
 539                 http.StatusOK,
 540                 "foo",
 541         )
 542 }
 543
 544 func (s *IntegrationSuite) TestCollectionSharingToken(c *check.C) {
 545         s.testVhostRedirectTokenToCookie(c, "GET",
 546                 "example.com/c="+arvadostest.FooFileCollectionUUID+"/t="+arvadostest.FooFileCollectionSharingToken+"/foo",
 547                 "",
 548                 nil,
 549                 "",
 550                 http.StatusOK,
 551                 "foo",
 552         )
 553         // Same valid sharing token, but requesting a different collection
 554         s.testVhostRedirectTokenToCookie(c, "GET",
 555                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.FooFileCollectionSharingToken+"/foo",
 556                 "",
 557                 nil,
 558                 "",
 559                 http.StatusNotFound,
 560                 regexp.QuoteMeta(notFoundMessage+"\n"),
 561         )
 562 }
 563
 564 // Bad token in URL is 404 Not Found because it doesn't make sense to
 565 // retry the same URL with different authorization.
 566 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
 567         s.testVhostRedirectTokenToCookie(c, "GET",
 568                 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
 569                 "",
 570                 nil,
 571                 "",
 572                 http.StatusNotFound,
 573                 regexp.QuoteMeta(notFoundMessage+"\n"),
 574         )
 575 }
 576
 577 // Bad token in a cookie (even if it got there via our own
 578 // query-string-to-cookie redirect) is, in principle, retryable via
 579 // wb2-login-and-redirect flow.
 580 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
 581         // Inline
 582         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 583                 arvadostest.FooCollection+".example.com/foo",
 584                 "?api_token=thisisabogustoken",
 585                 http.Header{"Sec-Fetch-Mode": {"navigate"}},
 586                 "",
 587                 http.StatusSeeOther,
 588                 "",
 589         )
 590         u, err := url.Parse(resp.Header().Get("Location"))
 591         c.Assert(err, check.IsNil)
 592         c.Logf("redirected to %s", u)
 593         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 594         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 595         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
 596
 597         // Download/attachment indicated by ?disposition=attachment
 598         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 599                 arvadostest.FooCollection+".example.com/foo",
 600                 "?api_token=thisisabogustoken&disposition=attachment",
 601                 http.Header{"Sec-Fetch-Mode": {"navigate"}},
 602                 "",
 603                 http.StatusSeeOther,
 604                 "",
 605         )
 606         u, err = url.Parse(resp.Header().Get("Location"))
 607         c.Assert(err, check.IsNil)
 608         c.Logf("redirected to %s", u)
 609         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 610         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "")
 611         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 612
 613         // Download/attachment indicated by vhost
 614         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 615                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
 616                 "?api_token=thisisabogustoken",
 617                 http.Header{"Sec-Fetch-Mode": {"navigate"}},
 618                 "",
 619                 http.StatusSeeOther,
 620                 "",
 621         )
 622         u, err = url.Parse(resp.Header().Get("Location"))
 623         c.Assert(err, check.IsNil)
 624         c.Logf("redirected to %s", u)
 625         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 626         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "")
 627         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 628
 629         // Without "Sec-Fetch-Mode: navigate" header, just 401.
 630         s.testVhostRedirectTokenToCookie(c, "GET",
 631                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
 632                 "?api_token=thisisabogustoken",
 633                 http.Header{"Sec-Fetch-Mode": {"cors"}},
 634                 "",
 635                 http.StatusUnauthorized,
 636                 regexp.QuoteMeta(unauthorizedMessage+"\n"),
 637         )
 638         s.testVhostRedirectTokenToCookie(c, "GET",
 639                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
 640                 "?api_token=thisisabogustoken",
 641                 nil,
 642                 "",
 643                 http.StatusUnauthorized,
 644                 regexp.QuoteMeta(unauthorizedMessage+"\n"),
 645         )
 646 }
 647
 648 func (s *IntegrationSuite) TestVhostRedirectWithNoCache(c *check.C) {
 649         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 650                 arvadostest.FooCollection+".example.com/foo",
 651                 "?api_token=thisisabogustoken",
 652                 http.Header{
 653                         "Sec-Fetch-Mode": {"navigate"},
 654                         "Cache-Control":  {"no-cache"},
 655                 },
 656                 "",
 657                 http.StatusSeeOther,
 658                 "",
 659         )
 660         u, err := url.Parse(resp.Header().Get("Location"))
 661         c.Assert(err, check.IsNil)
 662         c.Logf("redirected to %s", u)
 663         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 664         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 665         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
 666 }
 667
 668 func (s *IntegrationSuite) TestNoTokenWorkbench2LoginFlow(c *check.C) {
 669         for _, trial := range []struct {
 670                 anonToken    bool
 671                 cacheControl string
 672         }{
 673                 {},
 674                 {cacheControl: "no-cache"},
 675                 {anonToken: true},
 676                 {anonToken: true, cacheControl: "no-cache"},
 677         } {
 678                 c.Logf("trial: %+v", trial)
 679
 680                 if trial.anonToken {
 681                         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 682                 } else {
 683                         s.handler.Cluster.Users.AnonymousUserToken = ""
 684                 }
 685                 req, err := http.NewRequest("GET", "http://"+arvadostest.FooCollection+".example.com/foo", nil)
 686                 c.Assert(err, check.IsNil)
 687                 req.Header.Set("Sec-Fetch-Mode", "navigate")
 688                 if trial.cacheControl != "" {
 689                         req.Header.Set("Cache-Control", trial.cacheControl)
 690                 }
 691                 resp := httptest.NewRecorder()
 692                 s.handler.ServeHTTP(resp, req)
 693                 c.Check(resp.Code, check.Equals, http.StatusSeeOther)
 694                 u, err := url.Parse(resp.Header().Get("Location"))
 695                 c.Assert(err, check.IsNil)
 696                 c.Logf("redirected to %q", u)
 697                 c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 698                 c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 699                 c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
 700         }
 701 }
 702
 703 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
 704         s.testVhostRedirectTokenToCookie(c, "GET",
 705                 "example.com/c="+arvadostest.FooCollection+"/foo",
 706                 "?api_token="+arvadostest.ActiveToken,
 707                 nil,
 708                 "",
 709                 http.StatusBadRequest,
 710                 regexp.QuoteMeta("cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n"),
 711         )
 712 }
 713
 714 // If client requests an attachment by putting ?disposition=attachment
 715 // in the query string, and gets redirected, the redirect target
 716 // should respond with an attachment.
 717 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
 718         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 719                 arvadostest.FooCollection+".example.com/foo",
 720                 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
 721                 nil,
 722                 "",
 723                 http.StatusOK,
 724                 "foo",
 725         )
 726         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 727 }
 728
 729 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
 730         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 731         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 732                 "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
 733                 "?api_token="+arvadostest.ActiveToken,
 734                 nil,
 735                 "",
 736                 http.StatusOK,
 737                 "foo",
 738         )
 739         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 740 }
 741
 742 func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
 743         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 744         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 745                 "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
 746                 "?api_token="+arvadostest.ActiveToken,
 747                 nil,
 748                 "",
 749                 http.StatusOK,
 750                 "waz",
 751         )
 752         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 753         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 754                 "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz",
 755                 "?api_token="+arvadostest.ActiveToken,
 756                 nil,
 757                 "",
 758                 http.StatusOK,
 759                 "waz",
 760         )
 761         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 762 }
 763
 764 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
 765         s.handler.Cluster.Collections.TrustAllContent = true
 766         s.testVhostRedirectTokenToCookie(c, "GET",
 767                 "example.com/c="+arvadostest.FooCollection+"/foo",
 768                 "?api_token="+arvadostest.ActiveToken,
 769                 nil,
 770                 "",
 771                 http.StatusOK,
 772                 "foo",
 773         )
 774 }
 775
 776 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
 777         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
 778
 779         s.testVhostRedirectTokenToCookie(c, "GET",
 780                 "example.com/c="+arvadostest.FooCollection+"/foo",
 781                 "?api_token="+arvadostest.ActiveToken,
 782                 nil,
 783                 "",
 784                 http.StatusBadRequest,
 785                 regexp.QuoteMeta("cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n"),
 786         )
 787
 788         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 789                 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
 790                 "?api_token="+arvadostest.ActiveToken,
 791                 nil,
 792                 "",
 793                 http.StatusOK,
 794                 "foo",
 795         )
 796         c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
 797 }
 798
 799 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
 800         s.testVhostRedirectTokenToCookie(c, "POST",
 801                 arvadostest.FooCollection+".example.com/foo",
 802                 "",
 803                 http.Header{"Content-Type": {"application/x-www-form-urlencoded"}},
 804                 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
 805                 http.StatusOK,
 806                 "foo",
 807         )
 808 }
 809
 810 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
 811         s.testVhostRedirectTokenToCookie(c, "POST",
 812                 arvadostest.FooCollection+".example.com/foo",
 813                 "",
 814                 http.Header{"Content-Type": {"application/x-www-form-urlencoded"}},
 815                 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
 816                 http.StatusNotFound,
 817                 regexp.QuoteMeta(notFoundMessage+"\n"),
 818         )
 819 }
 820
 821 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
 822         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 823         s.testVhostRedirectTokenToCookie(c, "GET",
 824                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 825                 "",
 826                 nil,
 827                 "",
 828                 http.StatusOK,
 829                 "Hello world\n",
 830         )
 831 }
 832
 833 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
 834         s.handler.Cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
 835         s.testVhostRedirectTokenToCookie(c, "GET",
 836                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 837                 "",
 838                 nil,
 839                 "",
 840                 http.StatusUnauthorized,
 841                 "Authorization tokens are not accepted here: .*\n",
 842         )
 843 }
 844
 845 func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
 846         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 847
 848         client := arvados.NewClientFromEnv()
 849         client.AuthToken = arvadostest.ActiveToken
 850         fs, err := (&arvados.Collection{}).FileSystem(client, nil)
 851         c.Assert(err, check.IsNil)
 852         f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777)
 853         c.Assert(err, check.IsNil)
 854         f.Close()
 855         mtxt, err := fs.MarshalManifest(".")
 856         c.Assert(err, check.IsNil)
 857         var coll arvados.Collection
 858         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 859                 "collection": map[string]string{
 860                         "manifest_text": mtxt,
 861                 },
 862         })
 863         c.Assert(err, check.IsNil)
 864
 865         u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
 866         req := &http.Request{
 867                 Method:     "GET",
 868                 Host:       u.Host,
 869                 URL:        u,
 870                 RequestURI: u.RequestURI(),
 871                 Header: http.Header{
 872                         "Authorization": {"Bearer " + client.AuthToken},
 873                 },
 874         }
 875         resp := httptest.NewRecorder()
 876         s.handler.ServeHTTP(resp, req)
 877         c.Check(resp.Code, check.Equals, http.StatusOK)
 878         c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\&#34;odd&#39; path chars.*`)
 879 }
 880
 881 func (s *IntegrationSuite) TestForwardSlashSubstitution(c *check.C) {
 882         arv := arvados.NewClientFromEnv()
 883         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 884         s.handler.Cluster.Collections.ForwardSlashNameSubstitution = "{SOLIDUS}"
 885         name := "foo/bar/baz"
 886         nameShown := strings.Replace(name, "/", "{SOLIDUS}", -1)
 887         nameShownEscaped := strings.Replace(name, "/", "%7bSOLIDUS%7d", -1)
 888
 889         client := arvados.NewClientFromEnv()
 890         client.AuthToken = arvadostest.ActiveToken
 891         fs, err := (&arvados.Collection{}).FileSystem(client, nil)
 892         c.Assert(err, check.IsNil)
 893         f, err := fs.OpenFile("filename", os.O_CREATE, 0777)
 894         c.Assert(err, check.IsNil)
 895         f.Close()
 896         mtxt, err := fs.MarshalManifest(".")
 897         c.Assert(err, check.IsNil)
 898         var coll arvados.Collection
 899         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 900                 "collection": map[string]string{
 901                         "manifest_text": mtxt,
 902                         "name":          name,
 903                         "owner_uuid":    arvadostest.AProjectUUID,
 904                 },
 905         })
 906         c.Assert(err, check.IsNil)
 907         defer arv.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
 908
 909         base := "http://download.example.com/by_id/" + coll.OwnerUUID + "/"
 910         for tryURL, expectRegexp := range map[string]string{
 911                 base:                          `(?ms).*href="./` + nameShownEscaped + `/"\S+` + nameShown + `.*`,
 912                 base + nameShownEscaped + "/": `(?ms).*href="./filename"\S+filename.*`,
 913         } {
 914                 u, _ := url.Parse(tryURL)
 915                 req := &http.Request{
 916                         Method:     "GET",
 917                         Host:       u.Host,
 918                         URL:        u,
 919                         RequestURI: u.RequestURI(),
 920                         Header: http.Header{
 921                                 "Authorization": {"Bearer " + client.AuthToken},
 922                         },
 923                 }
 924                 resp := httptest.NewRecorder()
 925                 s.handler.ServeHTTP(resp, req)
 926                 c.Check(resp.Code, check.Equals, http.StatusOK)
 927                 c.Check(resp.Body.String(), check.Matches, expectRegexp)
 928         }
 929 }
 930
 931 // XHRs can't follow redirect-with-cookie so they rely on method=POST
 932 // and disposition=attachment (telling us it's acceptable to respond
 933 // with content instead of a redirect) and an Origin header that gets
 934 // added automatically by the browser (telling us it's desirable to do
 935 // so).
 936 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
 937         u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
 938         req := &http.Request{
 939                 Method:     "POST",
 940                 Host:       u.Host,
 941                 URL:        u,
 942                 RequestURI: u.RequestURI(),
 943                 Header: http.Header{
 944                         "Origin":       {"https://origin.example"},
 945                         "Content-Type": {"application/x-www-form-urlencoded"},
 946                 },
 947                 Body: ioutil.NopCloser(strings.NewReader(url.Values{
 948                         "api_token":   {arvadostest.ActiveToken},
 949                         "disposition": {"attachment"},
 950                 }.Encode())),
 951         }
 952         resp := httptest.NewRecorder()
 953         s.handler.ServeHTTP(resp, req)
 954         c.Check(resp.Code, check.Equals, http.StatusOK)
 955         c.Check(resp.Body.String(), check.Equals, "foo")
 956         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
 957
 958         // GET + Origin header is representative of both AJAX GET
 959         // requests and inline images via <IMG crossorigin="anonymous"
 960         // src="...">.
 961         u.RawQuery = "api_token=" + url.QueryEscape(arvadostest.ActiveTokenV2)
 962         req = &http.Request{
 963                 Method:     "GET",
 964                 Host:       u.Host,
 965                 URL:        u,
 966                 RequestURI: u.RequestURI(),
 967                 Header: http.Header{
 968                         "Origin": {"https://origin.example"},
 969                 },
 970         }
 971         resp = httptest.NewRecorder()
 972         s.handler.ServeHTTP(resp, req)
 973         c.Check(resp.Code, check.Equals, http.StatusOK)
 974         c.Check(resp.Body.String(), check.Equals, "foo")
 975         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
 976 }
 977
 978 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString string, reqHeader http.Header, reqBody string, expectStatus int, matchRespBody string) *httptest.ResponseRecorder {
 979         if reqHeader == nil {
 980                 reqHeader = http.Header{}
 981         }
 982         u, _ := url.Parse(`http://` + hostPath + queryString)
 983         c.Logf("requesting %s", u)
 984         req := &http.Request{
 985                 Method:     method,
 986                 Host:       u.Host,
 987                 URL:        u,
 988                 RequestURI: u.RequestURI(),
 989                 Header:     reqHeader,
 990                 Body:       ioutil.NopCloser(strings.NewReader(reqBody)),
 991         }
 992
 993         resp := httptest.NewRecorder()
 994         defer func() {
 995                 c.Check(resp.Code, check.Equals, expectStatus)
 996                 c.Check(resp.Body.String(), check.Matches, matchRespBody)
 997         }()
 998
 999         s.handler.ServeHTTP(resp, req)
1000         if resp.Code != http.StatusSeeOther {
1001                 return resp
1002         }
1003         c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
1004         c.Check(strings.Split(resp.Header().Get("Location"), "?")[0], check.Equals, "http://"+hostPath)
1005         cookies := (&http.Response{Header: resp.Header()}).Cookies()
1006
1007         u, err := u.Parse(resp.Header().Get("Location"))
1008         c.Assert(err, check.IsNil)
1009         c.Logf("following redirect to %s", u)
1010         req = &http.Request{
1011                 Method:     "GET",
1012                 Host:       u.Host,
1013                 URL:        u,
1014                 RequestURI: u.RequestURI(),
1015                 Header:     reqHeader,
1016         }
1017         for _, c := range cookies {
1018                 req.AddCookie(c)
1019         }
1020
1021         resp = httptest.NewRecorder()
1022         s.handler.ServeHTTP(resp, req)
1023
1024         if resp.Code != http.StatusSeeOther {
1025                 c.Check(resp.Header().Get("Location"), check.Equals, "")
1026         }
1027         return resp
1028 }
1029
1030 func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) {
1031         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
1032         s.testDirectoryListing(c)
1033 }
1034
1035 func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) {
1036         s.handler.Cluster.Users.AnonymousUserToken = ""
1037         s.testDirectoryListing(c)
1038 }
1039
1040 func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
1041         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1042         authHeader := http.Header{
1043                 "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
1044         }
1045         for _, trial := range []struct {
1046                 uri      string
1047                 header   http.Header
1048                 expect   []string
1049                 redirect string
1050                 cutDirs  int
1051         }{
1052                 {
1053                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
1054                         header:  authHeader,
1055                         expect:  []string{"dir1/foo", "dir1/bar"},
1056                         cutDirs: 0,
1057                 },
1058                 {
1059                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
1060                         header:  authHeader,
1061                         expect:  []string{"foo", "bar"},
1062                         cutDirs: 1,
1063                 },
1064                 {
1065                         // URLs of this form ignore authHeader, and
1066                         // FooAndBarFilesInDirUUID isn't public, so
1067                         // this returns 401.
1068                         uri:    "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
1069                         header: authHeader,
1070                         expect: nil,
1071                 },
1072                 {
1073                         uri:     "download.example.com/users/active/foo_file_in_dir/",
1074                         header:  authHeader,
1075                         expect:  []string{"dir1/"},
1076                         cutDirs: 3,
1077                 },
1078                 {
1079                         uri:     "download.example.com/users/active/foo_file_in_dir/dir1/",
1080                         header:  authHeader,
1081                         expect:  []string{"bar"},
1082                         cutDirs: 4,
1083                 },
1084                 {
1085                         uri:     "download.example.com/",
1086                         header:  authHeader,
1087                         expect:  []string{"users/"},
1088                         cutDirs: 0,
1089                 },
1090                 {
1091                         uri:      "download.example.com/users",
1092                         header:   authHeader,
1093                         redirect: "/users/",
1094                         expect:   []string{"active/"},
1095                         cutDirs:  1,
1096                 },
1097                 {
1098                         uri:     "download.example.com/users/",
1099                         header:  authHeader,
1100                         expect:  []string{"active/"},
1101                         cutDirs: 1,
1102                 },
1103                 {
1104                         uri:      "download.example.com/users/active",
1105                         header:   authHeader,
1106                         redirect: "/users/active/",
1107                         expect:   []string{"foo_file_in_dir/"},
1108                         cutDirs:  2,
1109                 },
1110                 {
1111                         uri:     "download.example.com/users/active/",
1112                         header:  authHeader,
1113                         expect:  []string{"foo_file_in_dir/"},
1114                         cutDirs: 2,
1115                 },
1116                 {
1117                         uri:     "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
1118                         header:  nil,
1119                         expect:  []string{"dir1/foo", "dir1/bar"},
1120                         cutDirs: 4,
1121                 },
1122                 {
1123                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
1124                         header:  nil,
1125                         expect:  []string{"dir1/foo", "dir1/bar"},
1126                         cutDirs: 2,
1127                 },
1128                 {
1129                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken,
1130                         header:  nil,
1131                         expect:  []string{"dir1/foo", "dir1/bar"},
1132                         cutDirs: 2,
1133                 },
1134                 {
1135                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID,
1136                         header:  authHeader,
1137                         expect:  []string{"dir1/foo", "dir1/bar"},
1138                         cutDirs: 1,
1139                 },
1140                 {
1141                         uri:      "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
1142                         header:   authHeader,
1143                         redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
1144                         expect:   []string{"foo", "bar"},
1145                         cutDirs:  2,
1146                 },
1147                 {
1148                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
1149                         header:  authHeader,
1150                         expect:  []string{"foo", "bar"},
1151                         cutDirs: 3,
1152                 },
1153                 {
1154                         uri:      arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
1155                         header:   authHeader,
1156                         redirect: "/dir1/",
1157                         expect:   []string{"foo", "bar"},
1158                         cutDirs:  1,
1159                 },
1160                 {
1161                         uri:    "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
1162                         header: authHeader,
1163                         expect: nil,
1164                 },
1165                 {
1166                         uri:     "download.example.com/c=" + arvadostest.WazVersion1Collection,
1167                         header:  authHeader,
1168                         expect:  []string{"waz"},
1169                         cutDirs: 1,
1170                 },
1171                 {
1172                         uri:     "download.example.com/by_id/" + arvadostest.WazVersion1Collection,
1173                         header:  authHeader,
1174                         expect:  []string{"waz"},
1175                         cutDirs: 2,
1176                 },
1177         } {
1178                 comment := check.Commentf("HTML: %q => %q", trial.uri, trial.expect)
1179                 resp := httptest.NewRecorder()
1180                 u := mustParseURL("//" + trial.uri)
1181                 req := &http.Request{
1182                         Method:     "GET",
1183                         Host:       u.Host,
1184                         URL:        u,
1185                         RequestURI: u.RequestURI(),
1186                         Header:     copyHeader(trial.header),
1187                 }
1188                 s.handler.ServeHTTP(resp, req)
1189                 var cookies []*http.Cookie
1190                 for resp.Code == http.StatusSeeOther {
1191                         u, _ := req.URL.Parse(resp.Header().Get("Location"))
1192                         req = &http.Request{
1193                                 Method:     "GET",
1194                                 Host:       u.Host,
1195                                 URL:        u,
1196                                 RequestURI: u.RequestURI(),
1197                                 Header:     copyHeader(trial.header),
1198                         }
1199                         cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
1200                         for _, c := range cookies {
1201                                 req.AddCookie(c)
1202                         }
1203                         resp = httptest.NewRecorder()
1204                         s.handler.ServeHTTP(resp, req)
1205                 }
1206                 if trial.redirect != "" {
1207                         c.Check(req.URL.Path, check.Equals, trial.redirect, comment)
1208                 }
1209                 if trial.expect == nil {
1210                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1211                 } else {
1212                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
1213                         for _, e := range trial.expect {
1214                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`, comment)
1215                         }
1216                         c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`, comment)
1217                 }
1218
1219                 comment = check.Commentf("WebDAV: %q => %q", trial.uri, trial.expect)
1220                 req = &http.Request{
1221                         Method:     "OPTIONS",
1222                         Host:       u.Host,
1223                         URL:        u,
1224                         RequestURI: u.RequestURI(),
1225                         Header:     copyHeader(trial.header),
1226                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
1227                 }
1228                 resp = httptest.NewRecorder()
1229                 s.handler.ServeHTTP(resp, req)
1230                 if trial.expect == nil {
1231                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1232                 } else {
1233                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
1234                 }
1235
1236                 req = &http.Request{
1237                         Method:     "PROPFIND",
1238                         Host:       u.Host,
1239                         URL:        u,
1240                         RequestURI: u.RequestURI(),
1241                         Header:     copyHeader(trial.header),
1242                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
1243                 }
1244                 resp = httptest.NewRecorder()
1245                 s.handler.ServeHTTP(resp, req)
1246                 if trial.expect == nil {
1247                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1248                 } else {
1249                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment)
1250                         for _, e := range trial.expect {
1251                                 if strings.HasSuffix(e, "/") {
1252                                         e = filepath.Join(u.Path, e) + "/"
1253                                 } else {
1254                                         e = filepath.Join(u.Path, e)
1255                                 }
1256                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+e+`</D:href>.*`, comment)
1257                         }
1258                 }
1259         }
1260 }
1261
1262 func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
1263         arv := arvados.NewClientFromEnv()
1264         var newCollection arvados.Collection
1265         err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1266                 "collection": map[string]string{
1267                         "owner_uuid":    arvadostest.ActiveUserUUID,
1268                         "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
1269                         "name":          "keep-web test collection",
1270                 },
1271                 "ensure_unique_name": true,
1272         })
1273         c.Assert(err, check.IsNil)
1274         defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
1275
1276         var updated arvados.Collection
1277         for _, fnm := range []string{"foo.txt", "bar.txt"} {
1278                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
1279                 u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
1280                 req := &http.Request{
1281                         Method:     "DELETE",
1282                         Host:       u.Host,
1283                         URL:        u,
1284                         RequestURI: u.RequestURI(),
1285                         Header: http.Header{
1286                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
1287                         },
1288                 }
1289                 resp := httptest.NewRecorder()
1290                 s.handler.ServeHTTP(resp, req)
1291                 c.Check(resp.Code, check.Equals, http.StatusNoContent)
1292
1293                 updated = arvados.Collection{}
1294                 err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
1295                 c.Check(err, check.IsNil)
1296                 c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`)
1297                 c.Logf("updated manifest_text %q", updated.ManifestText)
1298         }
1299         c.Check(updated.ManifestText, check.Equals, "")
1300 }
1301
1302 func (s *IntegrationSuite) TestFileContentType(c *check.C) {
1303         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1304
1305         client := arvados.NewClientFromEnv()
1306         client.AuthToken = arvadostest.ActiveToken
1307         arv, err := arvadosclient.New(client)
1308         c.Assert(err, check.Equals, nil)
1309         kc, err := keepclient.MakeKeepClient(arv)
1310         c.Assert(err, check.Equals, nil)
1311
1312         fs, err := (&arvados.Collection{}).FileSystem(client, kc)
1313         c.Assert(err, check.IsNil)
1314
1315         trials := []struct {
1316                 filename    string
1317                 content     string
1318                 contentType string
1319         }{
1320                 {"picture.txt", "BMX bikes are small this year\n", "text/plain; charset=utf-8"},
1321                 {"picture.bmp", "BMX bikes are small this year\n", "image/(x-ms-)?bmp"},
1322                 {"picture.jpg", "BMX bikes are small this year\n", "image/jpeg"},
1323                 {"picture1", "BMX bikes are small this year\n", "image/bmp"},            // content sniff; "BM" is the magic signature for .bmp
1324                 {"picture2", "Cars are small this year\n", "text/plain; charset=utf-8"}, // content sniff
1325         }
1326         for _, trial := range trials {
1327                 f, err := fs.OpenFile(trial.filename, os.O_CREATE|os.O_WRONLY, 0777)
1328                 c.Assert(err, check.IsNil)
1329                 _, err = f.Write([]byte(trial.content))
1330                 c.Assert(err, check.IsNil)
1331                 c.Assert(f.Close(), check.IsNil)
1332         }
1333         mtxt, err := fs.MarshalManifest(".")
1334         c.Assert(err, check.IsNil)
1335         var coll arvados.Collection
1336         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1337                 "collection": map[string]string{
1338                         "manifest_text": mtxt,
1339                 },
1340         })
1341         c.Assert(err, check.IsNil)
1342
1343         for _, trial := range trials {
1344                 u, _ := url.Parse("http://download.example.com/by_id/" + coll.UUID + "/" + trial.filename)
1345                 req := &http.Request{
1346                         Method:     "GET",
1347                         Host:       u.Host,
1348                         URL:        u,
1349                         RequestURI: u.RequestURI(),
1350                         Header: http.Header{
1351                                 "Authorization": {"Bearer " + client.AuthToken},
1352                         },
1353                 }
1354                 resp := httptest.NewRecorder()
1355                 s.handler.ServeHTTP(resp, req)
1356                 c.Check(resp.Code, check.Equals, http.StatusOK)
1357                 c.Check(resp.Header().Get("Content-Type"), check.Matches, trial.contentType)
1358                 c.Check(resp.Body.String(), check.Equals, trial.content)
1359         }
1360 }
1361
1362 func (s *IntegrationSuite) TestKeepClientBlockCache(c *check.C) {
1363         s.handler.Cluster.Collections.WebDAVCache.MaxBlockEntries = 42
1364         c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Not(check.Equals), 42)
1365         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/t=" + arvadostest.ActiveToken + "/foo")
1366         req := &http.Request{
1367                 Method:     "GET",
1368                 Host:       u.Host,
1369                 URL:        u,
1370                 RequestURI: u.RequestURI(),
1371         }
1372         resp := httptest.NewRecorder()
1373         s.handler.ServeHTTP(resp, req)
1374         c.Check(resp.Code, check.Equals, http.StatusOK)
1375         c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Equals, 42)
1376 }
1377
1378 // Writing to a collection shouldn't affect its entry in the
1379 // PDH-to-manifest cache.
1380 func (s *IntegrationSuite) TestCacheWriteCollectionSamePDH(c *check.C) {
1381         arv, err := arvadosclient.MakeArvadosClient()
1382         c.Assert(err, check.Equals, nil)
1383         arv.ApiToken = arvadostest.ActiveToken
1384
1385         u := mustParseURL("http://x.example/testfile")
1386         req := &http.Request{
1387                 Method:     "GET",
1388                 Host:       u.Host,
1389                 URL:        u,
1390                 RequestURI: u.RequestURI(),
1391                 Header:     http.Header{"Authorization": {"Bearer " + arv.ApiToken}},
1392         }
1393
1394         checkWithID := func(id string, status int) {
1395                 req.URL.Host = strings.Replace(id, "+", "-", -1) + ".example"
1396                 req.Host = req.URL.Host
1397                 resp := httptest.NewRecorder()
1398                 s.handler.ServeHTTP(resp, req)
1399                 c.Check(resp.Code, check.Equals, status)
1400         }
1401
1402         var colls [2]arvados.Collection
1403         for i := range colls {
1404                 err := arv.Create("collections",
1405                         map[string]interface{}{
1406                                 "ensure_unique_name": true,
1407                                 "collection": map[string]interface{}{
1408                                         "name": "test collection",
1409                                 },
1410                         }, &colls[i])
1411                 c.Assert(err, check.Equals, nil)
1412         }
1413
1414         // Populate cache with empty collection
1415         checkWithID(colls[0].PortableDataHash, http.StatusNotFound)
1416
1417         // write a file to colls[0]
1418         reqPut := *req
1419         reqPut.Method = "PUT"
1420         reqPut.URL.Host = colls[0].UUID + ".example"
1421         reqPut.Host = req.URL.Host
1422         reqPut.Body = ioutil.NopCloser(bytes.NewBufferString("testdata"))
1423         resp := httptest.NewRecorder()
1424         s.handler.ServeHTTP(resp, &reqPut)
1425         c.Check(resp.Code, check.Equals, http.StatusCreated)
1426
1427         // new file should not appear in colls[1]
1428         checkWithID(colls[1].PortableDataHash, http.StatusNotFound)
1429         checkWithID(colls[1].UUID, http.StatusNotFound)
1430
1431         checkWithID(colls[0].UUID, http.StatusOK)
1432 }
1433
1434 func copyHeader(h http.Header) http.Header {
1435         hc := http.Header{}
1436         for k, v := range h {
1437                 hc[k] = append([]string(nil), v...)
1438         }
1439         return hc
1440 }
1441
1442 func (s *IntegrationSuite) checkUploadDownloadRequest(c *check.C, req *http.Request,
1443         successCode int, direction string, perm bool, userUuid, collectionUuid, collectionPDH, filepath string) {
1444
1445         client := arvados.NewClientFromEnv()
1446         client.AuthToken = arvadostest.AdminToken
1447         var logentries arvados.LogList
1448         limit1 := 1
1449         err := client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil,
1450                 arvados.ResourceListParams{
1451                         Limit: &limit1,
1452                         Order: "created_at desc"})
1453         c.Check(err, check.IsNil)
1454         c.Check(logentries.Items, check.HasLen, 1)
1455         lastLogId := logentries.Items[0].ID
1456         c.Logf("lastLogId: %d", lastLogId)
1457
1458         var logbuf bytes.Buffer
1459         logger := logrus.New()
1460         logger.Out = &logbuf
1461         resp := httptest.NewRecorder()
1462         req = req.WithContext(ctxlog.Context(context.Background(), logger))
1463         s.handler.ServeHTTP(resp, req)
1464
1465         if perm {
1466                 c.Check(resp.Result().StatusCode, check.Equals, successCode)
1467                 c.Check(logbuf.String(), check.Matches, `(?ms).*msg="File `+direction+`".*`)
1468                 c.Check(logbuf.String(), check.Not(check.Matches), `(?ms).*level=error.*`)
1469
1470                 deadline := time.Now().Add(time.Second)
1471                 for {
1472                         c.Assert(time.Now().After(deadline), check.Equals, false, check.Commentf("timed out waiting for log entry"))
1473                         logentries = arvados.LogList{}
1474                         err = client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil,
1475                                 arvados.ResourceListParams{
1476                                         Filters: []arvados.Filter{
1477                                                 {Attr: "event_type", Operator: "=", Operand: "file_" + direction},
1478                                                 {Attr: "object_uuid", Operator: "=", Operand: userUuid},
1479                                         },
1480                                         Limit: &limit1,
1481                                         Order: "created_at desc",
1482                                 })
1483                         c.Assert(err, check.IsNil)
1484                         if len(logentries.Items) > 0 &&
1485                                 logentries.Items[0].ID > lastLogId &&
1486                                 logentries.Items[0].ObjectUUID == userUuid &&
1487                                 logentries.Items[0].Properties["collection_uuid"] == collectionUuid &&
1488                                 (collectionPDH == "" || logentries.Items[0].Properties["portable_data_hash"] == collectionPDH) &&
1489                                 logentries.Items[0].Properties["collection_file_path"] == filepath {
1490                                 break
1491                         }
1492                         c.Logf("logentries.Items: %+v", logentries.Items)
1493                         time.Sleep(50 * time.Millisecond)
1494                 }
1495         } else {
1496                 c.Check(resp.Result().StatusCode, check.Equals, http.StatusForbidden)
1497                 c.Check(logbuf.String(), check.Equals, "")
1498         }
1499 }
1500
1501 func (s *IntegrationSuite) TestDownloadLoggingPermission(c *check.C) {
1502         u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
1503
1504         s.handler.Cluster.Collections.TrustAllContent = true
1505
1506         for _, adminperm := range []bool{true, false} {
1507                 for _, userperm := range []bool{true, false} {
1508                         s.handler.Cluster.Collections.WebDAVPermission.Admin.Download = adminperm
1509                         s.handler.Cluster.Collections.WebDAVPermission.User.Download = userperm
1510
1511                         // Test admin permission
1512                         req := &http.Request{
1513                                 Method:     "GET",
1514                                 Host:       u.Host,
1515                                 URL:        u,
1516                                 RequestURI: u.RequestURI(),
1517                                 Header: http.Header{
1518                                         "Authorization": {"Bearer " + arvadostest.AdminToken},
1519                                 },
1520                         }
1521                         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", adminperm,
1522                                 arvadostest.AdminUserUUID, arvadostest.FooCollection, arvadostest.FooCollectionPDH, "foo")
1523
1524                         // Test user permission
1525                         req = &http.Request{
1526                                 Method:     "GET",
1527                                 Host:       u.Host,
1528                                 URL:        u,
1529                                 RequestURI: u.RequestURI(),
1530                                 Header: http.Header{
1531                                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
1532                                 },
1533                         }
1534                         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", userperm,
1535                                 arvadostest.ActiveUserUUID, arvadostest.FooCollection, arvadostest.FooCollectionPDH, "foo")
1536                 }
1537         }
1538
1539         s.handler.Cluster.Collections.WebDAVPermission.User.Download = true
1540
1541         for _, tryurl := range []string{"http://" + arvadostest.MultilevelCollection1 + ".keep-web.example/dir1/subdir/file1",
1542                 "http://keep-web/users/active/multilevel_collection_1/dir1/subdir/file1"} {
1543
1544                 u = mustParseURL(tryurl)
1545                 req := &http.Request{
1546                         Method:     "GET",
1547                         Host:       u.Host,
1548                         URL:        u,
1549                         RequestURI: u.RequestURI(),
1550                         Header: http.Header{
1551                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
1552                         },
1553                 }
1554                 s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true,
1555                         arvadostest.ActiveUserUUID, arvadostest.MultilevelCollection1, arvadostest.MultilevelCollection1PDH, "dir1/subdir/file1")
1556         }
1557
1558         u = mustParseURL("http://" + strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + ".keep-web.example/foo")
1559         req := &http.Request{
1560                 Method:     "GET",
1561                 Host:       u.Host,
1562                 URL:        u,
1563                 RequestURI: u.RequestURI(),
1564                 Header: http.Header{
1565                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
1566                 },
1567         }
1568         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true,
1569                 arvadostest.ActiveUserUUID, "", arvadostest.FooCollectionPDH, "foo")
1570 }
1571
1572 func (s *IntegrationSuite) TestUploadLoggingPermission(c *check.C) {
1573         for _, adminperm := range []bool{true, false} {
1574                 for _, userperm := range []bool{true, false} {
1575
1576                         arv := arvados.NewClientFromEnv()
1577                         arv.AuthToken = arvadostest.ActiveToken
1578
1579                         var coll arvados.Collection
1580                         err := arv.RequestAndDecode(&coll,
1581                                 "POST",
1582                                 "/arvados/v1/collections",
1583                                 nil,
1584                                 map[string]interface{}{
1585                                         "ensure_unique_name": true,
1586                                         "collection": map[string]interface{}{
1587                                                 "name": "test collection",
1588                                         },
1589                                 })
1590                         c.Assert(err, check.Equals, nil)
1591
1592                         u := mustParseURL("http://" + coll.UUID + ".keep-web.example/bar")
1593
1594                         s.handler.Cluster.Collections.WebDAVPermission.Admin.Upload = adminperm
1595                         s.handler.Cluster.Collections.WebDAVPermission.User.Upload = userperm
1596
1597                         // Test admin permission
1598                         req := &http.Request{
1599                                 Method:     "PUT",
1600                                 Host:       u.Host,
1601                                 URL:        u,
1602                                 RequestURI: u.RequestURI(),
1603                                 Header: http.Header{
1604                                         "Authorization": {"Bearer " + arvadostest.AdminToken},
1605                                 },
1606                                 Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
1607                         }
1608                         s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", adminperm,
1609                                 arvadostest.AdminUserUUID, coll.UUID, "", "bar")
1610
1611                         // Test user permission
1612                         req = &http.Request{
1613                                 Method:     "PUT",
1614                                 Host:       u.Host,
1615                                 URL:        u,
1616                                 RequestURI: u.RequestURI(),
1617                                 Header: http.Header{
1618                                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
1619                                 },
1620                                 Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
1621                         }
1622                         s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", userperm,
1623                                 arvadostest.ActiveUserUUID, coll.UUID, "", "bar")
1624                 }
1625         }
1626 }