services/keep-web/handler_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package keepweb
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "fmt"
  11         "html"
  12         "io"
  13         "io/ioutil"
  14         "net/http"
  15         "net/http/httptest"
  16         "net/url"
  17         "os"
  18         "path/filepath"
  19         "regexp"
  20         "strings"
  21         "time"
  22
  23         "git.arvados.org/arvados.git/lib/config"
  24         "git.arvados.org/arvados.git/sdk/go/arvados"
  25         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  26         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  27         "git.arvados.org/arvados.git/sdk/go/auth"
  28         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  29         "git.arvados.org/arvados.git/sdk/go/keepclient"
  30         "github.com/prometheus/client_golang/prometheus"
  31         "github.com/sirupsen/logrus"
  32         check "gopkg.in/check.v1"
  33 )
  34
  35 var _ = check.Suite(&UnitSuite{})
  36
  37 func init() {
  38         arvados.DebugLocksPanicMode = true
  39 }
  40
  41 type UnitSuite struct {
  42         cluster *arvados.Cluster
  43         handler *handler
  44 }
  45
  46 func (s *UnitSuite) SetUpTest(c *check.C) {
  47         logger := ctxlog.TestLogger(c)
  48         ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), logger)
  49         ldr.Path = "-"
  50         cfg, err := ldr.Load()
  51         c.Assert(err, check.IsNil)
  52         cc, err := cfg.GetCluster("")
  53         c.Assert(err, check.IsNil)
  54         s.cluster = cc
  55         s.handler = &handler{
  56                 Cluster: cc,
  57                 Cache: cache{
  58                         cluster:  cc,
  59                         logger:   logger,
  60                         registry: prometheus.NewRegistry(),
  61                 },
  62         }
  63 }
  64
  65 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
  66         h := s.handler
  67         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
  68         req := &http.Request{
  69                 Method:     "OPTIONS",
  70                 Host:       u.Host,
  71                 URL:        u,
  72                 RequestURI: u.RequestURI(),
  73                 Header: http.Header{
  74                         "Origin":                        {"https://workbench.example"},
  75                         "Access-Control-Request-Method": {"POST"},
  76                 },
  77         }
  78
  79         // Check preflight for an allowed request
  80         resp := httptest.NewRecorder()
  81         h.ServeHTTP(resp, req)
  82         c.Check(resp.Code, check.Equals, http.StatusOK)
  83         c.Check(resp.Body.String(), check.Equals, "")
  84         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
  85         c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
  86         c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout")
  87
  88         // Check preflight for a disallowed request
  89         resp = httptest.NewRecorder()
  90         req.Header.Set("Access-Control-Request-Method", "MAKE-COFFEE")
  91         h.ServeHTTP(resp, req)
  92         c.Check(resp.Body.String(), check.Equals, "")
  93         c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
  94 }
  95
  96 func (s *UnitSuite) TestEmptyResponse(c *check.C) {
  97         for _, trial := range []struct {
  98                 dataExists    bool
  99                 sendIMSHeader bool
 100                 expectStatus  int
 101                 logRegexp     string
 102         }{
 103                 // If we return no content due to a Keep read error,
 104                 // we should emit a log message.
 105                 {false, false, http.StatusOK, `(?ms).*only wrote 0 bytes.*`},
 106
 107                 // If we return no content because the client sent an
 108                 // If-Modified-Since header, our response should be
 109                 // 304.  We still expect a "File download" log since it
 110                 // counts as a file access for auditing.
 111                 {true, true, http.StatusNotModified, `(?ms).*msg="File download".*`},
 112         } {
 113                 c.Logf("trial: %+v", trial)
 114                 arvadostest.StartKeep(2, true)
 115                 if trial.dataExists {
 116                         arv, err := arvadosclient.MakeArvadosClient()
 117                         c.Assert(err, check.IsNil)
 118                         arv.ApiToken = arvadostest.ActiveToken
 119                         kc, err := keepclient.MakeKeepClient(arv)
 120                         c.Assert(err, check.IsNil)
 121                         _, _, err = kc.PutB([]byte("foo"))
 122                         c.Assert(err, check.IsNil)
 123                 }
 124
 125                 u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
 126                 req := &http.Request{
 127                         Method:     "GET",
 128                         Host:       u.Host,
 129                         URL:        u,
 130                         RequestURI: u.RequestURI(),
 131                         Header: http.Header{
 132                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
 133                         },
 134                 }
 135                 if trial.sendIMSHeader {
 136                         req.Header.Set("If-Modified-Since", strings.Replace(time.Now().UTC().Format(time.RFC1123), "UTC", "GMT", -1))
 137                 }
 138
 139                 var logbuf bytes.Buffer
 140                 logger := logrus.New()
 141                 logger.Out = &logbuf
 142                 req = req.WithContext(ctxlog.Context(context.Background(), logger))
 143
 144                 resp := httptest.NewRecorder()
 145                 s.handler.ServeHTTP(resp, req)
 146                 c.Check(resp.Code, check.Equals, trial.expectStatus)
 147                 c.Check(resp.Body.String(), check.Equals, "")
 148
 149                 c.Log(logbuf.String())
 150                 c.Check(logbuf.String(), check.Matches, trial.logRegexp)
 151         }
 152 }
 153
 154 func (s *UnitSuite) TestInvalidUUID(c *check.C) {
 155         bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-"
 156         token := arvadostest.ActiveToken
 157         for _, trial := range []string{
 158                 "http://keep-web/c=" + bogusID + "/foo",
 159                 "http://keep-web/c=" + bogusID + "/t=" + token + "/foo",
 160                 "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo",
 161                 "http://keep-web/collections/" + bogusID + "/foo",
 162                 "http://" + bogusID + ".keep-web/" + bogusID + "/foo",
 163                 "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
 164         } {
 165                 c.Log(trial)
 166                 u := mustParseURL(trial)
 167                 req := &http.Request{
 168                         Method:     "GET",
 169                         Host:       u.Host,
 170                         URL:        u,
 171                         RequestURI: u.RequestURI(),
 172                 }
 173                 resp := httptest.NewRecorder()
 174                 s.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 175                 s.handler.ServeHTTP(resp, req)
 176                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 177         }
 178 }
 179
 180 func mustParseURL(s string) *url.URL {
 181         r, err := url.Parse(s)
 182         if err != nil {
 183                 panic("parse URL: " + s)
 184         }
 185         return r
 186 }
 187
 188 func (s *IntegrationSuite) TestVhost404(c *check.C) {
 189         for _, testURL := range []string{
 190                 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
 191                 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
 192         } {
 193                 resp := httptest.NewRecorder()
 194                 u := mustParseURL(testURL)
 195                 req := &http.Request{
 196                         Method:     "GET",
 197                         URL:        u,
 198                         RequestURI: u.RequestURI(),
 199                 }
 200                 s.handler.ServeHTTP(resp, req)
 201                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 202                 c.Check(resp.Body.String(), check.Equals, notFoundMessage+"\n")
 203         }
 204 }
 205
 206 // An authorizer modifies an HTTP request to make use of the given
 207 // token -- by adding it to a header, cookie, query param, or whatever
 208 // -- and returns the HTTP status code we should expect from keep-web if
 209 // the token is invalid.
 210 type authorizer func(*http.Request, string) int
 211
 212 func (s *IntegrationSuite) TestVhostViaAuthzHeaderOAuth2(c *check.C) {
 213         s.doVhostRequests(c, authzViaAuthzHeaderOAuth2)
 214 }
 215 func authzViaAuthzHeaderOAuth2(r *http.Request, tok string) int {
 216         r.Header.Add("Authorization", "Bearer "+tok)
 217         return http.StatusUnauthorized
 218 }
 219 func (s *IntegrationSuite) TestVhostViaAuthzHeaderBearer(c *check.C) {
 220         s.doVhostRequests(c, authzViaAuthzHeaderBearer)
 221 }
 222 func authzViaAuthzHeaderBearer(r *http.Request, tok string) int {
 223         r.Header.Add("Authorization", "Bearer "+tok)
 224         return http.StatusUnauthorized
 225 }
 226
 227 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
 228         s.doVhostRequests(c, authzViaCookieValue)
 229 }
 230 func authzViaCookieValue(r *http.Request, tok string) int {
 231         r.AddCookie(&http.Cookie{
 232                 Name:  "arvados_api_token",
 233                 Value: auth.EncodeTokenCookie([]byte(tok)),
 234         })
 235         return http.StatusUnauthorized
 236 }
 237
 238 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
 239         s.doVhostRequests(c, authzViaPath)
 240 }
 241 func authzViaPath(r *http.Request, tok string) int {
 242         r.URL.Path = "/t=" + tok + r.URL.Path
 243         return http.StatusNotFound
 244 }
 245
 246 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
 247         s.doVhostRequests(c, authzViaQueryString)
 248 }
 249 func authzViaQueryString(r *http.Request, tok string) int {
 250         r.URL.RawQuery = "api_token=" + tok
 251         return http.StatusUnauthorized
 252 }
 253
 254 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
 255         s.doVhostRequests(c, authzViaPOST)
 256 }
 257 func authzViaPOST(r *http.Request, tok string) int {
 258         r.Method = "POST"
 259         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 260         r.Body = ioutil.NopCloser(strings.NewReader(
 261                 url.Values{"api_token": {tok}}.Encode()))
 262         return http.StatusUnauthorized
 263 }
 264
 265 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
 266         s.doVhostRequests(c, authzViaPOST)
 267 }
 268 func authzViaXHRPOST(r *http.Request, tok string) int {
 269         r.Method = "POST"
 270         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 271         r.Header.Add("Origin", "https://origin.example")
 272         r.Body = ioutil.NopCloser(strings.NewReader(
 273                 url.Values{
 274                         "api_token":   {tok},
 275                         "disposition": {"attachment"},
 276                 }.Encode()))
 277         return http.StatusUnauthorized
 278 }
 279
 280 // Try some combinations of {url, token} using the given authorization
 281 // mechanism, and verify the result is correct.
 282 func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
 283         for _, hostPath := range []string{
 284                 arvadostest.FooCollection + ".example.com/foo",
 285                 arvadostest.FooCollection + "--collections.example.com/foo",
 286                 arvadostest.FooCollection + "--collections.example.com/_/foo",
 287                 arvadostest.FooCollectionPDH + ".example.com/foo",
 288                 strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + "--collections.example.com/foo",
 289                 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
 290         } {
 291                 c.Log("doRequests: ", hostPath)
 292                 s.doVhostRequestsWithHostPath(c, authz, hostPath)
 293         }
 294 }
 295
 296 func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
 297         for _, tok := range []string{
 298                 arvadostest.ActiveToken,
 299                 arvadostest.ActiveToken[:15],
 300                 arvadostest.SpectatorToken,
 301                 "bogus",
 302                 "",
 303         } {
 304                 u := mustParseURL("http://" + hostPath)
 305                 req := &http.Request{
 306                         Method:     "GET",
 307                         Host:       u.Host,
 308                         URL:        u,
 309                         RequestURI: u.RequestURI(),
 310                         Header:     http.Header{},
 311                 }
 312                 failCode := authz(req, tok)
 313                 req, resp := s.doReq(req)
 314                 code, body := resp.Code, resp.Body.String()
 315
 316                 // If the initial request had a (non-empty) token
 317                 // showing in the query string, we should have been
 318                 // redirected in order to hide it in a cookie.
 319                 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
 320
 321                 if tok == arvadostest.ActiveToken {
 322                         c.Check(code, check.Equals, http.StatusOK)
 323                         c.Check(body, check.Equals, "foo")
 324                 } else {
 325                         c.Check(code >= 400, check.Equals, true)
 326                         c.Check(code < 500, check.Equals, true)
 327                         if tok == arvadostest.SpectatorToken {
 328                                 // Valid token never offers to retry
 329                                 // with different credentials.
 330                                 c.Check(code, check.Equals, http.StatusNotFound)
 331                         } else {
 332                                 // Invalid token can ask to retry
 333                                 // depending on the authz method.
 334                                 c.Check(code, check.Equals, failCode)
 335                         }
 336                         if code == 404 {
 337                                 c.Check(body, check.Equals, notFoundMessage+"\n")
 338                         } else {
 339                                 c.Check(body, check.Equals, unauthorizedMessage+"\n")
 340                         }
 341                 }
 342         }
 343 }
 344
 345 func (s *IntegrationSuite) TestVhostPortMatch(c *check.C) {
 346         for _, host := range []string{"download.example.com", "DOWNLOAD.EXAMPLE.COM"} {
 347                 for _, port := range []string{"80", "443", "8000"} {
 348                         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = fmt.Sprintf("download.example.com:%v", port)
 349                         u := mustParseURL(fmt.Sprintf("http://%v/by_id/%v/foo", host, arvadostest.FooCollection))
 350                         req := &http.Request{
 351                                 Method:     "GET",
 352                                 Host:       u.Host,
 353                                 URL:        u,
 354                                 RequestURI: u.RequestURI(),
 355                                 Header:     http.Header{"Authorization": []string{"Bearer " + arvadostest.ActiveToken}},
 356                         }
 357                         req, resp := s.doReq(req)
 358                         code, _ := resp.Code, resp.Body.String()
 359
 360                         if port == "8000" {
 361                                 c.Check(code, check.Equals, 401)
 362                         } else {
 363                                 c.Check(code, check.Equals, 200)
 364                         }
 365                 }
 366         }
 367 }
 368
 369 func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
 370         resp := httptest.NewRecorder()
 371         s.handler.ServeHTTP(resp, req)
 372         if resp.Code != http.StatusSeeOther {
 373                 return req, resp
 374         }
 375         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 376         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 377         req = &http.Request{
 378                 Method:     "GET",
 379                 Host:       u.Host,
 380                 URL:        u,
 381                 RequestURI: u.RequestURI(),
 382                 Header:     http.Header{},
 383         }
 384         for _, c := range cookies {
 385                 req.AddCookie(c)
 386         }
 387         return s.doReq(req)
 388 }
 389
 390 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
 391         s.testVhostRedirectTokenToCookie(c, "GET",
 392                 arvadostest.FooCollection+".example.com/foo",
 393                 "?api_token="+arvadostest.ActiveToken,
 394                 nil,
 395                 "",
 396                 http.StatusOK,
 397                 "foo",
 398         )
 399 }
 400
 401 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
 402         s.testVhostRedirectTokenToCookie(c, "GET",
 403                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
 404                 "",
 405                 nil,
 406                 "",
 407                 http.StatusOK,
 408                 "foo",
 409         )
 410 }
 411
 412 // Bad token in URL is 404 Not Found because it doesn't make sense to
 413 // retry the same URL with different authorization.
 414 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
 415         s.testVhostRedirectTokenToCookie(c, "GET",
 416                 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
 417                 "",
 418                 nil,
 419                 "",
 420                 http.StatusNotFound,
 421                 notFoundMessage+"\n",
 422         )
 423 }
 424
 425 // Bad token in a cookie (even if it got there via our own
 426 // query-string-to-cookie redirect) is, in principle, retryable via
 427 // wb2-login-and-redirect flow.
 428 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
 429         // Inline
 430         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 431                 arvadostest.FooCollection+".example.com/foo",
 432                 "?api_token=thisisabogustoken",
 433                 http.Header{"Sec-Fetch-Mode": {"navigate"}},
 434                 "",
 435                 http.StatusSeeOther,
 436                 "",
 437         )
 438         u, err := url.Parse(resp.Header().Get("Location"))
 439         c.Assert(err, check.IsNil)
 440         c.Logf("redirected to %s", u)
 441         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 442         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 443         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
 444
 445         // Download/attachment indicated by ?disposition=attachment
 446         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 447                 arvadostest.FooCollection+".example.com/foo",
 448                 "?api_token=thisisabogustoken&disposition=attachment",
 449                 http.Header{"Sec-Fetch-Mode": {"navigate"}},
 450                 "",
 451                 http.StatusSeeOther,
 452                 "",
 453         )
 454         u, err = url.Parse(resp.Header().Get("Location"))
 455         c.Assert(err, check.IsNil)
 456         c.Logf("redirected to %s", u)
 457         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 458         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "")
 459         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 460
 461         // Download/attachment indicated by vhost
 462         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 463                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
 464                 "?api_token=thisisabogustoken",
 465                 http.Header{"Sec-Fetch-Mode": {"navigate"}},
 466                 "",
 467                 http.StatusSeeOther,
 468                 "",
 469         )
 470         u, err = url.Parse(resp.Header().Get("Location"))
 471         c.Assert(err, check.IsNil)
 472         c.Logf("redirected to %s", u)
 473         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 474         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "")
 475         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 476
 477         // Without "Sec-Fetch-Mode: navigate" header, just 401.
 478         s.testVhostRedirectTokenToCookie(c, "GET",
 479                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
 480                 "?api_token=thisisabogustoken",
 481                 http.Header{"Sec-Fetch-Mode": {"cors"}},
 482                 "",
 483                 http.StatusUnauthorized,
 484                 unauthorizedMessage+"\n",
 485         )
 486         s.testVhostRedirectTokenToCookie(c, "GET",
 487                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
 488                 "?api_token=thisisabogustoken",
 489                 nil,
 490                 "",
 491                 http.StatusUnauthorized,
 492                 unauthorizedMessage+"\n",
 493         )
 494 }
 495
 496 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
 497         s.testVhostRedirectTokenToCookie(c, "GET",
 498                 "example.com/c="+arvadostest.FooCollection+"/foo",
 499                 "?api_token="+arvadostest.ActiveToken,
 500                 nil,
 501                 "",
 502                 http.StatusBadRequest,
 503                 "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n",
 504         )
 505 }
 506
 507 // If client requests an attachment by putting ?disposition=attachment
 508 // in the query string, and gets redirected, the redirect target
 509 // should respond with an attachment.
 510 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
 511         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 512                 arvadostest.FooCollection+".example.com/foo",
 513                 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
 514                 nil,
 515                 "",
 516                 http.StatusOK,
 517                 "foo",
 518         )
 519         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 520 }
 521
 522 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
 523         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 524         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 525                 "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
 526                 "?api_token="+arvadostest.ActiveToken,
 527                 nil,
 528                 "",
 529                 http.StatusOK,
 530                 "foo",
 531         )
 532         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 533 }
 534
 535 func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
 536         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 537         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 538                 "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
 539                 "?api_token="+arvadostest.ActiveToken,
 540                 nil,
 541                 "",
 542                 http.StatusOK,
 543                 "waz",
 544         )
 545         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 546         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 547                 "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz",
 548                 "?api_token="+arvadostest.ActiveToken,
 549                 nil,
 550                 "",
 551                 http.StatusOK,
 552                 "waz",
 553         )
 554         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 555 }
 556
 557 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
 558         s.handler.Cluster.Collections.TrustAllContent = true
 559         s.testVhostRedirectTokenToCookie(c, "GET",
 560                 "example.com/c="+arvadostest.FooCollection+"/foo",
 561                 "?api_token="+arvadostest.ActiveToken,
 562                 nil,
 563                 "",
 564                 http.StatusOK,
 565                 "foo",
 566         )
 567 }
 568
 569 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
 570         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
 571
 572         s.testVhostRedirectTokenToCookie(c, "GET",
 573                 "example.com/c="+arvadostest.FooCollection+"/foo",
 574                 "?api_token="+arvadostest.ActiveToken,
 575                 nil,
 576                 "",
 577                 http.StatusBadRequest,
 578                 "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n",
 579         )
 580
 581         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 582                 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
 583                 "?api_token="+arvadostest.ActiveToken,
 584                 nil,
 585                 "",
 586                 http.StatusOK,
 587                 "foo",
 588         )
 589         c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
 590 }
 591
 592 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
 593         s.testVhostRedirectTokenToCookie(c, "POST",
 594                 arvadostest.FooCollection+".example.com/foo",
 595                 "",
 596                 http.Header{"Content-Type": {"application/x-www-form-urlencoded"}},
 597                 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
 598                 http.StatusOK,
 599                 "foo",
 600         )
 601 }
 602
 603 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
 604         s.testVhostRedirectTokenToCookie(c, "POST",
 605                 arvadostest.FooCollection+".example.com/foo",
 606                 "",
 607                 http.Header{"Content-Type": {"application/x-www-form-urlencoded"}},
 608                 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
 609                 http.StatusNotFound,
 610                 notFoundMessage+"\n",
 611         )
 612 }
 613
 614 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
 615         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 616         s.testVhostRedirectTokenToCookie(c, "GET",
 617                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 618                 "",
 619                 nil,
 620                 "",
 621                 http.StatusOK,
 622                 "Hello world\n",
 623         )
 624 }
 625
 626 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
 627         s.handler.Cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
 628         s.testVhostRedirectTokenToCookie(c, "GET",
 629                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 630                 "",
 631                 nil,
 632                 "",
 633                 http.StatusNotFound,
 634                 notFoundMessage+"\n",
 635         )
 636 }
 637
 638 func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
 639         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 640
 641         client := arvados.NewClientFromEnv()
 642         client.AuthToken = arvadostest.ActiveToken
 643         fs, err := (&arvados.Collection{}).FileSystem(client, nil)
 644         c.Assert(err, check.IsNil)
 645         f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777)
 646         c.Assert(err, check.IsNil)
 647         f.Close()
 648         mtxt, err := fs.MarshalManifest(".")
 649         c.Assert(err, check.IsNil)
 650         var coll arvados.Collection
 651         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 652                 "collection": map[string]string{
 653                         "manifest_text": mtxt,
 654                 },
 655         })
 656         c.Assert(err, check.IsNil)
 657
 658         u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
 659         req := &http.Request{
 660                 Method:     "GET",
 661                 Host:       u.Host,
 662                 URL:        u,
 663                 RequestURI: u.RequestURI(),
 664                 Header: http.Header{
 665                         "Authorization": {"Bearer " + client.AuthToken},
 666                 },
 667         }
 668         resp := httptest.NewRecorder()
 669         s.handler.ServeHTTP(resp, req)
 670         c.Check(resp.Code, check.Equals, http.StatusOK)
 671         c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\&#34;odd&#39; path chars.*`)
 672 }
 673
 674 func (s *IntegrationSuite) TestForwardSlashSubstitution(c *check.C) {
 675         arv := arvados.NewClientFromEnv()
 676         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 677         s.handler.Cluster.Collections.ForwardSlashNameSubstitution = "{SOLIDUS}"
 678         name := "foo/bar/baz"
 679         nameShown := strings.Replace(name, "/", "{SOLIDUS}", -1)
 680         nameShownEscaped := strings.Replace(name, "/", "%7bSOLIDUS%7d", -1)
 681
 682         client := arvados.NewClientFromEnv()
 683         client.AuthToken = arvadostest.ActiveToken
 684         fs, err := (&arvados.Collection{}).FileSystem(client, nil)
 685         c.Assert(err, check.IsNil)
 686         f, err := fs.OpenFile("filename", os.O_CREATE, 0777)
 687         c.Assert(err, check.IsNil)
 688         f.Close()
 689         mtxt, err := fs.MarshalManifest(".")
 690         c.Assert(err, check.IsNil)
 691         var coll arvados.Collection
 692         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 693                 "collection": map[string]string{
 694                         "manifest_text": mtxt,
 695                         "name":          name,
 696                         "owner_uuid":    arvadostest.AProjectUUID,
 697                 },
 698         })
 699         c.Assert(err, check.IsNil)
 700         defer arv.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
 701
 702         base := "http://download.example.com/by_id/" + coll.OwnerUUID + "/"
 703         for tryURL, expectRegexp := range map[string]string{
 704                 base:                          `(?ms).*href="./` + nameShownEscaped + `/"\S+` + nameShown + `.*`,
 705                 base + nameShownEscaped + "/": `(?ms).*href="./filename"\S+filename.*`,
 706         } {
 707                 u, _ := url.Parse(tryURL)
 708                 req := &http.Request{
 709                         Method:     "GET",
 710                         Host:       u.Host,
 711                         URL:        u,
 712                         RequestURI: u.RequestURI(),
 713                         Header: http.Header{
 714                                 "Authorization": {"Bearer " + client.AuthToken},
 715                         },
 716                 }
 717                 resp := httptest.NewRecorder()
 718                 s.handler.ServeHTTP(resp, req)
 719                 c.Check(resp.Code, check.Equals, http.StatusOK)
 720                 c.Check(resp.Body.String(), check.Matches, expectRegexp)
 721         }
 722 }
 723
 724 // XHRs can't follow redirect-with-cookie so they rely on method=POST
 725 // and disposition=attachment (telling us it's acceptable to respond
 726 // with content instead of a redirect) and an Origin header that gets
 727 // added automatically by the browser (telling us it's desirable to do
 728 // so).
 729 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
 730         u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
 731         req := &http.Request{
 732                 Method:     "POST",
 733                 Host:       u.Host,
 734                 URL:        u,
 735                 RequestURI: u.RequestURI(),
 736                 Header: http.Header{
 737                         "Origin":       {"https://origin.example"},
 738                         "Content-Type": {"application/x-www-form-urlencoded"},
 739                 },
 740                 Body: ioutil.NopCloser(strings.NewReader(url.Values{
 741                         "api_token":   {arvadostest.ActiveToken},
 742                         "disposition": {"attachment"},
 743                 }.Encode())),
 744         }
 745         resp := httptest.NewRecorder()
 746         s.handler.ServeHTTP(resp, req)
 747         c.Check(resp.Code, check.Equals, http.StatusOK)
 748         c.Check(resp.Body.String(), check.Equals, "foo")
 749         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
 750
 751         // GET + Origin header is representative of both AJAX GET
 752         // requests and inline images via <IMG crossorigin="anonymous"
 753         // src="...">.
 754         u.RawQuery = "api_token=" + url.QueryEscape(arvadostest.ActiveTokenV2)
 755         req = &http.Request{
 756                 Method:     "GET",
 757                 Host:       u.Host,
 758                 URL:        u,
 759                 RequestURI: u.RequestURI(),
 760                 Header: http.Header{
 761                         "Origin": {"https://origin.example"},
 762                 },
 763         }
 764         resp = httptest.NewRecorder()
 765         s.handler.ServeHTTP(resp, req)
 766         c.Check(resp.Code, check.Equals, http.StatusOK)
 767         c.Check(resp.Body.String(), check.Equals, "foo")
 768         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
 769 }
 770
 771 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString string, reqHeader http.Header, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder {
 772         if reqHeader == nil {
 773                 reqHeader = http.Header{}
 774         }
 775         u, _ := url.Parse(`http://` + hostPath + queryString)
 776         c.Logf("requesting %s", u)
 777         req := &http.Request{
 778                 Method:     method,
 779                 Host:       u.Host,
 780                 URL:        u,
 781                 RequestURI: u.RequestURI(),
 782                 Header:     reqHeader,
 783                 Body:       ioutil.NopCloser(strings.NewReader(reqBody)),
 784         }
 785
 786         resp := httptest.NewRecorder()
 787         defer func() {
 788                 c.Check(resp.Code, check.Equals, expectStatus)
 789                 c.Check(resp.Body.String(), check.Equals, expectRespBody)
 790         }()
 791
 792         s.handler.ServeHTTP(resp, req)
 793         if resp.Code != http.StatusSeeOther {
 794                 return resp
 795         }
 796         c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
 797         c.Check(strings.Split(resp.Header().Get("Location"), "?")[0], check.Equals, "http://"+hostPath)
 798         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 799
 800         u, err := u.Parse(resp.Header().Get("Location"))
 801         c.Assert(err, check.IsNil)
 802         c.Logf("following redirect to %s", u)
 803         req = &http.Request{
 804                 Method:     "GET",
 805                 Host:       u.Host,
 806                 URL:        u,
 807                 RequestURI: u.RequestURI(),
 808                 Header:     reqHeader,
 809         }
 810         for _, c := range cookies {
 811                 req.AddCookie(c)
 812         }
 813
 814         resp = httptest.NewRecorder()
 815         s.handler.ServeHTTP(resp, req)
 816
 817         if resp.Code != http.StatusSeeOther {
 818                 c.Check(resp.Header().Get("Location"), check.Equals, "")
 819         }
 820         return resp
 821 }
 822
 823 func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) {
 824         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 825         s.testDirectoryListing(c)
 826 }
 827
 828 func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) {
 829         s.handler.Cluster.Users.AnonymousUserToken = ""
 830         s.testDirectoryListing(c)
 831 }
 832
 833 func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
 834         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 835         authHeader := http.Header{
 836                 "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
 837         }
 838         for _, trial := range []struct {
 839                 uri      string
 840                 header   http.Header
 841                 expect   []string
 842                 redirect string
 843                 cutDirs  int
 844         }{
 845                 {
 846                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
 847                         header:  authHeader,
 848                         expect:  []string{"dir1/foo", "dir1/bar"},
 849                         cutDirs: 0,
 850                 },
 851                 {
 852                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
 853                         header:  authHeader,
 854                         expect:  []string{"foo", "bar"},
 855                         cutDirs: 1,
 856                 },
 857                 {
 858                         // URLs of this form ignore authHeader, and
 859                         // FooAndBarFilesInDirUUID isn't public, so
 860                         // this returns 401.
 861                         uri:    "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
 862                         header: authHeader,
 863                         expect: nil,
 864                 },
 865                 {
 866                         uri:     "download.example.com/users/active/foo_file_in_dir/",
 867                         header:  authHeader,
 868                         expect:  []string{"dir1/"},
 869                         cutDirs: 3,
 870                 },
 871                 {
 872                         uri:     "download.example.com/users/active/foo_file_in_dir/dir1/",
 873                         header:  authHeader,
 874                         expect:  []string{"bar"},
 875                         cutDirs: 4,
 876                 },
 877                 {
 878                         uri:     "download.example.com/",
 879                         header:  authHeader,
 880                         expect:  []string{"users/"},
 881                         cutDirs: 0,
 882                 },
 883                 {
 884                         uri:      "download.example.com/users",
 885                         header:   authHeader,
 886                         redirect: "/users/",
 887                         expect:   []string{"active/"},
 888                         cutDirs:  1,
 889                 },
 890                 {
 891                         uri:     "download.example.com/users/",
 892                         header:  authHeader,
 893                         expect:  []string{"active/"},
 894                         cutDirs: 1,
 895                 },
 896                 {
 897                         uri:      "download.example.com/users/active",
 898                         header:   authHeader,
 899                         redirect: "/users/active/",
 900                         expect:   []string{"foo_file_in_dir/"},
 901                         cutDirs:  2,
 902                 },
 903                 {
 904                         uri:     "download.example.com/users/active/",
 905                         header:  authHeader,
 906                         expect:  []string{"foo_file_in_dir/"},
 907                         cutDirs: 2,
 908                 },
 909                 {
 910                         uri:     "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
 911                         header:  nil,
 912                         expect:  []string{"dir1/foo", "dir1/bar"},
 913                         cutDirs: 4,
 914                 },
 915                 {
 916                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
 917                         header:  nil,
 918                         expect:  []string{"dir1/foo", "dir1/bar"},
 919                         cutDirs: 2,
 920                 },
 921                 {
 922                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken,
 923                         header:  nil,
 924                         expect:  []string{"dir1/foo", "dir1/bar"},
 925                         cutDirs: 2,
 926                 },
 927                 {
 928                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID,
 929                         header:  authHeader,
 930                         expect:  []string{"dir1/foo", "dir1/bar"},
 931                         cutDirs: 1,
 932                 },
 933                 {
 934                         uri:      "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
 935                         header:   authHeader,
 936                         redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
 937                         expect:   []string{"foo", "bar"},
 938                         cutDirs:  2,
 939                 },
 940                 {
 941                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
 942                         header:  authHeader,
 943                         expect:  []string{"foo", "bar"},
 944                         cutDirs: 3,
 945                 },
 946                 {
 947                         uri:      arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
 948                         header:   authHeader,
 949                         redirect: "/dir1/",
 950                         expect:   []string{"foo", "bar"},
 951                         cutDirs:  1,
 952                 },
 953                 {
 954                         uri:    "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
 955                         header: authHeader,
 956                         expect: nil,
 957                 },
 958                 {
 959                         uri:     "download.example.com/c=" + arvadostest.WazVersion1Collection,
 960                         header:  authHeader,
 961                         expect:  []string{"waz"},
 962                         cutDirs: 1,
 963                 },
 964                 {
 965                         uri:     "download.example.com/by_id/" + arvadostest.WazVersion1Collection,
 966                         header:  authHeader,
 967                         expect:  []string{"waz"},
 968                         cutDirs: 2,
 969                 },
 970         } {
 971                 comment := check.Commentf("HTML: %q => %q", trial.uri, trial.expect)
 972                 resp := httptest.NewRecorder()
 973                 u := mustParseURL("//" + trial.uri)
 974                 req := &http.Request{
 975                         Method:     "GET",
 976                         Host:       u.Host,
 977                         URL:        u,
 978                         RequestURI: u.RequestURI(),
 979                         Header:     copyHeader(trial.header),
 980                 }
 981                 s.handler.ServeHTTP(resp, req)
 982                 var cookies []*http.Cookie
 983                 for resp.Code == http.StatusSeeOther {
 984                         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 985                         req = &http.Request{
 986                                 Method:     "GET",
 987                                 Host:       u.Host,
 988                                 URL:        u,
 989                                 RequestURI: u.RequestURI(),
 990                                 Header:     copyHeader(trial.header),
 991                         }
 992                         cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
 993                         for _, c := range cookies {
 994                                 req.AddCookie(c)
 995                         }
 996                         resp = httptest.NewRecorder()
 997                         s.handler.ServeHTTP(resp, req)
 998                 }
 999                 if trial.redirect != "" {
1000                         c.Check(req.URL.Path, check.Equals, trial.redirect, comment)
1001                 }
1002                 if trial.expect == nil {
1003                         if s.handler.Cluster.Users.AnonymousUserToken == "" {
1004                                 c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1005                         } else {
1006                                 c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
1007                         }
1008                 } else {
1009                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
1010                         for _, e := range trial.expect {
1011                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`, comment)
1012                         }
1013                         c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`, comment)
1014                 }
1015
1016                 comment = check.Commentf("WebDAV: %q => %q", trial.uri, trial.expect)
1017                 req = &http.Request{
1018                         Method:     "OPTIONS",
1019                         Host:       u.Host,
1020                         URL:        u,
1021                         RequestURI: u.RequestURI(),
1022                         Header:     copyHeader(trial.header),
1023                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
1024                 }
1025                 resp = httptest.NewRecorder()
1026                 s.handler.ServeHTTP(resp, req)
1027                 if trial.expect == nil {
1028                         if s.handler.Cluster.Users.AnonymousUserToken == "" {
1029                                 c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1030                         } else {
1031                                 c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
1032                         }
1033                 } else {
1034                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
1035                 }
1036
1037                 req = &http.Request{
1038                         Method:     "PROPFIND",
1039                         Host:       u.Host,
1040                         URL:        u,
1041                         RequestURI: u.RequestURI(),
1042                         Header:     copyHeader(trial.header),
1043                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
1044                 }
1045                 resp = httptest.NewRecorder()
1046                 s.handler.ServeHTTP(resp, req)
1047                 if trial.expect == nil {
1048                         if s.handler.Cluster.Users.AnonymousUserToken == "" {
1049                                 c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1050                         } else {
1051                                 c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
1052                         }
1053                 } else {
1054                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment)
1055                         for _, e := range trial.expect {
1056                                 if strings.HasSuffix(e, "/") {
1057                                         e = filepath.Join(u.Path, e) + "/"
1058                                 } else {
1059                                         e = filepath.Join(u.Path, e)
1060                                 }
1061                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+e+`</D:href>.*`, comment)
1062                         }
1063                 }
1064         }
1065 }
1066
1067 func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
1068         arv := arvados.NewClientFromEnv()
1069         var newCollection arvados.Collection
1070         err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1071                 "collection": map[string]string{
1072                         "owner_uuid":    arvadostest.ActiveUserUUID,
1073                         "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
1074                         "name":          "keep-web test collection",
1075                 },
1076                 "ensure_unique_name": true,
1077         })
1078         c.Assert(err, check.IsNil)
1079         defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
1080
1081         var updated arvados.Collection
1082         for _, fnm := range []string{"foo.txt", "bar.txt"} {
1083                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
1084                 u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
1085                 req := &http.Request{
1086                         Method:     "DELETE",
1087                         Host:       u.Host,
1088                         URL:        u,
1089                         RequestURI: u.RequestURI(),
1090                         Header: http.Header{
1091                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
1092                         },
1093                 }
1094                 resp := httptest.NewRecorder()
1095                 s.handler.ServeHTTP(resp, req)
1096                 c.Check(resp.Code, check.Equals, http.StatusNoContent)
1097
1098                 updated = arvados.Collection{}
1099                 err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
1100                 c.Check(err, check.IsNil)
1101                 c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`)
1102                 c.Logf("updated manifest_text %q", updated.ManifestText)
1103         }
1104         c.Check(updated.ManifestText, check.Equals, "")
1105 }
1106
1107 func (s *IntegrationSuite) TestFileContentType(c *check.C) {
1108         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1109
1110         client := arvados.NewClientFromEnv()
1111         client.AuthToken = arvadostest.ActiveToken
1112         arv, err := arvadosclient.New(client)
1113         c.Assert(err, check.Equals, nil)
1114         kc, err := keepclient.MakeKeepClient(arv)
1115         c.Assert(err, check.Equals, nil)
1116
1117         fs, err := (&arvados.Collection{}).FileSystem(client, kc)
1118         c.Assert(err, check.IsNil)
1119
1120         trials := []struct {
1121                 filename    string
1122                 content     string
1123                 contentType string
1124         }{
1125                 {"picture.txt", "BMX bikes are small this year\n", "text/plain; charset=utf-8"},
1126                 {"picture.bmp", "BMX bikes are small this year\n", "image/(x-ms-)?bmp"},
1127                 {"picture.jpg", "BMX bikes are small this year\n", "image/jpeg"},
1128                 {"picture1", "BMX bikes are small this year\n", "image/bmp"},            // content sniff; "BM" is the magic signature for .bmp
1129                 {"picture2", "Cars are small this year\n", "text/plain; charset=utf-8"}, // content sniff
1130         }
1131         for _, trial := range trials {
1132                 f, err := fs.OpenFile(trial.filename, os.O_CREATE|os.O_WRONLY, 0777)
1133                 c.Assert(err, check.IsNil)
1134                 _, err = f.Write([]byte(trial.content))
1135                 c.Assert(err, check.IsNil)
1136                 c.Assert(f.Close(), check.IsNil)
1137         }
1138         mtxt, err := fs.MarshalManifest(".")
1139         c.Assert(err, check.IsNil)
1140         var coll arvados.Collection
1141         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1142                 "collection": map[string]string{
1143                         "manifest_text": mtxt,
1144                 },
1145         })
1146         c.Assert(err, check.IsNil)
1147
1148         for _, trial := range trials {
1149                 u, _ := url.Parse("http://download.example.com/by_id/" + coll.UUID + "/" + trial.filename)
1150                 req := &http.Request{
1151                         Method:     "GET",
1152                         Host:       u.Host,
1153                         URL:        u,
1154                         RequestURI: u.RequestURI(),
1155                         Header: http.Header{
1156                                 "Authorization": {"Bearer " + client.AuthToken},
1157                         },
1158                 }
1159                 resp := httptest.NewRecorder()
1160                 s.handler.ServeHTTP(resp, req)
1161                 c.Check(resp.Code, check.Equals, http.StatusOK)
1162                 c.Check(resp.Header().Get("Content-Type"), check.Matches, trial.contentType)
1163                 c.Check(resp.Body.String(), check.Equals, trial.content)
1164         }
1165 }
1166
1167 func (s *IntegrationSuite) TestKeepClientBlockCache(c *check.C) {
1168         s.handler.Cluster.Collections.WebDAVCache.MaxBlockEntries = 42
1169         c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Not(check.Equals), 42)
1170         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/t=" + arvadostest.ActiveToken + "/foo")
1171         req := &http.Request{
1172                 Method:     "GET",
1173                 Host:       u.Host,
1174                 URL:        u,
1175                 RequestURI: u.RequestURI(),
1176         }
1177         resp := httptest.NewRecorder()
1178         s.handler.ServeHTTP(resp, req)
1179         c.Check(resp.Code, check.Equals, http.StatusOK)
1180         c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Equals, 42)
1181 }
1182
1183 // Writing to a collection shouldn't affect its entry in the
1184 // PDH-to-manifest cache.
1185 func (s *IntegrationSuite) TestCacheWriteCollectionSamePDH(c *check.C) {
1186         arv, err := arvadosclient.MakeArvadosClient()
1187         c.Assert(err, check.Equals, nil)
1188         arv.ApiToken = arvadostest.ActiveToken
1189
1190         u := mustParseURL("http://x.example/testfile")
1191         req := &http.Request{
1192                 Method:     "GET",
1193                 Host:       u.Host,
1194                 URL:        u,
1195                 RequestURI: u.RequestURI(),
1196                 Header:     http.Header{"Authorization": {"Bearer " + arv.ApiToken}},
1197         }
1198
1199         checkWithID := func(id string, status int) {
1200                 req.URL.Host = strings.Replace(id, "+", "-", -1) + ".example"
1201                 req.Host = req.URL.Host
1202                 resp := httptest.NewRecorder()
1203                 s.handler.ServeHTTP(resp, req)
1204                 c.Check(resp.Code, check.Equals, status)
1205         }
1206
1207         var colls [2]arvados.Collection
1208         for i := range colls {
1209                 err := arv.Create("collections",
1210                         map[string]interface{}{
1211                                 "ensure_unique_name": true,
1212                                 "collection": map[string]interface{}{
1213                                         "name": "test collection",
1214                                 },
1215                         }, &colls[i])
1216                 c.Assert(err, check.Equals, nil)
1217         }
1218
1219         // Populate cache with empty collection
1220         checkWithID(colls[0].PortableDataHash, http.StatusNotFound)
1221
1222         // write a file to colls[0]
1223         reqPut := *req
1224         reqPut.Method = "PUT"
1225         reqPut.URL.Host = colls[0].UUID + ".example"
1226         reqPut.Host = req.URL.Host
1227         reqPut.Body = ioutil.NopCloser(bytes.NewBufferString("testdata"))
1228         resp := httptest.NewRecorder()
1229         s.handler.ServeHTTP(resp, &reqPut)
1230         c.Check(resp.Code, check.Equals, http.StatusCreated)
1231
1232         // new file should not appear in colls[1]
1233         checkWithID(colls[1].PortableDataHash, http.StatusNotFound)
1234         checkWithID(colls[1].UUID, http.StatusNotFound)
1235
1236         checkWithID(colls[0].UUID, http.StatusOK)
1237 }
1238
1239 func copyHeader(h http.Header) http.Header {
1240         hc := http.Header{}
1241         for k, v := range h {
1242                 hc[k] = append([]string(nil), v...)
1243         }
1244         return hc
1245 }
1246
1247 func (s *IntegrationSuite) checkUploadDownloadRequest(c *check.C, req *http.Request,
1248         successCode int, direction string, perm bool, userUuid string, collectionUuid string, filepath string) {
1249
1250         client := arvados.NewClientFromEnv()
1251         client.AuthToken = arvadostest.AdminToken
1252         var logentries arvados.LogList
1253         limit1 := 1
1254         err := client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil,
1255                 arvados.ResourceListParams{
1256                         Limit: &limit1,
1257                         Order: "created_at desc"})
1258         c.Check(err, check.IsNil)
1259         c.Check(logentries.Items, check.HasLen, 1)
1260         lastLogId := logentries.Items[0].ID
1261
1262         var logbuf bytes.Buffer
1263         logger := logrus.New()
1264         logger.Out = &logbuf
1265         resp := httptest.NewRecorder()
1266         req = req.WithContext(ctxlog.Context(context.Background(), logger))
1267         s.handler.ServeHTTP(resp, req)
1268
1269         if perm {
1270                 c.Check(resp.Result().StatusCode, check.Equals, successCode)
1271                 c.Check(logbuf.String(), check.Matches, `(?ms).*msg="File `+direction+`".*`)
1272                 c.Check(logbuf.String(), check.Not(check.Matches), `(?ms).*level=error.*`)
1273
1274                 deadline := time.Now().Add(time.Second)
1275                 for {
1276                         c.Assert(time.Now().After(deadline), check.Equals, false, check.Commentf("timed out waiting for log entry"))
1277                         err = client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil,
1278                                 arvados.ResourceListParams{
1279                                         Filters: []arvados.Filter{
1280                                                 {Attr: "event_type", Operator: "=", Operand: "file_" + direction},
1281                                                 {Attr: "object_uuid", Operator: "=", Operand: userUuid},
1282                                         },
1283                                         Limit: &limit1,
1284                                         Order: "created_at desc",
1285                                 })
1286                         c.Assert(err, check.IsNil)
1287                         if len(logentries.Items) > 0 &&
1288                                 logentries.Items[0].ID > lastLogId &&
1289                                 logentries.Items[0].ObjectUUID == userUuid &&
1290                                 logentries.Items[0].Properties["collection_uuid"] == collectionUuid &&
1291                                 logentries.Items[0].Properties["collection_file_path"] == filepath {
1292                                 break
1293                         }
1294                         c.Logf("logentries.Items: %+v", logentries.Items)
1295                         time.Sleep(50 * time.Millisecond)
1296                 }
1297         } else {
1298                 c.Check(resp.Result().StatusCode, check.Equals, http.StatusForbidden)
1299                 c.Check(logbuf.String(), check.Equals, "")
1300         }
1301 }
1302
1303 func (s *IntegrationSuite) TestDownloadLoggingPermission(c *check.C) {
1304         u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
1305
1306         s.handler.Cluster.Collections.TrustAllContent = true
1307
1308         for _, adminperm := range []bool{true, false} {
1309                 for _, userperm := range []bool{true, false} {
1310                         s.handler.Cluster.Collections.WebDAVPermission.Admin.Download = adminperm
1311                         s.handler.Cluster.Collections.WebDAVPermission.User.Download = userperm
1312
1313                         // Test admin permission
1314                         req := &http.Request{
1315                                 Method:     "GET",
1316                                 Host:       u.Host,
1317                                 URL:        u,
1318                                 RequestURI: u.RequestURI(),
1319                                 Header: http.Header{
1320                                         "Authorization": {"Bearer " + arvadostest.AdminToken},
1321                                 },
1322                         }
1323                         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", adminperm,
1324                                 arvadostest.AdminUserUUID, arvadostest.FooCollection, "foo")
1325
1326                         // Test user permission
1327                         req = &http.Request{
1328                                 Method:     "GET",
1329                                 Host:       u.Host,
1330                                 URL:        u,
1331                                 RequestURI: u.RequestURI(),
1332                                 Header: http.Header{
1333                                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
1334                                 },
1335                         }
1336                         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", userperm,
1337                                 arvadostest.ActiveUserUUID, arvadostest.FooCollection, "foo")
1338                 }
1339         }
1340
1341         s.handler.Cluster.Collections.WebDAVPermission.User.Download = true
1342
1343         for _, tryurl := range []string{"http://" + arvadostest.MultilevelCollection1 + ".keep-web.example/dir1/subdir/file1",
1344                 "http://keep-web/users/active/multilevel_collection_1/dir1/subdir/file1"} {
1345
1346                 u = mustParseURL(tryurl)
1347                 req := &http.Request{
1348                         Method:     "GET",
1349                         Host:       u.Host,
1350                         URL:        u,
1351                         RequestURI: u.RequestURI(),
1352                         Header: http.Header{
1353                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
1354                         },
1355                 }
1356                 s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true,
1357                         arvadostest.ActiveUserUUID, arvadostest.MultilevelCollection1, "dir1/subdir/file1")
1358         }
1359
1360         u = mustParseURL("http://" + strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + ".keep-web.example/foo")
1361         req := &http.Request{
1362                 Method:     "GET",
1363                 Host:       u.Host,
1364                 URL:        u,
1365                 RequestURI: u.RequestURI(),
1366                 Header: http.Header{
1367                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
1368                 },
1369         }
1370         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true,
1371                 arvadostest.ActiveUserUUID, arvadostest.FooCollection, "foo")
1372 }
1373
1374 func (s *IntegrationSuite) TestUploadLoggingPermission(c *check.C) {
1375         for _, adminperm := range []bool{true, false} {
1376                 for _, userperm := range []bool{true, false} {
1377
1378                         arv := arvados.NewClientFromEnv()
1379                         arv.AuthToken = arvadostest.ActiveToken
1380
1381                         var coll arvados.Collection
1382                         err := arv.RequestAndDecode(&coll,
1383                                 "POST",
1384                                 "/arvados/v1/collections",
1385                                 nil,
1386                                 map[string]interface{}{
1387                                         "ensure_unique_name": true,
1388                                         "collection": map[string]interface{}{
1389                                                 "name": "test collection",
1390                                         },
1391                                 })
1392                         c.Assert(err, check.Equals, nil)
1393
1394                         u := mustParseURL("http://" + coll.UUID + ".keep-web.example/bar")
1395
1396                         s.handler.Cluster.Collections.WebDAVPermission.Admin.Upload = adminperm
1397                         s.handler.Cluster.Collections.WebDAVPermission.User.Upload = userperm
1398
1399                         // Test admin permission
1400                         req := &http.Request{
1401                                 Method:     "PUT",
1402                                 Host:       u.Host,
1403                                 URL:        u,
1404                                 RequestURI: u.RequestURI(),
1405                                 Header: http.Header{
1406                                         "Authorization": {"Bearer " + arvadostest.AdminToken},
1407                                 },
1408                                 Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
1409                         }
1410                         s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", adminperm,
1411                                 arvadostest.AdminUserUUID, coll.UUID, "bar")
1412
1413                         // Test user permission
1414                         req = &http.Request{
1415                                 Method:     "PUT",
1416                                 Host:       u.Host,
1417                                 URL:        u,
1418                                 RequestURI: u.RequestURI(),
1419                                 Header: http.Header{
1420                                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
1421                                 },
1422                                 Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
1423                         }
1424                         s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", userperm,
1425                                 arvadostest.ActiveUserUUID, coll.UUID, "bar")
1426                 }
1427         }
1428 }